From add23ab3f60ce2397d7faa0726759c08ec864aa9 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Wed, 26 Aug 2026 10:17:55 -0400 Subject: [PATCH 1/4] add json serialization format for tables in ska_api --- kadi_apps/blueprints/ska_api/api.py | 22 ++++++++----- kadi_apps/tests/test_ska_api.py | 48 ++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/kadi_apps/blueprints/ska_api/api.py b/kadi_apps/blueprints/ska_api/api.py index 590be26..7f740be 100644 --- a/kadi_apps/blueprints/ska_api/api.py +++ b/kadi_apps/blueprints/ska_api/api.py @@ -1,9 +1,8 @@ -from flask import Blueprint -from flask import request - -import logging import json +import logging +from flask import Blueprint, request +from Quaternion import Quat APPS = { ('agasc',): ['get_star', 'get_stars', 'get_agasc_cone'], @@ -140,14 +139,21 @@ def __init__(self, table_format=None, strict_encode=True, **kwargs): self.strict_encode = strict_encode def encode_table(self, obj): - if self.table_format not in ('rows', 'columns'): + if self.table_format not in ('rows', 'columns', 'json'): raise ValueError('table_format={} not allowed'.format(self.table_format)) obj = _replace_object_cols_with_str(obj) out = {name: obj[name].tolist() for name in obj.colnames} - if self.table_format == 'rows': + if self.table_format == 'json': + out = { + "meta": obj.meta, + "class_name": type(obj).__name__, + "full_name": f"{obj.__module__}.{type(obj).__name__}", + "columns": out, + } + elif self.table_format == 'rows': # Convert from dict of list to list of dict out = [{name: out[name][ii] for name in obj.colnames} for ii in range(len(obj))] @@ -155,8 +161,8 @@ def encode_table(self, obj): return out def default(self, obj): - from astropy.table import Table import numpy as np + from astropy.table import Table # Potentially convert something with a `table` property to an astropy Table. if hasattr(obj, 'table') and isinstance(obj.__class__.table, property): @@ -185,6 +191,8 @@ def default(self, obj): elif isinstance(obj, bytes): return obj.decode('utf-8') + elif isinstance(obj, Quat): + return obj.q.tolist() else: try: out = super(APIEncoder, self).default(obj) diff --git a/kadi_apps/tests/test_ska_api.py b/kadi_apps/tests/test_ska_api.py index c90bc08..142083f 100644 --- a/kadi_apps/tests/test_ska_api.py +++ b/kadi_apps/tests/test_ska_api.py @@ -1,10 +1,9 @@ +import re + import numpy as np import requests -import re from astropy.table import Table - -from kadi.commands import get_starcats -from kadi.commands import get_observations +from kadi.commands import get_observations, get_starcats from kadi_apps.blueprints.ska_api.api import _replace_object_cols_with_str @@ -199,6 +198,47 @@ def test_starcats(test_server): assert np.all(starcats[i] == starcats_api[i]) +def _interpret_aca_table_(json): + from proseco.catalog import ACATable, AcqTable, GuideTable + from Quaternion import Quat + meta = json["meta"].copy() + meta["acqs"] = AcqTable(meta["acqs"]["columns"], meta=meta["acqs"]["meta"]) + meta["guides"] = GuideTable(meta["guides"]["columns"], meta=meta["guides"]["meta"]) + meta["att"] = Quat(meta["att"]) + starcat = ACATable(json["columns"], meta=meta) + return starcat + + +def test_starcats_json(test_server): + api_url = f"{test_server['url']}/ska_api" + start = '2022:001' + stop = '2022:002' + obsid = None + starcats = get_starcats(start=start, stop=stop, obsid=obsid, scenario='flight') + url = f'{api_url}/kadi/commands/get_starcats?{start=}&{stop=}&scenario=flight&table_format=json' + r = requests.get(url) + starcats_api = [_interpret_aca_table_(cat) for cat in r.json()] + colnames = [ + 'slot', 'idx', 'id', 'type', 'sz', 'mag', 'maxmag', 'yang', 'zang', 'dim', 'res', 'halfw' + ] + for i in range(len(starcats)): + sc = starcats[i] + sc_api = starcats_api[i] + for col in colnames: + assert np.all(sc[col] == sc_api[col]) + assert np.all(sc == sc_api), f"starcat {sc.date} data does not match API output" + + assert np.all(sc.meta['acqs'] == sc_api.meta['acqs']), f"starcat {sc.date} acqs does not match API output" + assert np.all(sc.meta['guides'] == sc_api.meta['guides']), f"starcat {sc.date} guides does not match API output" + assert sc.date == sc_api.date, f"starcat {sc.date} date does not match API output" + assert sc.obsid == sc_api.obsid, f"starcat {sc.date} obsid does not match API output" + assert sc.duration == sc_api.duration, f"starcat {sc.date} duration does not match API output" + assert sc.detector == sc_api.detector, f"starcat {sc.date} detector does not match API output" + assert sc.sim_offset == sc_api.sim_offset, f"starcat {sc.date} sim_offset does not match API output" + assert sc.t_ccd_guide == sc_api.t_ccd_guide, f"starcat {sc.date} t_ccd_guide does not match API output" + assert sc.t_ccd_acq == sc_api.t_ccd_acq, f"starcat {sc.date} t_ccd_acq does not match API output" + + def test_observations(test_server): api_url = f"{test_server['url']}/ska_api" starcat_date = '2022:001:17:00:58.521' From c25ac2098d18246c77ef7b68422b24157839bf46 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Wed, 26 Aug 2026 10:17:56 -0400 Subject: [PATCH 2/4] Document json table_format and tighten its unit test --- kadi_apps/templates/api_index.html | 6 ++++-- kadi_apps/tests/test_ska_api.py | 11 ++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/kadi_apps/templates/api_index.html b/kadi_apps/templates/api_index.html index bb19d17..b35414c 100644 --- a/kadi_apps/templates/api_index.html +++ b/kadi_apps/templates/api_index.html @@ -52,10 +52,12 @@

Data format

(Content-type: application/json).

Table format

One special option which is common to all queries is the table_format, which can take the value -rows or columns (default=``rows``). This specifies whether to return any tabular data as +rows, columns or json (default=``rows``). This specifies whether to return any tabular data as either a list of dicts (rows) or a dict of lists (columns). For large query results the columns option will generally be more compact because the table column names are not repeated for -every row. For example:

+every row. The json option returns a dict with the columns (as a dict of lists) under the +columns key, and also includes the table metadata under the meta key together with the +table class name. For example:

{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}/kadi?start=2019:001&stop=2019:002&table_format=columns

Available entrypoints

diff --git a/kadi_apps/tests/test_ska_api.py b/kadi_apps/tests/test_ska_api.py index 142083f..3acdcba 100644 --- a/kadi_apps/tests/test_ska_api.py +++ b/kadi_apps/tests/test_ska_api.py @@ -198,7 +198,7 @@ def test_starcats(test_server): assert np.all(starcats[i] == starcats_api[i]) -def _interpret_aca_table_(json): +def _interpret_aca_table(json): from proseco.catalog import ACATable, AcqTable, GuideTable from Quaternion import Quat meta = json["meta"].copy() @@ -217,15 +217,12 @@ def test_starcats_json(test_server): starcats = get_starcats(start=start, stop=stop, obsid=obsid, scenario='flight') url = f'{api_url}/kadi/commands/get_starcats?{start=}&{stop=}&scenario=flight&table_format=json' r = requests.get(url) - starcats_api = [_interpret_aca_table_(cat) for cat in r.json()] - colnames = [ - 'slot', 'idx', 'id', 'type', 'sz', 'mag', 'maxmag', 'yang', 'zang', 'dim', 'res', 'halfw' - ] + assert r.ok + starcats_api = [_interpret_aca_table(cat) for cat in r.json()] + assert len(starcats) == len(starcats_api) for i in range(len(starcats)): sc = starcats[i] sc_api = starcats_api[i] - for col in colnames: - assert np.all(sc[col] == sc_api[col]) assert np.all(sc == sc_api), f"starcat {sc.date} data does not match API output" assert np.all(sc.meta['acqs'] == sc_api.meta['acqs']), f"starcat {sc.date} acqs does not match API output" From 34058c115e0eba47a4a16e3ab460e3b3b16acfe2 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Wed, 26 Aug 2026 10:44:31 -0400 Subject: [PATCH 3/4] Rename json table_format to full and add class info to Quat encoding --- kadi_apps/blueprints/ska_api/api.py | 13 ++++++++++--- kadi_apps/templates/api_index.html | 4 ++-- kadi_apps/tests/test_ska_api.py | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/kadi_apps/blueprints/ska_api/api.py b/kadi_apps/blueprints/ska_api/api.py index 7f740be..b57341f 100644 --- a/kadi_apps/blueprints/ska_api/api.py +++ b/kadi_apps/blueprints/ska_api/api.py @@ -139,18 +139,18 @@ def __init__(self, table_format=None, strict_encode=True, **kwargs): self.strict_encode = strict_encode def encode_table(self, obj): - if self.table_format not in ('rows', 'columns', 'json'): + if self.table_format not in ('rows', 'columns', 'full'): raise ValueError('table_format={} not allowed'.format(self.table_format)) obj = _replace_object_cols_with_str(obj) out = {name: obj[name].tolist() for name in obj.colnames} - if self.table_format == 'json': + if self.table_format == 'full': out = { "meta": obj.meta, "class_name": type(obj).__name__, - "full_name": f"{obj.__module__}.{type(obj).__name__}", + "full_name": f"{type(obj).__module__}.{type(obj).__name__}", "columns": out, } elif self.table_format == 'rows': @@ -192,7 +192,14 @@ def default(self, obj): return obj.decode('utf-8') elif isinstance(obj, Quat): + if self.table_format == 'full': + return { + "class_name": type(obj).__name__, + "full_name": f"{type(obj).__module__}.{type(obj).__name__}", + "q": obj.q.tolist(), + } return obj.q.tolist() + else: try: out = super(APIEncoder, self).default(obj) diff --git a/kadi_apps/templates/api_index.html b/kadi_apps/templates/api_index.html index b35414c..9dd6b72 100644 --- a/kadi_apps/templates/api_index.html +++ b/kadi_apps/templates/api_index.html @@ -52,10 +52,10 @@

Data format

(Content-type: application/json).

Table format

One special option which is common to all queries is the table_format, which can take the value -rows, columns or json (default=``rows``). This specifies whether to return any tabular data as +rows, columns or full (default=``rows``). This specifies whether to return any tabular data as either a list of dicts (rows) or a dict of lists (columns). For large query results the columns option will generally be more compact because the table column names are not repeated for -every row. The json option returns a dict with the columns (as a dict of lists) under the +every row. The full option returns a dict with the columns (as a dict of lists) under the columns key, and also includes the table metadata under the meta key together with the table class name. For example:

{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}/kadi?start=2019:001&stop=2019:002&table_format=columns

diff --git a/kadi_apps/tests/test_ska_api.py b/kadi_apps/tests/test_ska_api.py index 3acdcba..3e3f46d 100644 --- a/kadi_apps/tests/test_ska_api.py +++ b/kadi_apps/tests/test_ska_api.py @@ -204,18 +204,18 @@ def _interpret_aca_table(json): meta = json["meta"].copy() meta["acqs"] = AcqTable(meta["acqs"]["columns"], meta=meta["acqs"]["meta"]) meta["guides"] = GuideTable(meta["guides"]["columns"], meta=meta["guides"]["meta"]) - meta["att"] = Quat(meta["att"]) + meta["att"] = Quat(meta["att"]["q"]) starcat = ACATable(json["columns"], meta=meta) return starcat -def test_starcats_json(test_server): +def test_starcats_full(test_server): api_url = f"{test_server['url']}/ska_api" start = '2022:001' stop = '2022:002' obsid = None starcats = get_starcats(start=start, stop=stop, obsid=obsid, scenario='flight') - url = f'{api_url}/kadi/commands/get_starcats?{start=}&{stop=}&scenario=flight&table_format=json' + url = f'{api_url}/kadi/commands/get_starcats?{start=}&{stop=}&scenario=flight&table_format=full' r = requests.get(url) assert r.ok starcats_api = [_interpret_aca_table(cat) for cat in r.json()] From eff237237878f648f3c2ef33a5f18efcb2b48387 Mon Sep 17 00:00:00 2001 From: Javier Gonzalez Date: Thu, 10 Sep 2026 13:55:10 -0400 Subject: [PATCH 4/4] Encode any numpy scalar, Quantity and Time, and mark unserializable meta --- kadi_apps/blueprints/ska_api/api.py | 84 ++++++++++++++++++----- kadi_apps/templates/api_index.html | 4 +- kadi_apps/tests/test_ska_api.py | 100 ++++++++++++++++++++++++++-- 3 files changed, 167 insertions(+), 21 deletions(-) diff --git a/kadi_apps/blueprints/ska_api/api.py b/kadi_apps/blueprints/ska_api/api.py index b57341f..4863c16 100644 --- a/kadi_apps/blueprints/ska_api/api.py +++ b/kadi_apps/blueprints/ska_api/api.py @@ -132,6 +132,17 @@ def _replace_object_cols_with_str(tbl): return tbl +def _class_info(obj): + """Class name and fully-qualified class name of ``obj`` + + :returns: dict + """ + return { + "class_name": type(obj).__name__, + "full_name": f"{type(obj).__module__}.{type(obj).__name__}", + } + + class APIEncoder(json.JSONEncoder): def __init__(self, table_format=None, strict_encode=True, **kwargs): self.table_format = table_format or 'rows' @@ -148,9 +159,8 @@ def encode_table(self, obj): if self.table_format == 'full': out = { - "meta": obj.meta, - "class_name": type(obj).__name__, - "full_name": f"{type(obj).__module__}.{type(obj).__name__}", + "meta": self._json_safe(obj.meta), + **_class_info(obj), "columns": out, } elif self.table_format == 'rows': @@ -160,9 +170,45 @@ def encode_table(self, obj): return out + def encode_object(self, obj, plain, full): + """Encode a non-table object, adding class info for table_format='full' + + :param obj: the object being encoded + :param plain: the value to use for the 'rows' and 'columns' formats + :param full: dict of the values to include along with the class info + :returns: dict or ``plain`` + """ + if self.table_format == 'full': + return {**_class_info(obj), **full} + return plain + + def _json_safe(self, obj): + """Replace values within ``obj`` that cannot be JSON encoded with a marker + + Dicts and lists are traversed so only the offending value is replaced. + + :returns: the original object or a copy with unserializable values replaced + """ + if isinstance(obj, dict): + return {key: self._json_safe(val) for key, val in obj.items()} + + if isinstance(obj, (list, tuple)): + return [self._json_safe(val) for val in obj] + + try: + self.encode(obj) + except Exception: + # Note this gives no detail about the value, but unlike repr() it is stable + # (a repr often includes the memory address). + return {"__unserializable__": f"{type(obj).__module__}.{type(obj).__name__}"} + + return obj + def default(self, obj): import numpy as np from astropy.table import Table + from astropy.time import Time + from astropy.units import Quantity # Potentially convert something with a `table` property to an astropy Table. if hasattr(obj, 'table') and isinstance(obj.__class__.table, property): @@ -170,11 +216,9 @@ def default(self, obj): if isinstance(obj_table, Table): obj = obj_table - if type(obj) in [np.int32, np.int64]: - return int(obj) - - elif type(obj) in [np.float32, np.float64]: - return float(obj) + if isinstance(obj, np.generic): + # Any numpy scalar: bool_, all int/uint/float sizes and str_. + return obj.item() elif isinstance(obj, np.ma.MaskedArray): return { @@ -182,23 +226,33 @@ def default(self, obj): 'mask': obj.mask.tolist() } + elif isinstance(obj, Quantity): + # This must come before ndarray because Quantity is an ndarray subclass + # whose tolist() raises NotImplementedError. + value = obj.value.tolist() + return self.encode_object( + obj, value, {"value": value, "unit": obj.unit.to_string()} + ) + elif isinstance(obj, np.ndarray): return obj.tolist() elif isinstance(obj, Table): return self.encode_table(obj) + elif isinstance(obj, Time): + return self.encode_object( + obj, + obj.value, + {"value": obj.value, "format": obj.format, "scale": obj.scale}, + ) + elif isinstance(obj, bytes): return obj.decode('utf-8') elif isinstance(obj, Quat): - if self.table_format == 'full': - return { - "class_name": type(obj).__name__, - "full_name": f"{type(obj).__module__}.{type(obj).__name__}", - "q": obj.q.tolist(), - } - return obj.q.tolist() + q = obj.q.tolist() + return self.encode_object(obj, q, {"q": q}) else: try: diff --git a/kadi_apps/templates/api_index.html b/kadi_apps/templates/api_index.html index 9dd6b72..4e195f6 100644 --- a/kadi_apps/templates/api_index.html +++ b/kadi_apps/templates/api_index.html @@ -57,7 +57,9 @@

Table format

columns option will generally be more compact because the table column names are not repeated for every row. The full option returns a dict with the columns (as a dict of lists) under the columns key, and also includes the table metadata under the meta key together with the -table class name. For example:

+table class name. Metadata values that cannot be represented as JSON (for example an internal +object reference) are replaced by a __unserializable__ marker giving the class +that was dropped. For example:

{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}/kadi?start=2019:001&stop=2019:002&table_format=columns

Available entrypoints

diff --git a/kadi_apps/tests/test_ska_api.py b/kadi_apps/tests/test_ska_api.py index 3e3f46d..ecbf8a3 100644 --- a/kadi_apps/tests/test_ska_api.py +++ b/kadi_apps/tests/test_ska_api.py @@ -1,11 +1,17 @@ +import json import re +import astropy.units as u import numpy as np +import pytest import requests from astropy.table import Table +from astropy.time import Time +from cxotime import CxoTime from kadi.commands import get_observations, get_starcats +from Quaternion import Quat -from kadi_apps.blueprints.ska_api.api import _replace_object_cols_with_str +from kadi_apps.blueprints.ska_api.api import APIEncoder, _replace_object_cols_with_str def test_agasc_star(test_server): @@ -198,14 +204,13 @@ def test_starcats(test_server): assert np.all(starcats[i] == starcats_api[i]) -def _interpret_aca_table(json): +def _interpret_aca_table(data): from proseco.catalog import ACATable, AcqTable, GuideTable - from Quaternion import Quat - meta = json["meta"].copy() + meta = data["meta"].copy() meta["acqs"] = AcqTable(meta["acqs"]["columns"], meta=meta["acqs"]["meta"]) meta["guides"] = GuideTable(meta["guides"]["columns"], meta=meta["guides"]["meta"]) meta["att"] = Quat(meta["att"]["q"]) - starcat = ACATable(json["columns"], meta=meta) + starcat = ACATable(data["columns"], meta=meta) return starcat @@ -236,6 +241,91 @@ def test_starcats_full(test_server): assert sc.t_ccd_acq == sc_api.t_ccd_acq, f"starcat {sc.date} t_ccd_acq does not match API output" +def _encode_meta_value(val, table_format='full'): + """Put ``val`` in the meta of a table and return its encoded form""" + tbl = Table({'a': [1]}) + tbl.meta['val'] = val + out = json.loads(APIEncoder(table_format=table_format).encode(tbl)) + return out['meta']['val'] + + +@pytest.mark.parametrize( + 'val,expected', + [ + (np.bool_(True), True), + (np.int8(-1), -1), + (np.int16(-2), -2), + (np.int32(-3), -3), + (np.int64(-4), -4), + (np.uint8(1), 1), + (np.uint16(2), 2), + (np.uint32(3), 3), + (np.uint64(4), 4), + (np.float16(1.5), 1.5), + (np.float32(2.5), 2.5), + (np.float64(3.5), 3.5), + (np.str_('abc'), 'abc'), + ] +) +def test_encode_numpy_scalars(val, expected): + """Numpy scalars of any size or kind are encoded as the native Python value""" + assert _encode_meta_value(val) == expected + + +def test_encode_quantity(): + assert _encode_meta_value(3.0 * u.deg) == { + 'class_name': 'Quantity', + 'full_name': 'astropy.units.quantity.Quantity', + 'value': 3.0, + 'unit': 'deg', + } + assert _encode_meta_value([1.0, 2.0] * u.deg)['value'] == [1.0, 2.0] + + +def test_encode_time(): + """A Time is encoded with what is needed to reconstruct it""" + time = CxoTime('2022:001:12:00:00.000') + out = _encode_meta_value(time) + assert out['class_name'] == 'CxoTime' + assert CxoTime(out['value'], format=out['format'], scale=out['scale']) == time + assert _encode_meta_value(Time('2022-01-01'))['class_name'] == 'Time' + + +def test_encode_object_class_info_only_for_full(): + """Class info is included in the 'full' format only""" + quat = Quat([0, 0, 0, 1]) + assert _encode_meta_value(quat)['q'] == [0.0, 0.0, 0.0, 1.0] + # The other formats do not include meta at all, so encode the Quat directly. + for table_format in ('rows', 'columns'): + encoded = APIEncoder(table_format=table_format).encode(quat) + assert json.loads(encoded) == [0.0, 0.0, 0.0, 1.0] + + +def test_encode_unserializable_meta(): + """A meta value that cannot be encoded is replaced by a stable marker""" + tbl = Table({'a': [1]}) + tbl.meta['good'] = 1 + tbl.meta['nested'] = {'bad': object()} + out = json.loads(APIEncoder(table_format='full').encode(tbl)) + assert out['meta']['good'] == 1 + assert out['meta']['nested']['bad'] == {'__unserializable__': 'builtins.object'} + + +def test_cmds_full(test_server): + """get_cmds meta holds a weakref, which must not fail the request""" + api_url = f"{test_server['url']}/ska_api" + start = '2022:001' + stop = '2022:002' + url = f'{api_url}/kadi/commands/get_cmds?{start=}&{stop=}&scenario=flight&table_format=full' + r = requests.get(url) + assert r.ok + out = r.json() + assert out['class_name'] == 'CommandTable' + assert len(out['columns']['idx']) > 0 + rev_pars_dict = out['meta']['__attributes__']['rev_pars_dict'] + assert rev_pars_dict == {'__unserializable__': 'weakref.ReferenceType'} + + def test_observations(test_server): api_url = f"{test_server['url']}/ska_api" starcat_date = '2022:001:17:00:58.521'