Add json serialization format for tables in ska_api

kadi_apps/blueprints/ska_api/api.py

@@ -1,9 +1,8 @@
-from flask import Blueprint
-from flask import request
-
-import logging
 import json
+import logging
 
+from flask import Blueprint, request
+from Quaternion import Quat
 
 APPS = {
     ('agasc',): ['get_star', 'get_stars', 'get_agasc_cone'],
@@ -133,58 +132,128 @@ def _replace_object_cols_with_str(tbl):
     return tbl
 
 
+def _class_info(obj):
+    """Class name and fully-qualified class name of ``obj``
+
+    :returns: dict
+    """
+    return {
+        "class_name": type(obj).__name__,
+        "full_name": f"{type(obj).__module__}.{type(obj).__name__}",
+    }
+
+
 class APIEncoder(json.JSONEncoder):
     def __init__(self, table_format=None, strict_encode=True, **kwargs):
         self.table_format = table_format or 'rows'
         super(APIEncoder, self).__init__()
         self.strict_encode = strict_encode
 
     def encode_table(self, obj):
-        if self.table_format not in ('rows', 'columns'):
+        if self.table_format not in ('rows', 'columns', 'full'):
             raise ValueError('table_format={} not allowed'.format(self.table_format))
 
         obj = _replace_object_cols_with_str(obj)
 
         out = {name: obj[name].tolist() for name in obj.colnames}
 
-        if self.table_format == 'rows':
+        if self.table_format == 'full':
+            out = {
+                "meta": self._json_safe(obj.meta),
+                **_class_info(obj),
+                "columns": out,
+            }
+        elif self.table_format == 'rows':
             # Convert from dict of list to list of dict
             out = [{name: out[name][ii] for name in obj.colnames}
                    for ii in range(len(obj))]
 
         return out
 
+    def encode_object(self, obj, plain, full):
+        """Encode a non-table object, adding class info for table_format='full'
+
+        :param obj: the object being encoded
+        :param plain: the value to use for the 'rows' and 'columns' formats
+        :param full: dict of the values to include along with the class info
+        :returns: dict or ``plain``
+        """
+        if self.table_format == 'full':
+            return {**_class_info(obj), **full}
+        return plain
+
+    def _json_safe(self, obj):
+        """Replace values within ``obj`` that cannot be JSON encoded with a marker
+
+        Dicts and lists are traversed so only the offending value is replaced.
+
+        :returns: the original object or a copy with unserializable values replaced
+        """
+        if isinstance(obj, dict):
+            return {key: self._json_safe(val) for key, val in obj.items()}
+
+        if isinstance(obj, (list, tuple)):
+            return [self._json_safe(val) for val in obj]
+
+        try:
+            self.encode(obj)
+        except Exception:
+            # Note this gives no detail about the value, but unlike repr() it is stable
+            # (a repr often includes the memory address).
+            return {"__unserializable__": f"{type(obj).__module__}.{type(obj).__name__}"}
+
+        return obj
+
     def default(self, obj):
-        from astropy.table import Table
         import numpy as np
+        from astropy.table import Table
+        from astropy.time import Time
+        from astropy.units import Quantity
 
         # Potentially convert something with a `table` property to an astropy Table.
         if hasattr(obj, 'table') and isinstance(obj.__class__.table, property):
             obj_table = obj.table
             if isinstance(obj_table, Table):
                 obj = obj_table
 
-        if type(obj) in [np.int32, np.int64]:
-            return int(obj)
-
-        elif type(obj) in [np.float32, np.float64]:
-            return float(obj)
+        if isinstance(obj, np.generic):
+            # Any numpy scalar: bool_, all int/uint/float sizes and str_.
+            return obj.item()
 
         elif isinstance(obj, np.ma.MaskedArray):
             return {
                 'data': obj.tolist(),
                 'mask': obj.mask.tolist()
             }
 
+        elif isinstance(obj, Quantity):
+            # This must come before ndarray because Quantity is an ndarray subclass
+            # whose tolist() raises NotImplementedError.
+            value = obj.value.tolist()
+            return self.encode_object(
+                obj, value, {"value": value, "unit": obj.unit.to_string()}
+            )
+
         elif isinstance(obj, np.ndarray):
             return obj.tolist()
 
         elif isinstance(obj, Table):
             return self.encode_table(obj)
 
+        elif isinstance(obj, Time):
+            return self.encode_object(
+                obj,
+                obj.value,
+                {"value": obj.value, "format": obj.format, "scale": obj.scale},
+            )
+
         elif isinstance(obj, bytes):
             return obj.decode('utf-8')
 
+        elif isinstance(obj, Quat):
+            q = obj.q.tolist()
+            return self.encode_object(obj, q, {"q": q})
+
         else:
             try:
                 out = super(APIEncoder, self).default(obj)

kadi_apps/templates/api_index.html

@@ -52,10 +52,14 @@ <h3>Data format</h3>
 (<tt class="docutils literal"><span class="pre">Content-type:</span> application/json</tt>).</p>
 <h3>Table format</h3>
 <p>One special option which is common to all queries is the <tt class="docutils literal">table_format</tt>, which can take the value
-<tt class="docutils literal">rows</tt> or <tt class="docutils literal">columns</tt> (default=``rows``).  This specifies whether to return any tabular data as
+<tt class="docutils literal">rows</tt>, <tt class="docutils literal">columns</tt> or <tt class="docutils literal">full</tt> (default=``rows``).  This specifies whether to return any tabular data as
 either a list of dicts (<tt class="docutils literal">rows</tt>) or a dict of lists (<tt class="docutils literal">columns</tt>).  For large query results the
 <tt class="docutils literal">columns</tt> option will generally be more compact because the table column names are not repeated for
-every row. For example:</p>
+every row. The <tt class="docutils literal">full</tt> option returns a dict with the columns (as a dict of lists) under the
+<tt class="docutils literal">columns</tt> key, and also includes the table metadata under the <tt class="docutils literal">meta</tt> key together with the
+table class name. Metadata values that cannot be represented as JSON (for example an internal
+object reference) are replaced by a <tt class="docutils literal">__unserializable__</tt> marker giving the class
+that was dropped. For example:</p>
 <p><a class="reference external" href="{{ url_for('ska_api.api', path='kadi/events/manvrs/filter') }}?start=2019:001&amp;stop=2019:002&amp;table_format=columns">{{ url_for('ska_api.api', path='kadi/events/manvrs/filter', _external=True) }}/kadi?start=2019:001&amp;stop=2019:002&amp;table_format=columns</a></p>
 
 <h2>Available entrypoints</h2>

kadi_apps/tests/test_ska_api.py

@@ -1,12 +1,17 @@
+import json
+import re
+
+import astropy.units as u
 import numpy as np
+import pytest
 import requests
-import re
 from astropy.table import Table
+from astropy.time import Time
+from cxotime import CxoTime
+from kadi.commands import get_observations, get_starcats
+from Quaternion import Quat
 
-from kadi.commands import get_starcats
-from kadi.commands import get_observations
-
-from kadi_apps.blueprints.ska_api.api import _replace_object_cols_with_str
+from kadi_apps.blueprints.ska_api.api import APIEncoder, _replace_object_cols_with_str
 
 
 def test_agasc_star(test_server):
@@ -199,6 +204,128 @@ def test_starcats(test_server):
         assert np.all(starcats[i] == starcats_api[i])
 
 
+def _interpret_aca_table(data):
+    from proseco.catalog import ACATable, AcqTable, GuideTable
+    meta = data["meta"].copy()
+    meta["acqs"] = AcqTable(meta["acqs"]["columns"], meta=meta["acqs"]["meta"])
+    meta["guides"] = GuideTable(meta["guides"]["columns"], meta=meta["guides"]["meta"])
+    meta["att"] = Quat(meta["att"]["q"])
+    starcat = ACATable(data["columns"], meta=meta)
+    return starcat
+
+
+def test_starcats_full(test_server):
+    api_url = f"{test_server['url']}/ska_api"
+    start = '2022:001'
+    stop = '2022:002'
+    obsid = None
+    starcats = get_starcats(start=start, stop=stop, obsid=obsid, scenario='flight')
+    url = f'{api_url}/kadi/commands/get_starcats?{start=}&{stop=}&scenario=flight&table_format=full'
+    r = requests.get(url)
+    assert r.ok
+    starcats_api = [_interpret_aca_table(cat) for cat in r.json()]
+    assert len(starcats) == len(starcats_api)
+    for i in range(len(starcats)):
+        sc = starcats[i]
+        sc_api = starcats_api[i]
+        assert np.all(sc == sc_api), f"starcat {sc.date} data does not match API output"
+
+        assert np.all(sc.meta['acqs'] == sc_api.meta['acqs']), f"starcat {sc.date} acqs does not match API output"
+        assert np.all(sc.meta['guides'] == sc_api.meta['guides']), f"starcat {sc.date} guides does not match API output"
+        assert sc.date == sc_api.date, f"starcat {sc.date} date does not match API output"
+        assert sc.obsid == sc_api.obsid, f"starcat {sc.date} obsid does not match API output"
+        assert sc.duration == sc_api.duration, f"starcat {sc.date} duration does not match API output"
+        assert sc.detector == sc_api.detector, f"starcat {sc.date} detector does not match API output"
+        assert sc.sim_offset == sc_api.sim_offset, f"starcat {sc.date} sim_offset does not match API output"
+        assert sc.t_ccd_guide == sc_api.t_ccd_guide, f"starcat {sc.date} t_ccd_guide does not match API output"
+        assert sc.t_ccd_acq == sc_api.t_ccd_acq, f"starcat {sc.date} t_ccd_acq does not match API output"
+
+
+def _encode_meta_value(val, table_format='full'):
+    """Put ``val`` in the meta of a table and return its encoded form"""
+    tbl = Table({'a': [1]})
+    tbl.meta['val'] = val
+    out = json.loads(APIEncoder(table_format=table_format).encode(tbl))
+    return out['meta']['val']
+
+
+@pytest.mark.parametrize(
+    'val,expected',
+    [
+        (np.bool_(True), True),
+        (np.int8(-1), -1),
+        (np.int16(-2), -2),
+        (np.int32(-3), -3),
+        (np.int64(-4), -4),
+        (np.uint8(1), 1),
+        (np.uint16(2), 2),
+        (np.uint32(3), 3),
+        (np.uint64(4), 4),
+        (np.float16(1.5), 1.5),
+        (np.float32(2.5), 2.5),
+        (np.float64(3.5), 3.5),
+        (np.str_('abc'), 'abc'),
+    ]
+)
+def test_encode_numpy_scalars(val, expected):
+    """Numpy scalars of any size or kind are encoded as the native Python value"""
+    assert _encode_meta_value(val) == expected
+
+
+def test_encode_quantity():
+    assert _encode_meta_value(3.0 * u.deg) == {
+        'class_name': 'Quantity',
+        'full_name': 'astropy.units.quantity.Quantity',
+        'value': 3.0,
+        'unit': 'deg',
+    }
+    assert _encode_meta_value([1.0, 2.0] * u.deg)['value'] == [1.0, 2.0]
+
+
+def test_encode_time():
+    """A Time is encoded with what is needed to reconstruct it"""
+    time = CxoTime('2022:001:12:00:00.000')
+    out = _encode_meta_value(time)
+    assert out['class_name'] == 'CxoTime'
+    assert CxoTime(out['value'], format=out['format'], scale=out['scale']) == time
+    assert _encode_meta_value(Time('2022-01-01'))['class_name'] == 'Time'
+
+
+def test_encode_object_class_info_only_for_full():
+    """Class info is included in the 'full' format only"""
+    quat = Quat([0, 0, 0, 1])
+    assert _encode_meta_value(quat)['q'] == [0.0, 0.0, 0.0, 1.0]
+    # The other formats do not include meta at all, so encode the Quat directly.
+    for table_format in ('rows', 'columns'):
+        encoded = APIEncoder(table_format=table_format).encode(quat)
+        assert json.loads(encoded) == [0.0, 0.0, 0.0, 1.0]
+
+
+def test_encode_unserializable_meta():
+    """A meta value that cannot be encoded is replaced by a stable marker"""
+    tbl = Table({'a': [1]})
+    tbl.meta['good'] = 1
+    tbl.meta['nested'] = {'bad': object()}
+    out = json.loads(APIEncoder(table_format='full').encode(tbl))
+    assert out['meta']['good'] == 1
+    assert out['meta']['nested']['bad'] == {'__unserializable__': 'builtins.object'}
+
+
+def test_cmds_full(test_server):
+    """get_cmds meta holds a weakref, which must not fail the request"""
+    api_url = f"{test_server['url']}/ska_api"
+    start = '2022:001'
+    stop = '2022:002'
+    url = f'{api_url}/kadi/commands/get_cmds?{start=}&{stop=}&scenario=flight&table_format=full'
+    r = requests.get(url)
+    assert r.ok
+    out = r.json()
+    assert out['class_name'] == 'CommandTable'
+    assert len(out['columns']['idx']) > 0
+    rev_pars_dict = out['meta']['__attributes__']['rev_pars_dict']
+    assert rev_pars_dict == {'__unserializable__': 'weakref.ReferenceType'}
+
+
 def test_observations(test_server):
     api_url = f"{test_server['url']}/ska_api"
     starcat_date = '2022:001:17:00:58.521'