From 376d921e16e32e0065e176b297b45de22ae043f1 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 11 Jun 2026 19:26:13 +0200 Subject: [PATCH 01/13] Recover the jsonb Jsonb and jsonpath JsonPath collapsed C types libclang collapses the PG-vendored Jsonb (struct JsonbPair) and JsonPath (struct JsonPathParseItem) pointer types to int during header parsing, the same mechanism already handled for GSERIALIZED/GBOX/Interval/DateADT. Add both to the type-recovery map so the temporal-JSONB (tjsonb) and jsonbset functions carry their real Jsonb*/JsonPath* signatures in the generated IDL that every binding consumes (verified: jsonbset_value_n result recovers to Jsonb**). --- parser/typerecover.py | 2 ++ tests/test_typerecover.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/parser/typerecover.py b/parser/typerecover.py index 01d8153..49cff42 100644 --- a/parser/typerecover.py +++ b/parser/typerecover.py @@ -44,6 +44,8 @@ "GBOX": "GBOX", "BOX3D": "BOX3D", "AFFINE": "AFFINE", + "Jsonb": "Jsonb", + "JsonPath": "JsonPath", } _NAMES = "|".join(sorted(_TYPE_MAP, key=len, reverse=True)) diff --git a/tests/test_typerecover.py b/tests/test_typerecover.py index ed10993..de68f07 100644 --- a/tests/test_typerecover.py +++ b/tests/test_typerecover.py @@ -81,6 +81,22 @@ def test_no_gserialized_left_collapsed_to_int(self): self.assertGreater(len(geo_fns), 50, "GSERIALIZED* collapsed toward int — typerecover regression?") + # ---- jsonb Jsonb / jsonpath JsonPath (opaque PG types, collapse to int) - + + def test_jsonb_recovered_when_tjsonb_present(self): + # The temporal-JSONB surface is built only when MEOS is compiled with + # JSON=ON, so this assertion is conditional on the parsed source + # carrying it (skipped otherwise to stay source-agnostic). + jsonb_fns = [n for n in self.by_name + if "jsonb" in n.lower() or "tjsonb" in n.lower()] + if not jsonb_fns: + self.skipTest("source parsed without the JSON=ON tjsonb surface") + # An out-parameter that pre-fix came back as ``int **``. + self.assertIn("Jsonb **", self._param_ctypes("jsonbset_value_n")) + carriers = [f for f in self.by_name.values() if "Jsonb *" in json.dumps(f)] + self.assertGreater(len(carriers), 20, + "Jsonb* collapsed toward int — typerecover regression?") + # ---- other PG-vendored opaque types (Interval / DateADT / Datum / ...) - def test_interval_params_recovered(self): From 5b04de9a61ec13b27d77febf175dc17a9110a408 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 23 Jun 2026 09:33:42 +0200 Subject: [PATCH 02/13] Recover the quadbin Quadbin collapsed C type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quadbin is a typedef of uint64 (meos_quadbin.h). Unlike H3Index — whose name is erased to int by the host-collision prefix rename, so the existing recovery (slot cType == int) catches it — Quadbin's name SURVIVES as the cType while only the canonical collapses to int (its underlying uint64 is the part that erased). So the recovery must also fire when the slot spells the typedef name, not just int. Add "Quadbin": "uint64_t" to the type-recovery map AND generalize the recovery to match either the collapsed `int` spelling or the surviving `original` typedef name, rewriting both cType and the collapsed canonical to uint64_t. Every catalog-driven binding now binds the quadbin cell as a 64-bit integer (bigint), like H3Index, instead of disagreeing on the collapsed int (JMEOS mapped the Quadbin cType to a Pointer; the Spark generator mapped the collapsed int to Integer). --- parser/typerecover.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/parser/typerecover.py b/parser/typerecover.py index 01d8153..b77e0e3 100644 --- a/parser/typerecover.py +++ b/parser/typerecover.py @@ -35,6 +35,7 @@ "Timestamp": "Timestamp", "TimestampTz": "TimestampTz", "H3Index": "uint64_t", + "Quadbin": "uint64_t", "text": "text", "GSERIALIZED": "GSERIALIZED", "Interval": "Interval", @@ -75,7 +76,8 @@ def _recovery(fragment): suffix = (" " + stars) if stars else "" collapsed = f"{const}int{suffix}" recovered = f"{const}{_TYPE_MAP[m.group('base')]}{suffix}" - return collapsed, recovered + original = f"{const}{m.group('base')}{suffix}" + return collapsed, recovered, original def _parse_header_decls(headers_dir): @@ -123,12 +125,17 @@ def _apply(slot, recovery): """Rewrite a return/param slot in place; return 1 if rewritten.""" if not (recovery and isinstance(slot, dict)): return 0 - collapsed, recovered = recovery + collapsed, recovered, original = recovery key = "c" if "c" in slot else "cType" - if _nospace(slot.get(key)) != _nospace(collapsed): + # The base name is either erased to int by the host-collision prefix + # rename (slot spells `collapsed`), or it survives while only the + # canonical collapses (slot spells `original`, e.g. a MobilityDB typedef + # such as Quadbin whose uint64 underlying type was the part that erased). + recoverable = (_nospace(collapsed), _nospace(original)) + if _nospace(slot.get(key)) not in recoverable: return 0 slot[key] = recovered - if _nospace(slot.get("canonical")) == _nospace(collapsed): + if _nospace(slot.get("canonical")) in recoverable: slot["canonical"] = recovered return 1 From e824b4f9e208159278138338a77672a556ea0952 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Mon, 25 May 2026 10:29:29 +0200 Subject: [PATCH 03/13] Generate function shape (output arrays + nullability) from the headers The binding codegens consume each function's `shape` (array-return length, written-back output arrays, parameter nullability) from the IDL. Generate it from the headers instead of the hand-maintained meta stub, which had drifted to three functions and emitted no shape for the other 2900+ -- leaving every codegen to mis-handle the split/space-split/mvtgeom out-arrays and the nullable srs/maxt/state parameters. - parser/shapeinfer.py: infer arrayReturn/outputArrays from the parameter forms. A non-const `TYPE **` paired with a by-pointer `int *count` is a written-back out-array; a by-value `int count` marks a read-only in-array. - parser/nullable.py: read parameter nullability from the C Doxygen `@param ... may be NULL` source of truth and fold it into shape.nullable for the params present on each function. - run.py: wire both into the IDL build. Adds unit tests for both modules. --- parser/nullable.py | 68 ++++++++++++++++++++++++++++++++++++ parser/shapeinfer.py | 72 ++++++++++++++++++++++++++++++++++++++ run.py | 12 +++++++ tests/test_nullable.py | 75 ++++++++++++++++++++++++++++++++++++++++ tests/test_shapeinfer.py | 70 +++++++++++++++++++++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 parser/nullable.py create mode 100644 parser/shapeinfer.py create mode 100644 tests/test_nullable.py create mode 100644 tests/test_shapeinfer.py diff --git a/parser/nullable.py b/parser/nullable.py new file mode 100644 index 0000000..41c411c --- /dev/null +++ b/parser/nullable.py @@ -0,0 +1,68 @@ +"""Extract per-parameter nullability from the MEOS C Doxygen as the SoT. + +A MEOS function parameter accepts NULL iff its Doxygen ``@param`` line says so, +e.g. ``@param[in] srs Spatial reference system, may be `NULL```. This is the +single source of truth the codegens consume — grounded in the C code, keyed by +parameter name, and cross-checked in MobilityDB against the PG layer (a SQL +function declared without ``STRICT`` + the wrapper's ``PG_ARGISNULL`` guards). + +The extractor walks the MEOS sources, pairs each Doxygen block with the function +it documents, and records the params whose description carries a NULL note. The +result feeds ``shape.nullable`` in the IDL so every binding can guard the param. +""" +from __future__ import annotations + +import glob +import re +from pathlib import Path + +# Doxygen block immediately followed by a function definition (``name(...) {``). +_FUNC = re.compile( + r'/\*\*(?P.*?)\*/\s*\n' + r'(?:[A-Za-z_][\w\s\*]*?\n)?' # optional return-type line + r'(?P[a-z][a-z0-9_]*)\s*\(' + r'(?P[^;{]*?)\)\s*\{', + re.S) +# One @param entry: capture the (possibly comma-separated) names + description. +_PARAM = re.compile( + r'@param\[[^\]]*\]\s+(?P\w+(?:\s*,\s*\w+)*)\s+(?P.*?)' + r'(?=\n\s*\*\s*@|\*/|\Z)', re.S) +_NULLISH = re.compile(r'may be\s+`?NULL`?|can be\s+`?NULL`?|`?NULL`?\s+is allowed' + r'|or\s+`?NULL`?', re.I) + + +def extract_nullable(meos_root: str | Path) -> dict[str, list[str]]: + """Return ``{function: [nullable params]}`` from the MEOS C sources under + ``meos_root`` (scans both ``src`` and ``include``).""" + root = Path(meos_root) + out: dict[str, list[str]] = {} + files = glob.glob(str(root / "src/**/*.c"), recursive=True) + files += glob.glob(str(root / "include/**/*.h"), recursive=True) + for f in files: + txt = Path(f).read_text(errors="ignore") + for m in _FUNC.finditer(txt): + name = m.group("name") + for pm in _PARAM.finditer(m.group("doc")): + if not _NULLISH.search(pm.group("desc")): + continue + for p in (n.strip() for n in pm.group("names").split(",")): + out.setdefault(name, []) + if p and p not in out[name]: + out[name].append(p) + return out + + +def merge_nullable(idl: dict, meos_root: str | Path) -> tuple[dict, int]: + """Fold the extracted nullability into each function's ``shape.nullable``.""" + nul = extract_nullable(meos_root) + n = 0 + for func in idl["functions"]: + params = nul.get(func["name"]) + if not params: + continue + present = {p["name"] for p in func.get("params", [])} + keep = [p for p in params if p in present] + if keep: + func.setdefault("shape", {})["nullable"] = keep + n += len(keep) + return idl, n diff --git a/parser/shapeinfer.py b/parser/shapeinfer.py new file mode 100644 index 0000000..598372c --- /dev/null +++ b/parser/shapeinfer.py @@ -0,0 +1,72 @@ +"""Infer per-function output-array *shape* from the C signatures. + +MEOS array-returning functions follow one fixed convention, so the shape the +codegens need is fully derivable from the headers — no hand-maintained table: + + TYPE *f(..., int *count) -> returns an array of ``count`` + TYPE **f(..., TYPE **extra, int *count) -> primary array return PLUS one + or more parallel out-arrays + +The output length is always passed *by pointer* (``int *count``); an *input* +array instead carries its length *by value* (``int count``). That pointer/value +distinction is exactly how a written-back out-array is told apart from a +read-only in-array — e.g. ``temporal_time_split(..., TimestampTz **time_bins, +int *count)`` (out) versus ``tsequence_make(const TInstant **instants, int +count, ...)`` (in). + +This replaces the ``meta/meos-meta.json`` shape entries, which had drifted to a +3-function stub and silently mis-classified every out-array as an input +parameter, breaking the split / space-split / mvtgeom / normalize families in +every binding generated from the IDL. +""" +from __future__ import annotations + + +# Parameters that accept NULL by MEOS convention regardless of the function. +# ``srs`` is the optional spatial-reference string of every ``*_as_*json`` / +# text output function — passing NULL means "no CRS". Nullability is otherwise +# semantic (not signature-derivable), so this stays a narrow, named convention +# rather than a blanket rule; extend only when a binding's tests prove a param +# is passed None. +_NULLABLE_BY_CONVENTION = {"srs"} + + +def _out_count_param(func: dict) -> str | None: + """Return the name of the by-pointer output count param, if the function + has one. This is the marker that the function returns array(s).""" + for p in func.get("params", []): + if p["name"] == "count" and p.get("cType", "").strip() == "int *": + return p["name"] + return None + + +def _is_written_back_array(p: dict) -> bool: + """A non-const double (or higher) pointer parameter the callee allocates + and writes back, i.e. a parallel output array.""" + ct = p.get("cType", "") + return "**" in ct and not ct.lstrip().startswith("const") + + +def infer_shapes(idl: dict) -> tuple[dict, dict]: + """Populate ``func['shape']`` with ``arrayReturn``/``outputArrays`` derived + from the signatures. Returns ``(idl, stats)``. Idempotent and additive: + only the array-output families are touched, everything else is untouched.""" + n_arr = n_oa = 0 + for func in idl["functions"]: + count = _out_count_param(func) + if not count: + continue # not array-returning; nothing to infer + shape = func.setdefault("shape", {}) + # The primary pointer return takes its length from the output count. + ret = func.get("returnType", {}).get("c", "") + if ret.rstrip().endswith("*"): + shape.setdefault("arrayReturn", {})["lengthFrom"] = { + "kind": "param", "name": count} + n_arr += 1 + # Parallel written-back out-arrays (``TYPE **extra`` alongside count). + out = [{"param": p["name"]} for p in func["params"] + if p["name"] != count and _is_written_back_array(p)] + if out: + shape["outputArrays"] = out + n_oa += len(out) + return idl, {"arrayReturn": n_arr, "outputArrays": n_oa} diff --git a/run.py b/run.py index 640dd9e..81ec258 100644 --- a/run.py +++ b/run.py @@ -5,6 +5,8 @@ from parser.parser import parse_all_headers, merge_meta from parser.portable import attach_portable_aliases from parser.typerecover import recover_collapsed_types +from parser.shapeinfer import infer_shapes +from parser.nullable import merge_nullable HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -28,6 +30,16 @@ def main(): print(f" recovered {rec['returns']} return types, " f"{rec['params']} params from collapsed int", file=sys.stderr) + # 1c. Generate the codegen `shape` from the signatures + Doxygen, replacing + # the hand-maintained meta stub. outputArrays/arrayReturn come from the + # parameter forms; nullable comes from the C `@param ... may be NULL` SoT. + idl, sh = infer_shapes(idl) + print(f" inferred shape: {sh['arrayReturn']} array returns, " + f"{sh['outputArrays']} output arrays", file=sys.stderr) + idl, nn = merge_nullable(idl, HEADERS_DIR.parent) + print(f" nullable params from Doxygen `may be NULL`: {nn}", + file=sys.stderr) + # 2. Merge with manual metadata if META_PATH.exists(): print(f"[2/3] Merging with {META_PATH}...", file=sys.stderr) diff --git a/tests/test_nullable.py b/tests/test_nullable.py new file mode 100644 index 0000000..0d37bc4 --- /dev/null +++ b/tests/test_nullable.py @@ -0,0 +1,75 @@ +"""Regression tests for parser/nullable.py. + +Nullability is read from the C Doxygen `@param ... may be NULL` notes (the +source of truth) and folded into each function's ``shape.nullable`` for the +params that actually exist on the IDL function. + +Plain unittest, no pytest dependency; writes a tiny synthetic source tree. +""" +import tempfile +import unittest +from pathlib import Path + +from parser.nullable import extract_nullable, merge_nullable + +SAMPLE = ''' +/** + * @ingroup meos_temporal_inout + * @brief Return the MF-JSON representation + * @param[in] temp Temporal value + * @param[in] srs Spatial reference system, may be `NULL` + */ +char * +temporal_as_mfjson(const Temporal *temp, char *srs) +{ + return NULL; +} + +/** + * @brief Append an instant + * @param[in] inst Instant + * @param[in] maxt Maximum time interval, may be `NULL` + * @param[in] interp Interpolation + */ +Temporal * +temporal_append_tinstant(const TInstant *inst, const Interval *maxt, int interp) +{ + return NULL; +} +''' + + +class NullableTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + src = Path(self.tmp.name) / "src" + src.mkdir() + (src / "sample.c").write_text(SAMPLE) + (Path(self.tmp.name) / "include").mkdir() + + def tearDown(self): + self.tmp.cleanup() + + def test_extracts_only_may_be_null_params(self): + nul = extract_nullable(self.tmp.name) + self.assertEqual(nul["temporal_as_mfjson"], ["srs"]) + self.assertEqual(nul["temporal_append_tinstant"], ["maxt"]) + # `temp`, `inst`, `interp` carry no NULL note -> not nullable + self.assertNotIn("temp", nul["temporal_as_mfjson"]) + + def test_merge_only_existing_params(self): + idl = {"functions": [ + {"name": "temporal_as_mfjson", + "params": [{"name": "temp"}, {"name": "srs"}]}, + # function whose nullable param is NOT in its IDL signature + {"name": "temporal_append_tinstant", "params": [{"name": "inst"}]}, + ]} + idl, n = merge_nullable(idl, self.tmp.name) + self.assertEqual(idl["functions"][0]["shape"]["nullable"], ["srs"]) + # maxt absent from the IDL params -> not added + self.assertNotIn("shape", idl["functions"][1]) + self.assertEqual(n, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_shapeinfer.py b/tests/test_shapeinfer.py new file mode 100644 index 0000000..592cd41 --- /dev/null +++ b/tests/test_shapeinfer.py @@ -0,0 +1,70 @@ +"""Regression tests for parser/shapeinfer.py. + +The inferer derives array-output shape from the C signatures, replacing the +hand-maintained meta stub. The discriminator is the *count* parameter's form: + +* a written-back out-array pairs with a by-pointer ``int *count`` (the callee + fills the length) -> ``outputArrays`` + ``arrayReturn.lengthFrom`` +* a read-only in-array pairs with a by-value ``int count`` -> left untouched + +Plain unittest, no pytest dependency; fully synthetic IDL (no generated file). +""" +import unittest + +from parser.shapeinfer import infer_shapes + + +def _fn(name, ret, params): + return {"name": name, + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t} for n, t in params]} + + +class ShapeInferTests(unittest.TestCase): + def test_output_array_with_pointer_count(self): + # temporal_time_split-style: non-const ** out-array + by-pointer count + idl = {"functions": [_fn( + "temporal_time_split", "Temporal **", + [("temp", "const Temporal *"), ("duration", "const Interval *"), + ("torigin", "TimestampTz"), ("time_bins", "TimestampTz **"), + ("count", "int *")])]} + idl, stats = infer_shapes(idl) + sh = idl["functions"][0]["shape"] + self.assertEqual(sh["outputArrays"], [{"param": "time_bins"}]) + self.assertEqual(sh["arrayReturn"]["lengthFrom"], + {"kind": "param", "name": "count"}) + self.assertEqual(stats["outputArrays"], 1) + + def test_two_parallel_output_arrays(self): + idl = {"functions": [_fn( + "tfloat_value_time_split", "Temporal **", + [("temp", "const Temporal *"), ("vsize", "double"), + ("value_bins", "double **"), ("time_bins", "TimestampTz **"), + ("count", "int *")])]} + idl, _ = infer_shapes(idl) + self.assertEqual(idl["functions"][0]["shape"]["outputArrays"], + [{"param": "value_bins"}, {"param": "time_bins"}]) + + def test_input_array_with_value_count_untouched(self): + # tsequence_make-style: ** input array carries its length BY VALUE + idl = {"functions": [_fn( + "tsequence_make", "TSequence *", + [("instants", "const TInstant **"), ("count", "int"), + ("lower_inc", "bool")])]} + idl, stats = infer_shapes(idl) + self.assertNotIn("shape", idl["functions"][0]) + self.assertEqual(stats["outputArrays"], 0) + + def test_nonconst_input_array_with_value_count_untouched(self): + # tsequenceset_make_gaps-style: non-const ** but BY-VALUE count => input + idl = {"functions": [_fn( + "tsequenceset_make_gaps", "TSequenceSet *", + [("instants", "TInstant **"), ("count", "int"), + ("maxt", "const Interval *")])]} + idl, stats = infer_shapes(idl) + self.assertEqual(stats["outputArrays"], 0) + self.assertNotIn("shape", idl["functions"][0]) + + +if __name__ == "__main__": + unittest.main() From 2e3df74ba6e5ea25fc3a1a613f492931d5d75dd3 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Fri, 12 Jun 2026 21:02:09 +0200 Subject: [PATCH 04/13] Attach the @sqlfn SQL-name map to the catalog Follow each function's @csqlfn -> wrapper @sqlfn -> SQL name chain and attach the resulting SQL name to the catalog. The vendored-source root is overridable via MDB_SRC_ROOT so the @sqlfn (and @ingroup) extraction can be pointed at the same pinned MobilityDB checkout as the headers, keeping the generated catalog reproducibly equivalent to that pin. --- parser/sqlfn.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ run.py | 13 ++++++++ 2 files changed, 99 insertions(+) create mode 100644 parser/sqlfn.py diff --git a/parser/sqlfn.py b/parser/sqlfn.py new file mode 100644 index 0000000..4d8b2a9 --- /dev/null +++ b/parser/sqlfn.py @@ -0,0 +1,86 @@ +"""Attach the SQL-name map (@sqlfn / @sqlop) to the MEOS-API catalog. + +The catalog carries MEOS-C function names + C signatures, but bindings that +emit a SQL/UDF surface (MobilityDB SQL, MobilitySpark UDFs, MobilityDuck, …) +need the user-facing SQL name and operator. Both are machine-extractable from +the doxygen tag chain that already pervades the source: + + MEOS-C fn --@csqlfn #MobilityDB_C()--> MobilityDB-C wrapper + MobilityDB-C wrapper --@sqlfn sqlName() / @sqlop @p --> SQL name + op + +So: in meos/src `@csqlfn #Wrapper()` sits above the MEOS-C function (→ MEOS-C → +Wrapper); in mobilitydb/src `@sqlfn name()` + `@sqlop @p ` sit above +`Datum Wrapper(PG_FUNCTION_ARGS)` (→ Wrapper → name, op). Join on Wrapper. + +Adds per function (when the chain resolves): `sqlfn`, `sqlop`, `mdbC`. +""" +import re +from pathlib import Path + +_CSQLFN = re.compile(r"@csqlfn\s+#(\w+)\s*\(\)") +# After the doxygen close, the MEOS-C definition: an optional return-type line +# (no parens/braces/;/=), then `name(`. +_FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") +_SQLFN = re.compile(r"@sqlfn\s+(\w+)\s*\(\)") +_SQLOP = re.compile(r"@sqlop\s+@p\s+(\S+)") +_DATUM = re.compile(r"Datum\s+(\w+)\s*\(\s*PG_FUNCTION_ARGS") + + +def _meos_to_mdb(meos_src): + """MEOS-C function name -> MobilityDB-C wrapper name (from @csqlfn).""" + out = {} + for cf in Path(meos_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _CSQLFN.finditer(text): + mdb_c = m.group(1) + fm = _FNDEF.search(text, m.end()) + if fm: + out.setdefault(fm.group(1), mdb_c) + return out + + +def _mdb_to_sql(mdb_src): + """MobilityDB-C wrapper name -> ordered list of (sqlfn, sqlop). + + A shared PG wrapper can carry more than one @sqlfn (e.g. Temporal_derivative + is exposed as both derivative() and speed()), so collect ALL of them rather + than the first — otherwise the mapped SQL name is order-dependent. + """ + out = {} + for cf in Path(mdb_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _SQLFN.finditer(text): + sqlfn = m.group(1) + # @sqlop lives in the SAME doxygen block (before the closing */). + close = text.find("*/", m.end()) + block = text[m.start():close] if close != -1 else text[m.start():m.start() + 800] + op = _SQLOP.search(block) + dm = _DATUM.search(text, close if close != -1 else m.end()) + if dm: + entry = (sqlfn, op.group(1) if op else None) + lst = out.setdefault(dm.group(1), []) + if entry not in lst: + lst.append(entry) + return out + + +def attach_sqlfn_map(idl, meos_src, mdb_src): + m2d = _meos_to_mdb(meos_src) + d2s = _mdb_to_sql(mdb_src) + n = 0 + for f in idl["functions"]: + mdb_c = m2d.get(f["name"]) + if not mdb_c: + continue + lst = d2s.get(mdb_c) + if not lst: + continue + f["mdbC"] = mdb_c + f["sqlfn"] = lst[0][0] + if lst[0][1]: + f["sqlop"] = lst[0][1] + # Shared wrapper exposing >1 SQL name: record them all (binding picks). + if len(lst) > 1: + f["sqlfnAll"] = [s for s, _ in lst] + n += 1 + return idl, n diff --git a/run.py b/run.py index 81ec258..2942780 100644 --- a/run.py +++ b/run.py @@ -1,3 +1,4 @@ +import os import sys import json from pathlib import Path @@ -7,6 +8,7 @@ from parser.typerecover import recover_collapsed_types from parser.shapeinfer import infer_shapes from parser.nullable import merge_nullable +from parser.sqlfn import attach_sqlfn_map HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -52,6 +54,17 @@ def main(): file=sys.stderr) idl = attach_portable_aliases(idl, PORTABLE_PATH) + # 4. Attach the SQL-name map (@sqlfn/@sqlop) from the vendored source. + # The source root is overridable (MDB_SRC_ROOT) so a binding can point the + # @sqlfn/@ingroup extraction at the SAME pinned checkout as the headers, + # keeping the catalog reproducibly equivalent to that pin. + SRC_ROOT = Path(os.environ.get("MDB_SRC_ROOT", "./_mobilitydb")) + MEOS_SRC = SRC_ROOT / "meos" / "src" + MDB_SRC = SRC_ROOT / "mobilitydb" / "src" + if MEOS_SRC.exists() and MDB_SRC.exists(): + idl, nsql = attach_sqlfn_map(idl, MEOS_SRC, MDB_SRC) + print(f"[4/4] Attached {nsql} @sqlfn SQL names", file=sys.stderr) + idl_path = OUTPUT_DIR / "meos-idl.json" with open(idl_path, "w") as f: json.dump(idl, f, indent=2) From ab1a639aa4df68f9ea9e0ccf560ec22bac45d02e Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Mon, 15 Jun 2026 23:30:26 +0200 Subject: [PATCH 05/13] Guard against ever/always @csqlfn prefix mistags in meos/src MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A copy-paste @csqlfn in meos/src can point an ever/always spatial-relationship function (named _...) at the opposite-prefix MobilityDB-C wrapper — e.g. eintersects_tgeo_geo is tagged `@csqlfn #Aintersects_tgeo_geo`, so the chain resolves its @sqlfn to aIntersects instead of eIntersects. That silently drops the real (tgeo,geo) overload from the eIntersects group in a binding's overload dispatcher, leaving only a wrong-subtype backing reachable — which then raises a runtime "The temporal value must be of type tcbuffer" on a tgeompoint (observed in the MobilitySpark BerlinMOD bench, q17). The parser is faithful — it propagates whatever the source tags say — so this is a source defect to fix in meos/src, not a parser bug. Add lint_ea_sqlfn() and report the mismatches at catalog-gen so they surface loudly instead of shipping a wrong @sqlfn silently. Flags 5 live mistags (eintersects_tgeo_geo, etouches_tpoint_geo/_tcbuffer_geo/_tcbuffer_cbuffer, econtains_geo_trgeo); relayed to the source maintainers to correct the @csqlfn tags. --- parser/sqlfn.py | 24 ++++++++++++++++++++++++ run.py | 12 +++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/parser/sqlfn.py b/parser/sqlfn.py index 4d8b2a9..4d61136 100644 --- a/parser/sqlfn.py +++ b/parser/sqlfn.py @@ -84,3 +84,27 @@ def attach_sqlfn_map(idl, meos_src, mdb_src): f["sqlfnAll"] = [s for s, _ in lst] n += 1 return idl, n + + +# MEOS-C ever/always spatial-relationship functions are named _...; their +# @csqlfn must point at the matching _... wrapper. A copy-paste @csqlfn in +# meos/src (e.g. eintersects_tgeo_geo tagged #Aintersects_tgeo_geo) silently flips the +# resolved @sqlfn from eX to aX — which then drops the real overload from the eX dispatch +# group and lets a wrong subtype backing be reached (a runtime "must be of type ..." error +# in the bindings). The parser is faithful, so guard the SOURCE here: flag any function +# whose name e/a prefix disagrees with its resolved @sqlfn e/a prefix. +_EA_FAMILY = re.compile( + r"^(e|a)(intersects|disjoint|contains|contained|covers|coveredby|touches|" + r"dwithin|within|equals|crosses|overlaps)_") + + +def lint_ea_sqlfn(idl): + """Return [(meos_c_name, sqlfn)] where the function's ever/always (e/a) name prefix + contradicts its resolved @sqlfn — a source @csqlfn mistag in meos/src.""" + bad = [] + for f in idl["functions"]: + sf = f.get("sqlfn") + m = _EA_FAMILY.match(f["name"]) + if sf and m and re.match(r"^[ea][A-Z]", sf) and sf[0] != m.group(1): + bad.append((f["name"], sf)) + return bad diff --git a/run.py b/run.py index 2942780..40f325d 100644 --- a/run.py +++ b/run.py @@ -8,7 +8,7 @@ from parser.typerecover import recover_collapsed_types from parser.shapeinfer import infer_shapes from parser.nullable import merge_nullable -from parser.sqlfn import attach_sqlfn_map +from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -64,6 +64,16 @@ def main(): if MEOS_SRC.exists() and MDB_SRC.exists(): idl, nsql = attach_sqlfn_map(idl, MEOS_SRC, MDB_SRC) print(f"[4/4] Attached {nsql} @sqlfn SQL names", file=sys.stderr) + # Guard: a copy-paste @csqlfn in meos/src can point an ever/always function at + # the opposite-prefix wrapper (eintersects_* tagged #Aintersects_*), flipping its + # SQL name and breaking the binding overload dispatch. The parser is faithful, so + # surface the SOURCE mistag here rather than ship a wrong catalog silently. + ea_bad = lint_ea_sqlfn(idl) + if ea_bad: + print(f" ⚠ {len(ea_bad)} @csqlfn e/a-prefix mistag(es) in meos/src " + f"(fix at source — wrong @sqlfn resolved):", file=sys.stderr) + for cname, sf in ea_bad: + print(f" {cname} -> @sqlfn {sf}", file=sys.stderr) idl_path = OUTPUT_DIR / "meos-idl.json" with open(idl_path, "w") as f: From f5b272bc978eb5b7836124863e3306904e21abee Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Sun, 21 Jun 2026 21:18:28 +0200 Subject: [PATCH 06/13] Parse every #Wrapper() reference in a @csqlfn tag A single MEOS function can back more than one MobilityDB-C wrapper: the ever/always pair (eDisjoint/aDisjoint) shares one ea_* function, and the shift/scale/shift_scale trio shares one C function. Such functions carry one @csqlfn with comma- or space-separated #Wrapper() references that may continue across doxygen lines. _meos_to_mdb reads the whole tag value and returns every wrapper, and attach_sqlfn_map records all the SQL names a function exposes (sqlfn/sqlfnAll, mdbC/mdbCAll). --- parser/sqlfn.py | 62 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/parser/sqlfn.py b/parser/sqlfn.py index 4d61136..6391df2 100644 --- a/parser/sqlfn.py +++ b/parser/sqlfn.py @@ -17,7 +17,14 @@ import re from pathlib import Path -_CSQLFN = re.compile(r"@csqlfn\s+#(\w+)\s*\(\)") +# A @csqlfn tag carries one OR MORE #Wrapper() references — comma- or +# space-separated, and possibly continued across doxygen lines — because a single +# MEOS function can back several wrappers (the ever/always pair eDisjoint/aDisjoint +# share one ea_* function; the shift/scale/shift_scale trio share one C function). +# The tag value runs from @csqlfn up to the next doxygen tag or the comment close. +_CSQLFN = re.compile(r"@csqlfn\b") +_CSQLFN_REF = re.compile(r"#(\w+)\s*\(\)") +_CSQLFN_END = re.compile(r"@\w|\*/") # After the doxygen close, the MEOS-C definition: an optional return-type line # (no parens/braces/;/=), then `name(`. _FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") @@ -27,15 +34,29 @@ def _meos_to_mdb(meos_src): - """MEOS-C function name -> MobilityDB-C wrapper name (from @csqlfn).""" + """MEOS-C function name -> ordered list of MobilityDB-C wrapper names (from + @csqlfn). One MEOS function can back more than one wrapper — the ever/always + pair eDisjoint/aDisjoint share a single ea_* function tagged + `@csqlfn #Edisjoint_…() #Adisjoint_…()` — so each @csqlfn carries one or more + #Wrapper() references; collect them all (mirrors _mdb_to_sql collecting every + @sqlfn rather than the first).""" out = {} for cf in Path(meos_src).rglob("*.c"): text = cf.read_text(errors="ignore") for m in _CSQLFN.finditer(text): - mdb_c = m.group(1) + tail = text[m.end():] + end = _CSQLFN_END.search(tail) + value = tail[:end.start()] if end else tail + wrappers = _CSQLFN_REF.findall(value) + if not wrappers: + continue fm = _FNDEF.search(text, m.end()) - if fm: - out.setdefault(fm.group(1), mdb_c) + if not fm: + continue + lst = out.setdefault(fm.group(1), []) + for w in wrappers: + if w not in lst: + lst.append(w) return out @@ -69,19 +90,28 @@ def attach_sqlfn_map(idl, meos_src, mdb_src): d2s = _mdb_to_sql(mdb_src) n = 0 for f in idl["functions"]: - mdb_c = m2d.get(f["name"]) - if not mdb_c: + wrappers = m2d.get(f["name"]) + if not wrappers: continue - lst = d2s.get(mdb_c) - if not lst: + # A MEOS function can back several wrappers (the ever/always pair), each + # carrying its own @sqlfn; collect the (sqlfn, sqlop) pairs across all of + # them in order, keeping the primary (first) wrapper for back-compat. + pairs = [] + for w in wrappers: + for entry in d2s.get(w, []): + if entry not in pairs: + pairs.append(entry) + if not pairs: continue - f["mdbC"] = mdb_c - f["sqlfn"] = lst[0][0] - if lst[0][1]: - f["sqlop"] = lst[0][1] - # Shared wrapper exposing >1 SQL name: record them all (binding picks). - if len(lst) > 1: - f["sqlfnAll"] = [s for s, _ in lst] + f["mdbC"] = wrappers[0] + f["sqlfn"] = pairs[0][0] + if pairs[0][1]: + f["sqlop"] = pairs[0][1] + # Shared wrapper OR ever/always pair exposing >1 SQL name: record them all. + if len(pairs) > 1: + f["sqlfnAll"] = [s for s, _ in pairs] + if len(wrappers) > 1: + f["mdbCAll"] = wrappers n += 1 return idl, n From d97112528168e2a30e5a319e01d3c9cfd604b279 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 23 Jun 2026 10:27:41 +0200 Subject: [PATCH 07/13] Lint @sqlfn names that collide only by case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add lint_sqlfn_case_collisions next to lint_ea_sqlfn: it groups every @sqlfn (and sqlfnAll) name by its lower-case form and flags any group with more than one spelling (e.g. tDistance vs tdistance), wired into run.py as a loud warning. Such names are the SAME SQL function — PostgreSQL folds unquoted identifiers to lower case — so the clash is invisible in SQL and pg_regress. But the binding name is case-sensitive, and case-insensitive engines (Spark SQL) register every spelling under one UDF, so one silently shadows the other (e.g. the trgeometry tdistance shadowing the tgeo tDistance dispatch, returning NULL). Surfacing it at catalog generation lets the casing straggler be fixed at the MEOS-C @sqlfn source before it reaches any binding. --- parser/sqlfn.py | 18 ++++++++++++++++++ run.py | 13 ++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/parser/sqlfn.py b/parser/sqlfn.py index 6391df2..0c0fcf1 100644 --- a/parser/sqlfn.py +++ b/parser/sqlfn.py @@ -138,3 +138,21 @@ def lint_ea_sqlfn(idl): if sf and m and re.match(r"^[ea][A-Z]", sf) and sf[0] != m.group(1): bad.append((f["name"], sf)) return bad + + +def lint_sqlfn_case_collisions(idl): + """Return [(lower, [spelling, ...])] for @sqlfn names that collide + case-insensitively but differ in case (e.g. tDistance vs tdistance). + + PostgreSQL folds unquoted identifiers to lower case, so the two spell the + SAME SQL function and the clash is invisible in SQL / pg_regress. But the + binding name is taken case-SENSITIVELY, and case-insensitive engines (Spark + SQL, …) register every spelling under one UDF — so one silently shadows the + other. A canonical binding name must have exactly ONE spelling; surface a + casing straggler here before it reaches a binding.""" + by_lower = {} + for f in idl["functions"]: + for sf in [f.get("sqlfn"), *f.get("sqlfnAll", [])]: + if sf: + by_lower.setdefault(sf.lower(), set()).add(sf) + return sorted((lo, sorted(sp)) for lo, sp in by_lower.items() if len(sp) > 1) diff --git a/run.py b/run.py index 40f325d..139da5c 100644 --- a/run.py +++ b/run.py @@ -8,7 +8,7 @@ from parser.typerecover import recover_collapsed_types from parser.shapeinfer import infer_shapes from parser.nullable import merge_nullable -from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn +from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn, lint_sqlfn_case_collisions HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -74,6 +74,17 @@ def main(): f"(fix at source — wrong @sqlfn resolved):", file=sys.stderr) for cname, sf in ea_bad: print(f" {cname} -> @sqlfn {sf}", file=sys.stderr) + # Guard: @sqlfn names that differ only by case (e.g. tDistance vs tdistance) + # are the SAME SQL function (PostgreSQL folds the identifier) but DISTINCT + # binding names — a case-insensitive engine (Spark SQL) registers both under + # one UDF, so one silently shadows the other. Invisible in SQL; surface the + # casing straggler here, to be fixed at the MEOS-C @sqlfn source. + case_bad = lint_sqlfn_case_collisions(idl) + if case_bad: + print(f" ⚠ {len(case_bad)} @sqlfn case-collision(s) (pick ONE canonical " + f"spelling at the MEOS-C source — binding-breaking otherwise):", file=sys.stderr) + for _lo, spellings in case_bad: + print(f" {' vs '.join(spellings)}", file=sys.stderr) idl_path = OUTPUT_DIR / "meos-idl.json" with open(idl_path, "w") as f: From 12bab4705943b631f59ab62584197f45a9149445 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Sat, 13 Jun 2026 11:59:33 +0200 Subject: [PATCH 08/13] Attach the doxygen module group (@ingroup) to the catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse the @ingroup meos_ tag from each MEOS-C function's source comment (meos/src) and attach it as a 'group' field on every catalog function. Those groups are the structure of the MEOS reference manual / doxygen XML (meos_setspan_accessor, meos_temporal_comp_ever, meos_geo_constructor, ...), so every catalog-driven binding can organize its generated surface the same way the manual does — a function is found in the same place across all tools. The meos_internal_* groups are carried through too, so a binding can filter the non-user-facing surface. 2155 user-facing functions across 175 groups. --- parser/doxygroup.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ run.py | 7 +++++++ 2 files changed, 51 insertions(+) create mode 100644 parser/doxygroup.py diff --git a/parser/doxygroup.py b/parser/doxygroup.py new file mode 100644 index 0000000..b809d13 --- /dev/null +++ b/parser/doxygroup.py @@ -0,0 +1,44 @@ +"""Attach the doxygen module group (@ingroup) to the MEOS-API catalog. + +Every MEOS-C function carries an `@ingroup meos_` doxygen tag in its +source comment block (meos/src). Those groups ARE the structure of the MEOS +reference manual / doxygen XML — e.g. `meos_setspan_accessor`, +`meos_temporal_comp_ever`, `meos_geo_constructor`. Carrying the group into the +catalog lets every binding organize its generated surface the SAME way the +manual does, so a function is found in the same place across all tools. + +Adds per function (when found): `group`. The `meos_internal_*` groups are +MEOS-internal, not user-facing — they are tagged like any other so a binding +can filter them out, but they are NOT a separate concept here. +""" +import re +from pathlib import Path + +_INGROUP = re.compile(r"@ingroup\s+(meos_\w+)") +# Same shape as sqlfn._FNDEF: after the doxygen close, an optional return-type +# line (no parens/braces/;/=), then `name(`. +_FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") + + +def _name_to_group(meos_src): + """MEOS-C function name -> doxygen @ingroup group (first occurrence wins).""" + out = {} + for cf in Path(meos_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _INGROUP.finditer(text): + grp = m.group(1) + fm = _FNDEF.search(text, m.end()) + if fm: + out.setdefault(fm.group(1), grp) + return out + + +def attach_groups(idl, meos_src): + n2g = _name_to_group(meos_src) + n = 0 + for f in idl["functions"]: + g = n2g.get(f["name"]) + if g: + f["group"] = g + n += 1 + return idl, n diff --git a/run.py b/run.py index 139da5c..37c8df0 100644 --- a/run.py +++ b/run.py @@ -9,6 +9,7 @@ from parser.shapeinfer import infer_shapes from parser.nullable import merge_nullable from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn, lint_sqlfn_case_collisions +from parser.doxygroup import attach_groups HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -86,6 +87,12 @@ def main(): for _lo, spellings in case_bad: print(f" {' vs '.join(spellings)}", file=sys.stderr) + # 5. Attach the doxygen module group (@ingroup) from the vendored source, so + # bindings organize their generated surface like the reference manual. + if MEOS_SRC.exists(): + idl, ngrp = attach_groups(idl, MEOS_SRC) + print(f"[5/5] Attached {ngrp} doxygen @ingroup groups", file=sys.stderr) + idl_path = OUTPUT_DIR / "meos-idl.json" with open(idl_path, "w") as f: json.dump(idl, f, indent=2) From b129ec30e3a11ab5b654525a0d99c964ef4668d9 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Mon, 18 May 2026 09:08:03 +0200 Subject: [PATCH 09/13] feat: derive service-projection metadata for the catalog Adds parser/enrich.py: a clang-free pass that derives, for every function and opaque type, the metadata a service generator (OpenAPI/MCP/gRPC) needs but that C headers do not contain: - category (constructor/predicate/io/accessor/transformation/...) - typeEncodings: per opaque type, its text/MF-JSON/WKB in/out functions - network.exposable: whether the function maps to a stateless endpoint, with a precise reason when it does not (array/out-param, no encoder/ decoder, lifecycle, index) - wire: per-parameter and return request/response representation Matches the canonical C spellings libclang emits (struct-qualified names, unsigned char, int-returning predicates) and treats enums as scalars. Runs before merge_meta so every derived field is overridable from meta/meos-meta.json. Wired into run.py as step 2/3, documented in docs/enrichment.md, and tested on real signatures in tests/test_enrich.py (stdlib unittest, no libclang/pytest needed). Validated against the live MobilityDB master catalog: 2672 functions, 1790 (67%) stateless-exposable. --- README.md | 60 +++- docs/enrichment.md | 251 +++++++++++++++++ parser/enrich.py | 549 ++++++++++++++++++++++++++++++++++++ parser/extractors.py | 45 +++ parser/header_types.py | 123 ++++++++ report.py | 153 ++++++++++ run.py | 15 +- tests/test_coverage_gate.py | 53 ++++ tests/test_enrich.py | 287 +++++++++++++++++++ tests/test_header_types.py | 98 +++++++ tests/test_report.py | 72 +++++ 11 files changed, 1702 insertions(+), 4 deletions(-) create mode 100644 docs/enrichment.md create mode 100644 parser/enrich.py create mode 100644 parser/header_types.py create mode 100644 report.py create mode 100644 tests/test_coverage_gate.py create mode 100644 tests/test_enrich.py create mode 100644 tests/test_header_types.py create mode 100644 tests/test_report.py diff --git a/README.md b/README.md index 18da57d..f803a98 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This catalog is the foundation for generating language bindings (Python, Java, G - [How it works](#how-it-works) - [Getting started](#getting-started) - [Output format](#output-format) +- [Service-projection metadata](#service-projection-metadata) - [Adding metadata](#adding-metadata) - [Portable bare-name dialect](#portable-bare-name-dialect) @@ -60,10 +61,12 @@ service contracts generated from the same model: ## How it works -The pipeline runs in two steps: +The pipeline runs in four steps: 1. **Parser** — scans the MEOS `.h` header files using libclang and extracts every function signature, struct, and enum into structured JSON. -2. **Merger** — enriches the parser output with manual annotations from `meta/meos-meta.json`, such as documentation and memory ownership rules. +2. **Reconcile** — restores opaque types the PostgreSQL stub headers `#define` to `int` (`Interval`, `text`, …) from the header source, so they are not mistaken for `int *` out-parameters. +3. **Enrich** — derives the service-projection metadata (`category` / `typeEncodings` / `network` / `wire`). +4. **Merger** — applies manual annotations from `meta/meos-meta.json` (documentation, ownership, overrides) on top. ## Getting started @@ -127,9 +130,60 @@ A typical function entry looks like this: } ``` +## Service-projection metadata + +C headers describe *signatures*; they do not say what a function **is**, how an +opaque type crosses the wire, or whether an operation can be served +*statelessly*. A second pass (`parser/enrich.py`) derives that — the metadata a +service generator (OpenAPI, MCP, gRPC, …) needs to project MEOS onto a network +API. It runs **before** the merge, so every derived field is overridable from +`meta/meos-meta.json`. + +Each function gains a `category`, a `network` verdict, and a `wire` mapping: + +```json +{ + "name": "temporal_eq", + "returnType": { "c": "bool", "canonical": "int" }, + "params": [ { "name": "temp1", "canonical": "const struct Temporal *" }, + { "name": "temp2", "canonical": "const struct Temporal *" } ], + "category": "predicate", + "network": { "exposable": true, "method": "POST", "reason": null }, + "wire": { + "params": [ + { "name": "temp1", "kind": "serialized", "cType": "const struct Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] }, + { "name": "temp2", "kind": "serialized", "cType": "const struct Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] } + ], + "result": { "kind": "json", "json": "integer" } + } +} +``` + +(MEOS predicates return `int`, and libclang emits canonical spellings such as +`const struct Temporal *` — the enrichment matches those.) + +```text +Live coverage (MobilityDB master): 2161 public + 511 internal functions. +The service projects the public user API; internal (meos_internal*.h, +Datum-generic) is policy-excluded. + 1963 / 2161 = 91% of the public API stateless-exposable (verified). +``` + +The catalog also gains a top-level `typeEncodings` map (opaque type → its +in/out functions) and an `enrichment` summary (category counts, exposable +count) for coverage tracking. Non-exposable functions carry a precise +`reason` (`array-or-out-param:…`, `no-encoder:…`, `lifecycle`, `index`, …) so +generators can report exactly what they can and cannot emit. + +See [`docs/enrichment.md`](docs/enrichment.md) for the full contract and +[`tests/test_enrich.py`](tests/test_enrich.py) for worked examples on real +MEOS signatures (run: `python3 tests/test_enrich.py`). + ## Adding metadata -Manual annotations (ownership rules, additional documentation, deprecation flags, etc.) live in `meta/meos-meta.json`. The merger applies them on top of the libclang-parsed structure when generating the final catalog. +Manual annotations (ownership rules, additional documentation, deprecation flags, etc.) live in `meta/meos-meta.json`. The merger applies them on top of the libclang-parsed structure when generating the final catalog — including any field derived by the service-projection pass (e.g. correcting a `category` or forcing `network.exposable`). ## Portable bare-name dialect diff --git a/docs/enrichment.md b/docs/enrichment.md new file mode 100644 index 0000000..388cf09 --- /dev/null +++ b/docs/enrichment.md @@ -0,0 +1,251 @@ +# Service-projection enrichment + +The libclang parser can only report what is written in the C headers: +function signatures, structs, enums. Projecting MEOS onto a network service +(OpenAPI / MCP / gRPC) needs information the headers do **not** contain — what +each function *is*, how each opaque type crosses the wire, and whether an +operation can be served *statelessly*. + +`parser/enrich.py` derives that metadata from the parsed catalog. It runs +**before** `merge_meta`, so every field below can be overridden per +function/type from `meta/meos-meta.json`. + +> All values here are heuristic defaults. They are intended to be *curated* +> over time through `meta/meos-meta.json`; the heuristics give a correct-by- +> default starting point and keep coverage measurable. + +## 1. Function `category` + +Each function gets one `category` (first matching rule wins): + +| category | rule (by name / signature) | +|------------------|-------------------------------------------------------------| +| `lifecycle` | name starts `meos_` (init/finalize/configuration) | +| `index` | name starts `rtree_` | +| `io` | name matches an in/out encoding pattern (`_in`, `_out`, `_from_mfjson`, `_as_hexwkb`, `_from_wkb`, …) | +| `aggregate` | name ends `_transfn`/`_combinefn`/`_finalfn`, or `_tagg`/`_collect` | +| `predicate` | comparison/topological/temporal predicate by name (`*_eq`, `*_lt`, `contains_*`, `intersects_*`, `dwithin_*`, `ever_*`, `always_*`, `t{eq,contains,…}_*`, …) or returns `bool`. MEOS predicates return `int`, so name patterns — not the return type — drive this. | +| `constructor` | name ends `_make`/`_copy` | +| `setop` | name starts `union_`/`intersection_`/`minus_`/`difference_` | +| `conversion` | name contains `_to_`/`_from_`/`_from_base`/`_as_` | +| `accessor` | name ends with a component/property pattern (`_value`, `_srid`, `_duration`, `_num_*`, …) | +| `transformation` | default: value → value of the same family | + +## 2. `typeEncodings` + +A top-level map: opaque C type → how it round-trips to the wire. Built by +scanning the catalog for the type's own in/out functions. + +```json +"typeEncodings": { + "Temporal": { + "encodings": ["mfjson", "text", "wkb"], + "decoders": { "text": "temporal_in", "mfjson": "temporal_from_mfjson" }, + "encoders": { "text": "temporal_out", "wkb": "temporal_as_hexwkb" }, + "in": "temporal_in", + "out": "temporal_out" + } +} +``` + +- **decoder** — `const char * (+ aux) → T *` (`*_in`, `*_from_mfjson`, …) +- **encoder** — `const T * (+ aux) → char *` (`*_out`, `*_as_mfjson`, …) +- `in`/`out` — the preferred decoder/encoder, `text` > `mfjson` > `wkb`; + among candidates the **generic root** (`_in`/`_out`) is preferred + (so `temporal_out` serialises *every* subtype), else a deterministic + alphabetical pick. `in_aux`/`out_aux` carry the trailing args. + +> **Auxiliary arguments.** Real MEOS in/out wrappers (the public functions +> in the `*_meos.c` files) are not pure `(str)->T` / `(T)->str`: they take +> trailing *formatting* scalars — `temporal_out(temp, int maxdd)`, +> `*_as_mfjson(temp, with_bbox, flags, precision, srs)`. Those are safe to +> default (`maxdd`/`precision` → 15, flags/bbox → 0, `srs` → NULL), so the +> wrapper still satisfies the stateless contract; the defaults are recorded +> in `in_aux`/`out_aux` for the runtime to pass. A trailing arg that is +> *not* a defaultable formatting scalar disqualifies the wrapper: a +> semantic `*type` tag (`temporal_in`'s `temptype` — tagged +> `@ingroup meos_internal` in MEOS) or a pointer/array (`*_as_wkb`'s +> `size_out`). So polymorphic `Temporal` *decoding* resolves to a typed +> wrapper (`tbool_in`, …) — subtype-narrow on input; carrying the subtype +> on the wire for a universal decode is future work. *Encoding* is already +> universal via the generic `temporal_out`. + +The same data is folded onto each `structs[*]` entry as `serialization`. + +## 3. `network` and `wire` + +For every function: + +```json +"category": "predicate", +"network": { "exposable": true, "method": "POST", "reason": null }, +"wire": { + "params": [ + { "name": "temp1", "kind": "serialized", "cType": "Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] }, + { "name": "temp2", "kind": "serialized", "cType": "Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] } + ], + "result": { "kind": "json", "json": "boolean" } +} +``` + +`wire` element `kind`: + +| kind | meaning | JSON Schema hint | +|--------------|------------------------------------------------------|------------------| +| `json` | scalar; `json` ∈ integer/number/boolean/string. Enum types add `"enum": ""` | direct | +| `serialized` | opaque value carried as a string in its `encodings`; `decode`/`encode` names the MEOS function | `{"type":"string"}` + media type | +| `array` | JSON array of `element` (an `Elem` builder param, or an `Elem **`+count return) | `{"type":"array","items":…}` | +| `void` | no return value | `204` | +| `unsupported`| cannot be represented in a stateless request/response| — | + +> **Canonical spellings.** libclang emits canonical C, not source aliases: +> `struct Temporal *` (not `Temporal *`), `unsigned char` (not `uint8_t`), +> `long` (not `int64_t`), and MEOS uses `int` for booleans. The heuristics +> match canonical spellings; `struct`/`union`/`enum` qualifiers are stripped +> so `typeEncodings` keys are clean (`Temporal`, not `struct Temporal`). +> `enum` parameters are scalars; only declared structs become `serialized`. + +### Exposability + +`network.exposable` is `true` iff **every** parameter is `json` or +`serialized` **and** the result is `json`, `serialized`, or `void`. +Otherwise `exposable` is `false` and `reason` lists the blockers +(deduplicated, `;`-joined): + +| reason | cause | +|------------------------------|-------------------------------------------------------------| +| `lifecycle` / `index` | library plumbing, not a domain operation | +| `array-or-out-param:` | parameter is a pointer to a scalar/array or an out-parameter (`T **`, `int *`, `double *`, …) | +| `no-decoder:` | opaque parameter type has no parser function | +| `no-encoder:` | opaque return type has no serializer function | +| `unsupported-return:` | return cannot be represented on the wire | + +> **Out-parameters.** A common MEOS accessor shape — +> `bool f(.., T *result)` (or `T **result`) — returns its value through a +> trailing out-parameter, with the `bool`/`int` return acting as a presence +> flag (`void` = always present). Two safe shapes are recognised: a +> **scalar** `T *result` becomes the JSON `result`; an **opaque** +> `T **result` becomes a `serialized` result via the type's encoder. In +> both cases `from_outparam`/`out_ctype`/`presence_return` annotate the +> wire result, the function is `exposable`, and a false presence return +> maps to *no value* (HTTP 204). This recovers the public `*_value_n`, +> box-bound, and `*_value_at`/geo-accessor families. + +> **Input-array builders.** A builder taking `(Elem **arr, int count)` +> becomes one wire param of `kind: "array"` (element = the serialized +> `Elem`); the `count` is implicit (the JSON array length). This recovers +> `*_make` / `*_merge_array` / `*arr_to_*` builders whose element type is +> decodable. + +> **Array returns.** An accessor `Elem **f(.., int *count)` returning a +> freshly-allocated element array becomes a `result` of `kind: "array"` +> (element = the serialized `Elem`, `count_outparam` names the byref +> length). Recovers `temporal_instants`/`segments`/`sequences`, +> `tgeo_values`, `geo_pointarr`, … whose element type is encodable. + +This is the precise, machine-checkable boundary of what a generator can emit +today. Functions still blocked only by `array-or-out-param` (multi- or +array out-params, builder `T **`+count) are the candidates for the next, +hand-designed composite-endpoint unit; everything `exposable` can be +generated mechanically. + +## 4. Overriding + +`merge_meta` applies `meta/meos-meta.json` *after* enrichment, so any derived +field can be corrected by hand: + +```json +{ + "functions": { + "temporal_at_value": { "category": "transformation" }, + "some_fn": { "network": { "exposable": false, "reason": "side-effect" } } + } +} +``` + +## 5. Catalog summary + +`enrich_idl` adds an `enrichment` block for coverage tracking. Run against +the live MobilityDB `master` catalog (2672 functions, 47 structs, 6 enums): + +```json +"enrichment": { + "categoryCounts": { ... }, + "publicFunctions": 2161, + "internalFunctions": 511, + "exposableFunctions": 1963 +} +``` + +MEOS has two API surfaces: the **public user API** (`meos.h` + the public +type headers, 2161 functions) and the **internal programmer API** +(`meos_internal*.h`, 511 functions — type-erased `Datum`-generic, +undocumented for end users). A network service projects the *user* API, so +internal functions are **policy-excluded** (`reason: internal`, like +`lifecycle`/`index`); `132/133` `Datum` functions are internal and never +belonged in the parity denominator. + +So **1963 / 2161 = 91% of the public API** projects onto a stateless +endpoint as-is — verified by a strict invariant (every exposable function +has only scalar/enum/serialized/array params with a real decoder, and an +encodable result; 0 violations; 0 internal leaks). The 209-function public +remainder is dominated by **irregular** signatures with no clean stateless +shape (mixed/odd `array-or-out-param`, raw-array `unsupported-return`, +out-params of codec-less types) plus `SkipList` aggregate state and +`lifecycle`/`index` plumbing — all excluded with a truthful `reason`, +never silently mis-called. + +`report.py` emits `output/meos-coverage.json`: an **actionable worklist**. +Every non-exposable public function carries a `class` and a concrete +`suggest` — the precise upstream regularization that closes it — and +`byClass` ranks classes by leverage. Live (`master`, gap 198): + +| class | n | upstream action | +|---|---|---| +| `out-param-naming` | 76 | rename the lone out-parameter to `result` (one-liner each) | +| `plumbing` | 37 | none — `lifecycle`/`index`, intentionally not exposed | +| `stateful` | 30 | none — aggregate state; needs a stateful endpoint, not a stateless RPC | +| `array-return-shape` | 21 | add a trailing `int *count` (+ element encoder) | +| `multi-out` | 13 | return a struct, or split into single-result accessors | +| `other` / `no-codec` / `array-shape` / `internal-generic` | 11+7+3+1 | add a 1-arg `T_in`/`T_out`; keep `Datum`-generic internal | + +**Honest ceiling.** ~91% is the *safe principled* maximum for this layer. +The `out-param-naming` 76 are **not** closed on this side on purpose: +a name-agnostic "trailing pointer ⇒ out-parameter" rule would misread +genuine pointer *inputs* as out-parameters — silently wrong answers. +Correctness-over-coverage makes these a cheap **upstream** rename, now +enumerated. Everything remaining is therefore upstream (precisely +specified above) or definitional (`stateful`/`plumbing` — correct, +labelled exclusions, never silent). The catalog regenerates from headers, +so upstream fixes flow in automatically on the next run (no coupling). + +### Compensation register (retirement path) + +Each heuristic exists only to absorb a current irregularity and is +**deleted** once the irregularity is uniformized upstream: + +| compensation | absorbs | retire when upstream… | +|---|---|---| +| `header_types.reconcile` / `_preserved_opaque` | stub `#define`s erasing `Interval`/`text`/`Datum` to `int` | the public headers no longer route opaque types through `int` stubs | +| `_aux_specs` (default `maxdd`/mfjson flags) | in/out wrappers carrying formatting args | I/O wrappers are pure `(str)->T`/`(T)->str` | +| typed-decode (`tbool_in` for polymorphic `Temporal`) | generic `temporal_in` needing a `meosType` tag | a tag-free polymorphic decoder exists | +| out-param / array-builder / array-return shapes | irregular out/array signatures | signatures follow the canonical shapes in the worklist | + +`tests/test_coverage_gate.py` asserts coverage does not regress. + +> **Type reconciliation.** The PostgreSQL stub headers `#define` +> `Interval`/`text`/`TimestampTz`/… to `int` *before* libclang parses, so +> those opaque pointers reached the catalog as `int *` — +> indistinguishable from a real `int *` out-parameter. `parser/ +> header_types.py` re-scans the header *source* and restores the true +> named type wherever libclang produced a bare scalar but the source +> declares a distinct named pointer (scalar typedefs like `TimestampTz` +> are deliberately left resolved). This is primarily a *correctness* fix — +> `add_timestamptz_interval`'s `interv` is now honestly `Interval`, not a +> phantom `int *`. `text *` is then treated as a JSON string (it *is* one), +> and the opaque-codec gate spans *any* named non-scalar pointer type (not +> just parsed structs), so reconciled types register their own in/out +> (`Interval` ↔ `pg_interval_in`/`interval_out`). Together this lifts +> verified public coverage to 1963/2161 (91%). diff --git a/parser/enrich.py b/parser/enrich.py new file mode 100644 index 0000000..e699e85 --- /dev/null +++ b/parser/enrich.py @@ -0,0 +1,549 @@ +"""Service-projection enrichment. + +Derives, for every function and type in the parsed catalog, the metadata a +service generator (OpenAPI, MCP, gRPC, ...) needs but that *cannot* be read +from C headers: + +- ``category`` — a coarse semantic class (constructor, predicate, io, ...). +- ``typeEncodings``— for each opaque C type, how it round-trips to the wire + (text / MF-JSON / WKB) and the function names that do it. +- ``network`` — whether the function can be projected onto a *stateless* + endpoint, and if not, why. +- ``wire`` — per-parameter and return value, the concrete request / + response representation a generator should emit. + +Everything here is a heuristic *default*. The pass runs before the manual +merge step, so any field can be overridden per function/type from +``meta/meos-meta.json`` (the merger applies on top). This module is +deliberately free of any libclang dependency: it operates purely on the +parsed ``idl`` dict and is therefore unit-testable on its own. + +Note: libclang emits *canonical* C spellings — ``struct Temporal *`` (not +``Temporal *``), ``unsigned char`` (not ``uint8_t``), ``long`` (not +``int64_t``), and MEOS uses ``int`` for booleans. The heuristics below match +those canonical spellings. See ``docs/enrichment.md`` for the full contract. +""" + +import re + +# Ordered category vocabulary. The first matching rule wins. +CATEGORIES = ( + "lifecycle", # library/process setup, configuration, teardown + "index", # in-memory index objects (RTree, ...) + "io", # parse/serialize between a value and a wire encoding + "aggregate", # aggregate transition/combine/final functions + "predicate", # boolean question about value(s) + "constructor", # build a value (_make, _copy) + "setop", # set algebra (union/intersection/minus/...) + "conversion", # convert one value into another representation + "accessor", # read a component/property of a value + "transformation", # value -> value of the same family + "other", # anything not classified above +) + +# Canonical scalar spellings as emitted by libclang. +_INT_BASES = { + "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", + "long", "unsigned long", "long long", "unsigned long long", +} +_FLOAT_BASES = {"float", "double", "long double"} +_BOOL_BASES = {"bool", "_Bool"} +# char-like pointers carry text or bytes (WKB) — represented as a JSON string. +_STRING_PTR_BASES = {"char", "signed char", "unsigned char"} + +# name suffix -> wire encoding produced (encoder) / consumed (decoder) +_ENCODERS = [ + (re.compile(r"_out$"), "text"), + (re.compile(r"_as_text$"), "text"), + (re.compile(r"_as_e?wkt$"), "text"), + (re.compile(r"_as_mfjson$"), "mfjson"), + (re.compile(r"_as_geojson$"), "mfjson"), + (re.compile(r"_as_hex_?wkb$"), "wkb"), + (re.compile(r"_as_e?wkb$"), "wkb"), +] +_DECODERS = [ + (re.compile(r"_in$"), "text"), + (re.compile(r"_from_e?wkt$"), "text"), + (re.compile(r"_from_text$"), "text"), + (re.compile(r"_from_mfjson$"), "mfjson"), + (re.compile(r"_from_geojson$"), "mfjson"), + (re.compile(r"_from_hex_?wkb$"), "wkb"), + (re.compile(r"_from_e?wkb$"), "wkb"), +] +_IO_RE = [rx for rx, _ in _DECODERS + _ENCODERS] + +_LIFECYCLE_RE = re.compile(r"^meos_") +_INDEX_RE = re.compile(r"^rtree_") +_AGG_RE = re.compile(r"(_transfn|_combinefn|_finalfn)$|_tagg|_collect$") +_CONSTRUCTOR_RE = re.compile(r"(_make|_copy)$") +_SETOP_RE = re.compile(r"^(union|intersection|minus|difference)_") +_CONVERSION_RE = re.compile(r"_to_|_from_base|_from_|_as_") +_ACCESSOR_RE = re.compile( + r"(_values?|_start_value|_end_value|_min_value|_max_value|_srid|_timespan|" + r"_duration|_length|_num_[a-z]+|_n|_lower|_upper|_start_[a-z]+|_end_[a-z]+|" + r"_get_[a-z]+|_value_n)$" +) +# MEOS predicates return `int`, so they must be recognised by name. +_PREDICATE_RE = re.compile( + r"^(ever|always)_|(_eq|_ne|_lt|_le|_gt|_ge|_cmp)$|" + r"^(contains|contained|overlaps|overbefore|overafter|overleft|overright|" + r"overbelow|overabove|overfront|overback|left|right|below|above|front|" + r"back|before|after|adjacent|same|intersects|disjoint|touches|dwithin|" + r"covers|coveredby|equals|crosses|within|relate)_|" + r"^t(eq|ne|lt|le|gt|ge|contains|intersects|disjoint|touches|dwithin)_" +) + +_QUAL_RE = re.compile(r"\b(const|volatile|struct|union|enum)\b") + + +def _base(c_type: str) -> str: + """Bare type token: qualifiers, ``struct``/``union``/``enum`` and ``*`` + stripped (so ``const struct Temporal *`` -> ``Temporal``).""" + return " ".join(_QUAL_RE.sub(" ", c_type).replace("*", " ").split()) + + +def _ptr_depth(c_type: str) -> int: + return c_type.count("*") + + +def _scalar_wire(c_type: str, enums: set): + """Wire descriptor for a non-opaque scalar/string/enum, else ``None``.""" + base, depth = _base(c_type), _ptr_depth(c_type) + if base == "void" and depth == 0: + return {"kind": "void"} + # char-likes and PostgreSQL `text` are just strings on the wire. + if depth == 1 and (base in _STRING_PTR_BASES or base == "text"): + return {"kind": "json", "json": "string"} + if depth == 0: + if base in _BOOL_BASES: + return {"kind": "json", "json": "boolean"} + if base in _FLOAT_BASES: + return {"kind": "json", "json": "number"} + if base in _INT_BASES: + return {"kind": "json", "json": "integer"} + if base in enums: + return {"kind": "json", "json": "string", "enum": base} + return None + + +def _is_scalar_pointer(c_type: str, enums: set) -> bool: + """``int *`` / ``double *`` / ``interpType *`` etc. — an array/out-param, + as opposed to a pointer to an opaque value (``struct Temporal *``).""" + base, depth = _base(c_type), _ptr_depth(c_type) + if depth >= 2: + return True + if depth == 1 and base not in _STRING_PTR_BASES: + return (base in _INT_BASES or base in _FLOAT_BASES + or base in _BOOL_BASES or base in enums) + return False + + +def _aux_specs(params: list): + """Defaults for the trailing args of an in/out helper. + + Real MEOS in/out helpers are not pure ``(str)->T`` / ``(T)->str``: they + take trailing *formatting* scalars — ``temporal_out(temp, int maxdd)``, + ``*_as_mfjson(temp, with_bbox, flags, precision, srs)``. Those are safe + to default, so the helper still satisfies the stateless contract. + + Returns the aux spec list (one ``{name, kind, default}`` per trailing + param), or ``None`` if any trailing parameter is *not* a defaultable + formatting scalar — a semantic ``*type`` tag (``temporal_in``'s + ``temptype``), a pointer/array (``*_as_wkb``'s ``size_out``), etc. — + which disqualifies the helper entirely. + """ + specs = [] + for p in params: + sc = _scalar_wire(p["canonical"], set()) + if sc is None or sc.get("kind") != "json": + return None # pointer / array / opaque aux + nm = p["name"].lower() + if "type" in nm: # temptype/basetype/settype tag + return None + j = sc["json"] + if j == "integer": + default = (15 if any(k in nm for k in + ("maxdd", "decimal", "digit", "precision")) + else 0) + elif j == "number": + default = 0.0 + elif j == "boolean": + default = False + else: # string (e.g. srs) -> NULL + default = None + specs.append({"name": p["name"], "kind": j, "default": default}) + return specs + + +def build_type_encodings(functions: list, structs: set) -> dict: + """Scan the catalog for the in/out functions of every opaque struct. + + A *decoder* turns a wire string into an object (returns ``struct T *``, + first arg a char-like string). An *encoder* turns an object into a wire + string (returns ``char *``, first arg ``const struct T *``). Trailing + *formatting* scalars are allowed and defaulted (see ``_aux_specs``); a + non-defaultable trailing arg disqualifies the helper. Only declared + structs qualify, so primitives never register by accident. + """ + enc: dict[str, dict] = {} + + def slot(b: str) -> dict: + return enc.setdefault(b, {"encodings": set(), "encoders": {}, + "decoders": {}}) + + for fn in functions: + name = fn["name"] + ret = fn["returnType"]["canonical"] + params = fn.get("params", []) + if not params: + continue + p0 = params[0]["canonical"] + rb, rd = _base(ret), _ptr_depth(ret) + pb, pd = _base(p0), _ptr_depth(p0) + aux = _aux_specs(params[1:]) # None => non-defaultable trailing arg + + # Decoder: const char* (+ defaultable scalar aux) -> opaque struct + if (aux is not None and rd >= 1 and rb in structs + and pd == 1 and pb in _STRING_PTR_BASES): + for rx, encoding in _DECODERS: + if rx.search(name): + s = slot(rb) + s["encodings"].add(encoding) + s["decoders"].setdefault(encoding, {})[name] = aux + break + + # Encoder: const struct T* (+ defaultable scalar aux) -> char* + if (aux is not None and rd == 1 and rb in _STRING_PTR_BASES + and pd >= 1 and pb in structs): + for rx, encoding in _ENCODERS: + if rx.search(name): + s = slot(pb) + s["encodings"].add(encoding) + s["encoders"].setdefault(encoding, {})[name] = aux + break + + # Several typed functions can serve one encoding (e.g. tbool_in, + # tint_in, ... all decode a `Temporal *`). Prefer the *generic root* + # (`_in` / `_out`) so the chosen in/out works for every subtype + # (and `temporal_out` correctly serialises any subtype); fall back to a + # deterministic alphabetical pick otherwise. + order = ("text", "mfjson", "wkb") + dec_suffix = {"text": "_in", "mfjson": "_from_mfjson", + "wkb": "_from_hexwkb"} + enc_suffix = {"text": "_out", "mfjson": "_as_mfjson", + "wkb": "_as_hexwkb"} + + def choose(cands: dict, base: str, suffix: str) -> str: + generic = base.lower() + suffix + return generic if generic in cands else sorted(cands)[0] + + out: dict[str, dict] = {} + for base, s in enc.items(): + dec = {e: choose(c, base, dec_suffix[e]) + for e, c in s["decoders"].items()} + encd = {e: choose(c, base, enc_suffix[e]) + for e, c in s["encoders"].items()} + in_e = next((e for e in order if e in dec), None) + out_e = next((e for e in order if e in encd), None) + out[base] = { + "encodings": sorted(s["encodings"]), + "decoders": dec, + "encoders": encd, + "in": dec.get(in_e) if in_e else None, + "out": encd.get(out_e) if out_e else None, + "in_aux": s["decoders"][in_e][dec[in_e]] if in_e else [], + "out_aux": s["encoders"][out_e][encd[out_e]] if out_e else [], + } + return out + + +def classify_category(fn: dict) -> str: + name = fn["name"] + ret = fn["returnType"]["canonical"] + + if _LIFECYCLE_RE.match(name): + return "lifecycle" + if _INDEX_RE.match(name): + return "index" + if any(rx.search(name) for rx in _IO_RE): + return "io" + if _AGG_RE.search(name): + return "aggregate" + if _PREDICATE_RE.search(name) or _base(ret) in _BOOL_BASES: + return "predicate" + if _CONSTRUCTOR_RE.search(name): + return "constructor" + if _SETOP_RE.match(name): + return "setop" + if _CONVERSION_RE.search(name): + return "conversion" + if _ACCESSOR_RE.search(name): + return "accessor" + return "transformation" + + +# MEOS splits its API: the public *user* surface (meos.h + the public type +# headers) and the *internal* programmer surface. The latter is type-erased +# (``Datum``-generic), undocumented for end users, and must not be projected +# onto a network service — it is policy-excluded, like lifecycle/index. +_INTERNAL_FILES = {"meos_internal.h", "meos_internal_geo.h"} + + +def _outparam(fn: dict, enums: set, type_encodings: dict): + """A MEOS accessor of the form ``bool f(.., T *result)`` returns its + value through a trailing out-parameter, with the ``bool``/``int`` return + (or ``void``) as a presence flag. Two safe shapes: + + - ``scalar``: ``T *result`` where ``T`` is a JSON scalar. + - ``opaque``: ``T **result`` where ``T`` has an encoder (serialised). + + Returns ``(leading_params, outparam, mode)`` or ``(None, None, None)``. + """ + ps = fn.get("params", []) + ret = fn["returnType"]["canonical"] + if (not ps or _ptr_depth(ret) != 0 + or _base(ret) not in ("bool", "_Bool", "int", "void")): + return None, None, None + last = ps[-1] + if last["name"] not in ("result", "value"): + return None, None, None + c = last["canonical"] + if _ptr_depth(c) == 1: + pointee = c.replace("const", "").replace("*", "").strip() + sw = _scalar_wire(pointee, enums) + if sw is not None and sw.get("kind") == "json": + return ps[:-1], last, "scalar" + elif _ptr_depth(c) == 2: + te = type_encodings.get(_base(c)) + if te and te.get("out"): + return ps[:-1], last, "opaque" + return None, None, None + + +def _array_param(params: list, type_encodings: dict): + """A MEOS builder takes an element array as a ``(Elem **arr, int count)`` + pair. Detect the first such pair whose element type is decodable, so the + array can be projected as a JSON list (the ``count`` is then implicit). + Returns ``(arr_index, count_index)`` or ``(None, None)``. + """ + for i, p in enumerate(params): + c = p["canonical"] + if (_ptr_depth(c) == 2 and p["name"] not in ("result", "value") + and i + 1 < len(params)): + te = type_encodings.get(_base(c)) + nxt = params[i + 1]["canonical"] + if (te and te.get("in") and _base(nxt) == "int" + and _ptr_depth(nxt) == 0): + return i, i + 1 + return None, None + + +def _array_return(fn: dict, type_encodings: dict): + """A MEOS accessor that returns a freshly-allocated element array as + ``Elem **f(.., int *count)`` (e.g. ``temporal_sequences``). The element + type must be encodable. Returns ``(count_param_name, elem_base)`` or + ``(None, None)``. + """ + ret = fn["returnType"]["canonical"] + if _ptr_depth(ret) != 2: + return None, None + rb = _base(ret) + te = type_encodings.get(rb) + if not (te and te.get("out")): + return None, None + cnt = [p for p in fn.get("params", []) + if _ptr_depth(p["canonical"]) == 1 + and _base(p["canonical"]) in ("int", "long") + and p["name"] in ("count", "n", "nvalues", "size", "npoints")] + if len(cnt) != 1: + return None, None + return cnt[0]["name"], rb + + +def assess(fn: dict, type_encodings: dict, enums: set) -> tuple: + """Return ``(network, wire)`` for one function. + + Exposable over a stateless endpoint iff every parameter can be decoded + from the request and the return can be encoded into the response. + Pointer-to-scalar parameters (arrays / out-params) and opaque types + lacking an in/out function make it non-exposable; the reason is recorded. + """ + reasons: list[str] = [] + wire_params = [] + + out_lead, out_p, out_mode = _outparam(fn, enums, type_encodings) + eff = out_lead if out_p is not None else fn.get("params", []) + arr_i, count_i = _array_param(eff, type_encodings) + ret_count_name, ret_elem = _array_return(fn, type_encodings) + ret_count_i = (next((i for i, p in enumerate(eff) + if p["name"] == ret_count_name), None) + if ret_elem is not None else None) + + for idx, p in enumerate(eff): + if idx == count_i or idx == ret_count_i: + continue # array length is implicit + c = p["canonical"] + if idx == arr_i: + elem = type_encodings[_base(c)] + wire_params.append({ + "name": p["name"], "kind": "array", + "count_param": eff[count_i]["name"], + "element": { + "kind": "serialized", + "cType": " ".join(c.replace("*", " ").split()) + " *", + "decode": elem["in"], + "decode_aux": elem.get("in_aux", []), + "encodings": elem["encodings"], + }, + }) + continue + scalar = _scalar_wire(c, enums) + if scalar is not None: + wire_params.append({"name": p["name"], **scalar}) + continue + if _is_scalar_pointer(c, enums): + reasons.append(f"array-or-out-param:{p['name']}") + wire_params.append({"name": p["name"], "kind": "unsupported"}) + continue + base = _base(c) + te = type_encodings.get(base) + if te and te["in"]: + wire_params.append({ + "name": p["name"], "kind": "serialized", "cType": c, + "decode": te["in"], "decode_aux": te.get("in_aux", []), + "encodings": te["encodings"], + }) + else: + reasons.append(f"no-decoder:{base}") + wire_params.append({"name": p["name"], "kind": "unsupported"}) + + ret = fn["returnType"]["canonical"] + if out_p is not None: + # a bool/int C return is a presence flag; void = always present + presence = _base(ret) in ("bool", "_Bool", "int") + if out_mode == "scalar": + pointee = out_p["canonical"].replace("const", "").replace( + "*", "").strip() + wire_result = { + **_scalar_wire(pointee, enums), # kind:"json", json:… + "from_outparam": out_p["name"], + "out_ctype": out_p["canonical"], + "presence_return": presence, + } + else: # opaque T **result + te = type_encodings[_base(out_p["canonical"])] + wire_result = { + "kind": "serialized", + "cType": out_p["canonical"], + "encode": te["out"], "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"], + "from_outparam": out_p["name"], + "out_ctype": out_p["canonical"], + "presence_return": presence, + } + scalar = "handled" + elif ret_elem is not None: + te = type_encodings[ret_elem] + wire_result = { + "kind": "array", + "element": { + "kind": "serialized", + "cType": " ".join(ret.replace("*", " ").split()) + " *", + "encode": te["out"], "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"], + }, + "count_outparam": ret_count_name, + } + scalar = "handled" + else: + scalar = _scalar_wire(ret, enums) + if scalar == "handled": + pass + elif scalar is not None: + wire_result = scalar + elif _is_scalar_pointer(ret, enums): + reasons.append(f"unsupported-return:{ret}") + wire_result = {"kind": "unsupported"} + else: + base, depth = _base(ret), _ptr_depth(ret) + te = type_encodings.get(base) + if depth == 1 and te and te["out"]: + wire_result = {"kind": "serialized", "cType": ret, + "encode": te["out"], + "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"]} + else: + reasons.append( + f"no-encoder:{base}" if depth == 1 + else f"unsupported-return:{ret}" + ) + wire_result = {"kind": "unsupported"} + + if fn["category"] in ("lifecycle", "index"): + reasons.insert(0, fn["category"]) + if fn.get("api") == "internal": + reasons.insert(0, "internal") + + exposable = not reasons + network = { + "exposable": exposable, + "method": "POST" if exposable else None, + "reason": None if exposable else "; ".join(dict.fromkeys(reasons)), + } + return network, {"params": wire_params, "result": wire_result} + + +def enrich_idl(idl: dict) -> dict: + """Augment ``idl`` in place with service-projection metadata.""" + functions = idl.get("functions", []) + struct_names = {s["name"] for s in idl.get("structs", [])} + enum_names = {e["name"] for e in idl.get("enums", [])} + + # An opaque type is any *named* pointer type that is not a scalar / enum + # / string. Beyond parsed structs this also covers reconciled + # PostgreSQL/PostGIS types (`Interval`, `GBOX`, ...) so their own in/out + # wrappers can register a codec instead of being dead `no-decoder`s. + _scalarish = (_INT_BASES | _FLOAT_BASES | _BOOL_BASES | _STRING_PTR_BASES + | enum_names | {"void", "text"}) + opaque_names = set(struct_names) + for fn in functions: + for c in ([fn["returnType"]["canonical"]] + + [p["canonical"] for p in fn.get("params", [])]): + b = _base(c) + if _ptr_depth(c) >= 1 and b and b not in _scalarish: + opaque_names.add(b) + + type_encodings = build_type_encodings(functions, opaque_names) + + for fn in functions: + fn["api"] = ("internal" if fn.get("file") in _INTERNAL_FILES + else "public") + fn["category"] = classify_category(fn) + network, wire = assess(fn, type_encodings, enum_names) + fn["network"] = network + fn["wire"] = wire + + for struct in idl.get("structs", []): + te = type_encodings.get(struct["name"]) + if te: + struct["serialization"] = { + "encodings": te["encodings"], "in": te["in"], "out": te["out"], + } + idl["typeEncodings"] = type_encodings + + counts: dict[str, int] = {} + for fn in functions: + counts[fn["category"]] = counts.get(fn["category"], 0) + 1 + public = [fn for fn in functions if fn["api"] == "public"] + idl["enrichment"] = { + "categoryCounts": counts, + "publicFunctions": len(public), + "internalFunctions": len(functions) - len(public), + # Internal functions are policy-excluded, so this equals the + # public-exposable count — the meaningful parity numerator. + "exposableFunctions": sum( + 1 for fn in functions if fn["network"]["exposable"] + ), + } + return idl diff --git a/parser/extractors.py b/parser/extractors.py index a5855e9..e67f542 100644 --- a/parser/extractors.py +++ b/parser/extractors.py @@ -31,6 +31,48 @@ def _c_spelling(ty) -> str: return spelling +# Canonical spellings of plain C scalars/builtins. +_SCALAR_CANON = { + "void", "_Bool", "bool", "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", "long", + "unsigned long", "long long", "unsigned long long", + "float", "double", "long double", +} +# Named opaque types that the PostgreSQL *stub* headers collapse to a bare +# scalar even without a pointer (type-erased values). Kept by name so they +# read as themselves, not as the stub's underlying integer. +_EXPLICIT_OPAQUE = {"Datum"} + + +def _strip(s: str) -> str: + return " ".join( + re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", s) + .replace("*", " ").split() + ) + + +def _preserved_opaque(ty) -> str | None: + """Keep the *declared* name of opaque types the PG stubs canonicalise to + a bare scalar (``Interval *`` / ``text *`` -> ``const int *``, ``Datum`` + -> ``unsigned long``). A pointer whose typedef'd pointee resolves to a + plain scalar is, in practice, always a stubbed opaque struct — so the + declared spelling is the truthful one. Genuine scalar pointers + (``int *result``) are unaffected: their pointee is a builtin, not a + distinct typedef name. + """ + if ty.kind == clang.cindex.TypeKind.POINTER: + pointee = ty.get_pointee() + dname = _strip(pointee.spelling) + cname = _strip(pointee.get_canonical().spelling) + if (dname and dname not in _SCALAR_CANON and "(" not in dname + and cname in _SCALAR_CANON and dname != cname): + return ty.spelling.replace("_Bool", "bool") + return None + if _strip(ty.spelling) in _EXPLICIT_OPAQUE: + return _strip(ty.spelling) + return None + + def _canonical_c_spelling(ty) -> str: # Like ``_canonical_spelling`` but normalises boolean types to ``"bool"``. # Handles: @@ -42,6 +84,9 @@ def _canonical_c_spelling(ty) -> str: # Fallback: also catch _Bool reached through other typedef chains if ty.get_canonical().kind == clang.cindex.TypeKind.BOOL: return "bool" + preserved = _preserved_opaque(ty) + if preserved is not None: + return preserved return _canonical_spelling(ty) diff --git a/parser/header_types.py b/parser/header_types.py new file mode 100644 index 0000000..550f3ee --- /dev/null +++ b/parser/header_types.py @@ -0,0 +1,123 @@ +"""Recover opaque parameter/return types from the header *source*. + +The PostgreSQL stub headers ``#define`` several opaque types (``Interval``, +``text``, ``TimestampTz`` …) to ``int`` *before* libclang parses, so the +typedef name is destroyed: a ``const Interval *`` argument reaches the +catalog as ``const int *``, indistinguishable from a real ``int *`` +out-parameter and impossible to project correctly. + +The public MEOS headers, however, declare every function on a regular +``extern ();`` line with the *true* spellings. This +module re-scans that source and reconciles the catalog: where libclang +produced a bare scalar but the header says a distinct **named pointer** +type, the header is the truth. Scalar typedefs (``TimestampTz``, ``int64``, +enums) are deliberately left as their resolved scalar — only mis-rendered +*opaque pointers* are restored. + +Pure text + ``re``; no libclang. +""" + +import re +from pathlib import Path + +_COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.DOTALL) +_DECL = re.compile( + r"\bextern\b\s+(?P[^;{}]+?\([^;{}]*\))\s*;", re.DOTALL) +_FUNC = re.compile(r"^(?P.+?)\b(?P\w+)\s*\((?P.*)\)$", + re.DOTALL) + +_SCALARS = { + "void", "bool", "_Bool", "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", "long", + "unsigned long", "long long", "unsigned long long", "size_t", + "float", "double", "long double", + # scalar typedefs we *want* left resolved to their integer form + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", + "uint64", "int8_t", "int16_t", "int32_t", "int64_t", "uint8_t", + "uint16_t", "uint32_t", "uint64_t", "TimestampTz", "TimeADT", + "DateADT", "Timestamp", "Datum", "meosType", "interpType", +} + + +def _norm(t: str) -> str: + return " ".join(t.replace("*", " * ").split()) + + +def _base(t: str) -> str: + return " ".join( + re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", t) + .replace("*", " ").split() + ) + + +def _split_params(s: str) -> list: + out, depth, cur = [], 0, "" + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if ch == "," and depth == 0: + out.append(cur) + cur = "" + else: + cur += ch + if cur.strip(): + out.append(cur) + return out + + +def _param_type(p: str) -> str: + p = p.strip() + if p in ("void", ""): + return "void" + p = re.sub(r"\[[^\]]*\]", " *", p) # arr[] -> arr * + m = re.match(r"^(.*?)(\w+)\s*$", p, re.DOTALL) # strip trailing name + return _norm(m.group(1) if m and "*" not in m.group(2) else p) + + +def scan_headers(headers_dir: Path) -> dict: + """``{func_name: {"ret": type, "params": [type, …]}}`` from source.""" + out: dict = {} + for h in sorted(Path(headers_dir).glob("**/*.h")): + text = _COMMENT.sub(" ", h.read_text(errors="replace")) + for d in _DECL.finditer(text): + m = _FUNC.match(" ".join(d.group("sig").split())) + if not m: + continue + params = [_param_type(x) for x in _split_params(m.group("params"))] + out[m.group("name")] = { + "ret": _norm(m.group("ret")), + "params": params if params != ["void"] else [], + } + return out + + +def _restore(decl_canon: str, header_t: str, enums: set) -> str | None: + """Return the header type if ``decl_canon`` is a scalar but the header + says a distinct named *pointer* opaque type, else ``None``.""" + cb = _base(decl_canon) + hb, hd = _base(header_t), header_t.count("*") + if (hd >= 1 and hb and hb not in _SCALARS and hb not in enums + and cb in _SCALARS): + return _norm(header_t) + return None + + +def reconcile(idl: dict, headers_dir: Path) -> dict: + """Restore opaque pointer types the stub headers erased to ``int``.""" + headers = scan_headers(headers_dir) + enums = {e["name"] for e in idl.get("enums", [])} + for fn in idl.get("functions", []): + h = headers.get(fn["name"]) + if not h: + continue + rt = fn["returnType"] + fixed = _restore(rt["canonical"], h["ret"], enums) + if fixed: + rt["c"] = rt["canonical"] = fixed + for p, ht in zip(fn.get("params", []), h["params"]): + fixed = _restore(p["canonical"], ht, enums) + if fixed: + p["cType"] = p["canonical"] = fixed + return idl diff --git a/report.py b/report.py new file mode 100644 index 0000000..29faa70 --- /dev/null +++ b/report.py @@ -0,0 +1,153 @@ +# Emit a coverage / irregularity report from the enriched catalog. +# +# python run.py # produce output/meos-idl.json +# python report.py # -> output/meos-coverage.json (+ stderr summary) +# +# `worklist` is the actionable form: one entry per non-exposable public +# function with a `class` and a concrete `suggest` — the precise upstream +# regularization that would make it stateless-projectable (e.g. "rename the +# out-parameter to `result`", "add a trailing `int *count`", "return a +# struct instead of N out-parameters", "add a single-arg `T_in`/`T_out`"). +# `byClass` ranks the classes by size, so the upstream work is prioritised +# by leverage. Fixing an irregularity upstream removes its entry and lifts +# coverage toward 100%; internal (`meos_internal*.h`) functions and +# inherently-stateful aggregates are reported but are correct exclusions, +# never silent gaps. This is the direct, no-coupling input for the +# cross-repo API-uniformization workstream. + +import json +import re +import sys +from collections import Counter, defaultdict +from pathlib import Path + + +def _base(c: str) -> str: + return " ".join(re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", + c).replace("*", " ").split()) + + +def _dep(c: str) -> int: + return c.count("*") + + +def _classify(fn: dict): + """``(klass, suggestion)`` — the concrete upstream regularization that + would make this function stateless-projectable. Turns the worklist from + *what is broken* into *what to change upstream*. + """ + reason = fn["network"]["reason"] or "" + tags = {p.split(":")[0] for p in reason.split("; ")} + detail = {p.split(":", 1)[1] for p in reason.split("; ") if ":" in p} + params = fn.get("params", []) + ret = fn.get("returnType", {}).get("canonical", "") + + if tags & {"lifecycle", "index"}: + return ("plumbing", + "intentionally not exposed (process/library plumbing)") + if fn.get("category") == "aggregate" or "SkipList" in detail: + return ("stateful", + "stateful aggregation — expose via a stateful endpoint, " + "not a stateless RPC") + if "Datum" in detail or "MeosArray" in detail: + return ("internal-generic", + "Datum/array-generic — expose a typed variant; keep the " + "generic form internal") + + outptrs = [p for p in params if _dep(p["canonical"]) >= 2 + or (_dep(p["canonical"]) == 1 and _base(p["canonical"]) in + ("int", "long", "double", "float", "bool"))] + if "unsupported-return" in tags and _dep(ret) >= 1: + return ("array-return-shape", + f"array return `{ret}` lacks a length: add a trailing " + "`int *count` out-parameter (+ an element encoder)") + if len(outptrs) >= 2: + return ("multi-out", + f"{len(outptrs)} out-parameters: return a struct (or split " + "into separate single-result accessors)") + if len(outptrs) == 1 and outptrs[0]["name"] not in ("result", "value"): + return ("out-param-naming", + f"rename out-parameter `{outptrs[0]['name']}` to `result` " + "(`bool f(.., T *result)` convention)") + nocodec = sorted(d for d in detail if d[:1].isupper()) + if tags & {"no-decoder", "no-encoder"} and nocodec: + t = nocodec[0] + return ("no-codec", + f"type `{t}` has no stateless codec: add a single-argument " + f"`{t.lower()}_in`/`{t.lower()}_out` wrapper, or keep it " + "internal") + if "array-or-out-param" in tags: + return ("array-shape", + "pass element arrays as an adjacent `(Elem **arr, int " + "count)`; use `bool f(.., T *result)` for out-values") + return ("other", "regularize the signature to a stateless shape") + + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("output/meos-coverage.json") + + +def build_report(catalog: dict) -> dict: + fns = catalog.get("functions", []) + pub = [f for f in fns if f.get("api") == "public"] + exposable = [f for f in pub if f["network"]["exposable"]] + by_reason: dict = defaultdict(list) + for f in pub: + if f["network"]["exposable"]: + continue + # collapse "tag:detail; tag:detail" to the set of distinct tags + tags = sorted({p.split(":")[0] + for p in (f["network"]["reason"] or "").split("; ")}) + by_reason["; ".join(tags)].append(f["name"]) + + worklist = [] + for f in pub: + if f["network"]["exposable"]: + continue + klass, suggest = _classify(f) + worklist.append({ + "name": f["name"], "file": f.get("file"), + "reason": f["network"]["reason"], + "class": klass, "suggest": suggest, + }) + worklist.sort(key=lambda w: (w["class"], w["name"])) + by_class = Counter(w["class"] for w in worklist) + + total = len(pub) + n_exp = len(exposable) + return { + "publicTotal": total, + "exposable": n_exp, + "coveragePct": round(n_exp * 100 / total, 1) if total else 0, + "internalExcluded": len(fns) - total, + "gap": total - n_exp, + "byClass": dict(by_class.most_common()), + "byReason": {k: sorted(v) + for k, v in sorted(by_reason.items(), + key=lambda kv: -len(kv[1]))}, + # actionable: one upstream-change suggestion per gap function + "worklist": worklist, + } + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + catalog = json.loads(IN_PATH.read_text()) + if not any("network" in f for f in catalog.get("functions", [])): + sys.exit(f"{IN_PATH} is not enriched.") + + rep = build_report(catalog) + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(rep, indent=2)) + + print(f"[coverage] public {rep['exposable']}/{rep['publicTotal']} " + f"({rep['coveragePct']}%), gap {rep['gap']}, " + f"{rep['internalExcluded']} internal excluded → {OUT_PATH}", + file=sys.stderr) + for klass, n in rep["byClass"].items(): + print(f" {n:4d} {klass}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/run.py b/run.py index 37c8df0..98a447b 100644 --- a/run.py +++ b/run.py @@ -10,6 +10,8 @@ from parser.nullable import merge_nullable from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn, lint_sqlfn_case_collisions from parser.doxygroup import attach_groups +from parser.header_types import reconcile +from parser.enrich import enrich_idl HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -43,6 +45,15 @@ def main(): print(f" nullable params from Doxygen `may be NULL`: {nn}", file=sys.stderr) + # 1d. Restore opaque types the PG stub headers #define'd away to int. + print(f" reconciling types from header source...", file=sys.stderr) + idl = reconcile(idl, HEADERS_DIR) + + # 1e. Derive service-projection metadata (category / encodings / network). + # Runs before the merge so manual annotations override the heuristics. + print(f" enriching {len(idl['functions'])} functions...", file=sys.stderr) + idl = enrich_idl(idl) + # 2. Merge with manual metadata if META_PATH.exists(): print(f"[2/3] Merging with {META_PATH}...", file=sys.stderr) @@ -99,7 +110,9 @@ def main(): print(f" → {idl_path} written", file=sys.stderr) pa = idl.get("portableAliases", {}).get("count", 0) - print(f"\nDone: {len(idl['functions'])} functions, " + exposable = idl.get("enrichment", {}).get("exposableFunctions", 0) + print(f"\nDone: {len(idl['functions'])} functions " + f"({exposable} stateless-exposable), " f"{len(idl['structs'])} structs, " f"{len(idl['enums'])} enums, " f"{pa} portable bare-name aliases", file=sys.stderr) diff --git a/tests/test_coverage_gate.py b/tests/test_coverage_gate.py new file mode 100644 index 0000000..5b0e8e9 --- /dev/null +++ b/tests/test_coverage_gate.py @@ -0,0 +1,53 @@ +"""Coverage non-regression gate. python3 tests/test_coverage_gate.py + +Skipped unless an enriched ``output/meos-idl.json`` is present (so CI +without libclang still passes). When present, it asserts public coverage +does not regress below the established floor and that the worklist stays +consistent — a heuristic change that silently drops coverage fails here. +""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from report import build_report + +_CATALOG = Path(__file__).resolve().parents[1] / "output" / "meos-idl.json" +_FLOOR = 90.0 # public %, ratchet up as upstream uniformization lands + + +@unittest.skipUnless(_CATALOG.exists(), "run `python run.py` first") +class CoverageGateTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.r = build_report(json.loads(_CATALOG.read_text())) + + def test_public_coverage_not_regressed(self): + self.assertGreaterEqual( + self.r["coveragePct"], _FLOOR, + f"public coverage {self.r['coveragePct']}% < floor {_FLOOR}% " + "— a heuristic change dropped coverage") + + def test_worklist_consistent(self): + # one actionable entry per gap; classes partition the gap + self.assertEqual(len(self.r["worklist"]), self.r["gap"]) + self.assertEqual(sum(self.r["byClass"].values()), self.r["gap"]) + self.assertTrue(all(w["suggest"] and w["class"] + for w in self.r["worklist"])) + + def test_no_internal_or_silent_gap(self): + # internal never counted as public; every gap has a reason + cat = json.loads(_CATALOG.read_text()) + for f in cat["functions"]: + if f.get("api") == "internal": + continue + if not f["network"]["exposable"]: + self.assertTrue(f["network"]["reason"], + f"{f['name']}: gap without a reason") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_enrich.py b/tests/test_enrich.py new file mode 100644 index 0000000..df2b823 --- /dev/null +++ b/tests/test_enrich.py @@ -0,0 +1,287 @@ +"""Unit tests for parser/enrich.py. + +Runs without libclang or pytest: python3 tests/test_enrich.py + +The fixture uses the *canonical* C spellings libclang actually emits +(``struct Temporal *``, ``unsigned char``, ``int`` for booleans, enum +parameters), so the assertions double as a specification. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.enrich import enrich_idl, classify_category, build_type_encodings + + +def fn(name, ret, *params): + return { + "name": name, + "file": "meos.h", + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t} for t, n in params], + } + + +T = "const struct Temporal *" +FUNCTIONS = [ + fn("temporal_in", "struct Temporal *", ("const char *", "str")), + fn("temporal_out", "char *", (T, "temp")), + fn("temporal_from_mfjson", "struct Temporal *", ("const char *", "str")), + fn("temporal_as_hexwkb", "char *", + (T, "temp"), ("unsigned char", "variant"), ("int *", "size_out")), + fn("bigintset_in", "struct Set *", ("const char *", "str")), + fn("bigintset_out", "char *", ("const struct Set *", "set")), + fn("temporal_eq", "int", (T, "temp1"), (T, "temp2")), + fn("tpoint_speed", "struct Temporal *", (T, "temp")), + fn("tjsonb_to_ttext", "struct Temporal *", (T, "temp")), + fn("union_set_set", "struct Set *", + ("const struct Set *", "s1"), ("const struct Set *", "s2")), + fn("temporal_num_instants", "int", (T, "temp")), + fn("tsequence_make", "struct TSequence *", + ("struct TInstant **", "instants"), ("int", "count"), + ("interpType", "interp")), + fn("tjsonb_value_at_timestamptz", "int", + (T, "temp"), ("long", "t"), ("int", "strict"), + ("struct Jsonb **", "value")), + fn("temporal_set_interp", "struct Temporal *", + (T, "temp"), ("interpType", "interp")), + fn("meos_initialize", "void"), + fn("rtree_insert", "void", + ("struct RTree *", "rtree"), ("void *", "box"), ("long", "id")), + fn("temporal_timestamps", "int *", (T, "temp"), ("int *", "count")), + # aux args: a defaultable formatting scalar (maxdd) is allowed and + # defaulted; a semantic *type tag disqualifies the helper entirely. + fn("box_in", "struct Box *", ("const char *", "str")), + fn("box_out", "char *", + ("const struct Box *", "box"), ("int", "maxdd")), + fn("weird_in", "struct Weird *", + ("const char *", "str"), ("int", "basetype")), + # An otherwise-exposable function living in the internal header: it must + # be policy-excluded (api=internal), like the programmer Datum API. + dict(fn("internal_op", "struct Temporal *", (T, "temp")), + file="meos_internal.h"), + # Scalar out-parameter accessor: bool f(.., int *result) — the value is + # returned through the trailing out-param, the bool is a presence flag. + fn("setspan_value_n", "int", + ("const struct Set *", "s"), ("int", "n"), ("int *", "result")), + # Opaque out-parameter accessor: bool f(.., Box **result) — the value + # comes back as an opaque pointer, serialised via the type's encoder. + fn("boxset_value_n", "int", + ("const struct Set *", "s"), ("int", "n"), + ("struct Box **", "result")), + # Input-array builder: f(Elem **arr, int count) — the (array,count) pair + # becomes one JSON-array wire param; the count is implicit. + fn("temporal_merge_array", "struct Temporal *", + ("struct Temporal **", "temparr"), ("int", "count")), + # Array return: Elem **f(.., int *count) — a freshly-allocated element + # array; the count out-param is implicit, result is a JSON array. + fn("temporal_components", "struct Temporal **", + (T, "temp"), ("int *", "count")), +] + +STRUCTS = [{"name": n, "fields": []} for n in + ("Temporal", "TSequence", "Set", "RTree", "Jsonb", "TInstant", + "Box", "Weird")] +ENUMS = [{"name": "interpType", "values": []}] + + +def make_idl(): + return enrich_idl({ + "functions": [dict(f, returnType=dict(f["returnType"]), + params=[dict(p) for p in f["params"]]) + for f in FUNCTIONS], + "structs": [dict(s) for s in STRUCTS], + "enums": [dict(e) for e in ENUMS], + }) + + +def by_name(idl): + return {f["name"]: f for f in idl["functions"]} + + +class CategoryTests(unittest.TestCase): + def test_categories(self): + c = {f["name"]: classify_category(f) for f in FUNCTIONS} + self.assertEqual(c["temporal_in"], "io") + self.assertEqual(c["temporal_from_mfjson"], "io") + self.assertEqual(c["temporal_as_hexwkb"], "io") + self.assertEqual(c["temporal_eq"], "predicate") # returns int + self.assertEqual(c["tpoint_speed"], "transformation") + self.assertEqual(c["tjsonb_to_ttext"], "conversion") + self.assertEqual(c["union_set_set"], "setop") + self.assertEqual(c["temporal_num_instants"], "accessor") + self.assertEqual(c["tsequence_make"], "constructor") + self.assertEqual(c["meos_initialize"], "lifecycle") + self.assertEqual(c["rtree_insert"], "index") + + +class TypeEncodingTests(unittest.TestCase): + def setUp(self): + self.te = build_type_encodings( + FUNCTIONS, {s["name"] for s in STRUCTS}) + + def test_struct_prefix_stripped_and_round_trip(self): + self.assertIn("Temporal", self.te) # not "struct Temporal" + # temporal_as_hexwkb is still excluded — its `size_out` is a pointer + # (out-param), not a defaultable scalar — so no wkb encoder here. + self.assertEqual(self.te["Temporal"]["encodings"], + ["mfjson", "text"]) + self.assertEqual(self.te["Temporal"]["in"], "temporal_in") + self.assertEqual(self.te["Temporal"]["out"], "temporal_out") + self.assertEqual(self.te["Set"]["in"], "bigintset_in") + self.assertEqual(self.te["Set"]["out"], "bigintset_out") + + def test_defaultable_aux_accepted_type_tag_rejected(self): + # box_out(box, int maxdd) qualifies; maxdd defaults to 15. + self.assertEqual(self.te["Box"]["out"], "box_out") + self.assertEqual(self.te["Box"]["out_aux"], + [{"name": "maxdd", "kind": "integer", + "default": 15}]) + self.assertEqual(self.te["Box"]["in"], "box_in") + self.assertEqual(self.te["Box"]["in_aux"], []) + # weird_in(str, int basetype): the *type tag disqualifies it, so + # Weird gets no decoder at all. + self.assertNotIn("Weird", self.te) + + def test_no_primitive_or_intermediate_false_positives(self): + self.assertNotIn("int", self.te) # was a real false positive + self.assertNotIn("char", self.te) + self.assertNotIn("TSequence", self.te) # builder-only type + for k in self.te: + self.assertNotIn("struct ", k) + + def test_struct_serialization_folded(self): + s = {x["name"]: x for x in make_idl()["structs"]} + self.assertIn("serialization", s["Temporal"]) + self.assertNotIn("serialization", s["TSequence"]) + + +class ExposabilityTests(unittest.TestCase): + def setUp(self): + self.fns = by_name(make_idl()) + + def n(self, name): + return self.fns[name]["network"] + + def test_int_returning_predicate_exposable(self): + self.assertTrue(self.n("temporal_eq")["exposable"]) + self.assertEqual(self.fns["temporal_eq"]["wire"]["result"], + {"kind": "json", "json": "integer"}) + + def test_serialized_round_trip(self): + w = self.fns["tpoint_speed"]["wire"] + self.assertTrue(self.n("tpoint_speed")["exposable"]) + self.assertEqual(w["params"][0]["kind"], "serialized") + self.assertEqual(w["params"][0]["decode"], "temporal_in") + self.assertEqual(w["result"]["encode"], "temporal_out") + + def test_enum_param_is_scalar_and_exposable(self): + f = self.fns["temporal_set_interp"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual(f["wire"]["params"][1], + {"name": "interp", "kind": "json", + "json": "string", "enum": "interpType"}) + + def test_io_parse_serialize_exposable(self): + for name in ("temporal_in", "temporal_out", "temporal_from_mfjson", + "bigintset_in", "bigintset_out"): + self.assertTrue(self.n(name)["exposable"], name) + + def test_out_param_not_exposable(self): + r = self.n("temporal_as_hexwkb")["reason"] + self.assertFalse(self.n("temporal_as_hexwkb")["exposable"]) + self.assertIn("array-or-out-param:size_out", r) + r2 = self.n("tjsonb_value_at_timestamptz")["reason"] + self.assertIn("array-or-out-param:value", r2) + + def test_array_param_and_missing_encoder(self): + r = self.n("tsequence_make")["reason"] + self.assertFalse(self.n("tsequence_make")["exposable"]) + self.assertIn("array-or-out-param:instants", r) + self.assertIn("no-encoder:TSequence", r) + + def test_array_return_not_exposable(self): + r = self.n("temporal_timestamps")["reason"] + self.assertFalse(self.n("temporal_timestamps")["exposable"]) + self.assertIn("unsupported-return:int *", r) + + def test_lifecycle_and_index_not_exposable(self): + self.assertIn("lifecycle", self.n("meos_initialize")["reason"]) + self.assertIn("index", self.n("rtree_insert")["reason"]) + + +class ApiClassificationTests(unittest.TestCase): + def setUp(self): + self.fns = by_name(make_idl()) + + def test_internal_policy_excluded(self): + f = self.fns["internal_op"] + self.assertEqual(f["api"], "internal") + self.assertFalse(f["network"]["exposable"]) + self.assertIn("internal", f["network"]["reason"]) + + def test_public_default(self): + self.assertEqual(self.fns["temporal_eq"]["api"], "public") + self.assertTrue(self.fns["temporal_eq"]["network"]["exposable"]) + + def test_scalar_outparam_projected_as_result(self): + f = self.fns["setspan_value_n"] + self.assertTrue(f["network"]["exposable"]) + pnames = [p["name"] for p in f["wire"]["params"]] + self.assertEqual(pnames, ["s", "n"]) # 'result' not a param + r = f["wire"]["result"] + self.assertEqual(r["kind"], "json") + self.assertEqual(r["json"], "integer") + self.assertEqual(r["from_outparam"], "result") + self.assertTrue(r["presence_return"]) # int return = presence + + def test_opaque_outparam_projected_as_serialized(self): + f = self.fns["boxset_value_n"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual([p["name"] for p in f["wire"]["params"]], ["s", "n"]) + r = f["wire"]["result"] + self.assertEqual(r["kind"], "serialized") # opaque -> encoded + self.assertEqual(r["encode"], "box_out") + self.assertEqual(r["from_outparam"], "result") + self.assertTrue(r["presence_return"]) + + def test_array_return(self): + f = self.fns["temporal_components"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual([p["name"] for p in f["wire"]["params"]], ["temp"]) + r = f["wire"]["result"] + self.assertEqual(r["kind"], "array") + self.assertEqual(r["count_outparam"], "count") + self.assertEqual(r["element"]["kind"], "serialized") + self.assertEqual(r["element"]["encode"], "temporal_out") + + def test_input_array_builder(self): + f = self.fns["temporal_merge_array"] + self.assertTrue(f["network"]["exposable"]) + params = f["wire"]["params"] + self.assertEqual(len(params), 1) # count is implicit + a = params[0] + self.assertEqual(a["name"], "temparr") + self.assertEqual(a["kind"], "array") + self.assertEqual(a["count_param"], "count") + self.assertEqual(a["element"]["kind"], "serialized") + self.assertEqual(a["element"]["decode"], "temporal_in") + self.assertEqual(f["wire"]["result"]["kind"], "serialized") + + +class SummaryTests(unittest.TestCase): + def test_enrichment_summary(self): + e = make_idl()["enrichment"] + self.assertEqual(sum(e["categoryCounts"].values()), len(FUNCTIONS)) + self.assertEqual(e["internalFunctions"], 1) # internal_op + self.assertEqual(e["publicFunctions"], len(FUNCTIONS) - 1) + # 13 + setspan_value_n + boxset_value_n + temporal_merge_array + # + temporal_components (array return); internal_op excluded. + self.assertEqual(e["exposableFunctions"], 17) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_header_types.py b/tests/test_header_types.py new file mode 100644 index 0000000..a91cb74 --- /dev/null +++ b/tests/test_header_types.py @@ -0,0 +1,98 @@ +"""Unit tests for parser/header_types.py. + +Runs without libclang or pytest: python3 tests/test_header_types.py +""" + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.header_types import scan_headers, reconcile + +_HEADER = """ +/* a comment + spanning lines */ +extern TimestampTz add_timestamptz_interval(TimestampTz t, + const Interval *interv); +extern bool contains_set_text(const Set *s, text *t); // trailing comment +extern bool bigintset_value_n(const Set *s, int n, int64 *result); +extern Temporal *temporal_copy(const Temporal *temp); +extern void meos_initialize(void); +""" + + +class ScanTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.TemporaryDirectory() + (Path(self.d.name) / "meos.h").write_text(_HEADER) + self.h = scan_headers(Path(self.d.name)) + + def tearDown(self): + self.d.cleanup() + + def test_signatures_recovered(self): + self.assertEqual(self.h["add_timestamptz_interval"]["params"], + ["TimestampTz", "const Interval *"]) + self.assertEqual(self.h["contains_set_text"]["params"], + ["const Set *", "text *"]) + self.assertEqual(self.h["meos_initialize"]["params"], []) + self.assertEqual(self.h["temporal_copy"]["ret"], "Temporal *") + + +class ReconcileTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.TemporaryDirectory() + (Path(self.d.name) / "meos.h").write_text(_HEADER) + + def tearDown(self): + self.d.cleanup() + + def idl(self): + # Mimic the libclang output *after* the stub erased the names. + return { + "enums": [], + "functions": [ + {"name": "add_timestamptz_interval", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "t", "cType": "int", "canonical": "int"}, + {"name": "interv", "cType": "const int *", + "canonical": "const int *"}]}, + {"name": "contains_set_text", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "s", "cType": "const struct Set *", + "canonical": "const struct Set *"}, + {"name": "t", "cType": "int *", + "canonical": "int *"}]}, + {"name": "bigintset_value_n", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "s", "cType": "const struct Set *", + "canonical": "const struct Set *"}, + {"name": "n", "cType": "int", "canonical": "int"}, + {"name": "result", "cType": "int *", + "canonical": "int *"}]}, + ], + } + + def test_opaque_pointers_restored_scalars_left_alone(self): + idl = reconcile(self.idl(), Path(self.d.name)) + f = {x["name"]: x for x in idl["functions"]} + # const Interval * restored from the header source + self.assertEqual(f["add_timestamptz_interval"]["params"][1]["canonical"], + "const Interval *") + # TimestampTz return stays the resolved scalar (int) — not restored + self.assertEqual(f["add_timestamptz_interval"]["returnType"]["canonical"], + "int") + # text * restored + self.assertEqual(f["contains_set_text"]["params"][1]["canonical"], + "text *") + # genuine int* out-param (header also says int64*) is a scalar + # pointer -> left exactly as libclang produced it + self.assertEqual(f["bigintset_value_n"]["params"][2]["canonical"], + "int *") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..213a921 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,72 @@ +"""Unit tests for report.py. python3 tests/test_report.py""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from report import build_report + + +def f(name, api, exposable, reason=None, ret="int", params=None): + return {"name": name, "api": api, + "returnType": {"canonical": ret}, "params": params or [], + "network": {"exposable": exposable, "reason": reason}} + + +CATALOG = {"functions": [ + f("temporal_eq", "public", True), + f("tpoint_speed", "public", True), + f("tsequence_make", "public", False, "array-or-out-param:instants"), + f("geo_collect", "public", False, + "array-or-out-param:a; no-decoder:GBOX"), + f("temporal_tagg", "public", False, "no-decoder:SkipList"), + f("meos_initialize", "public", False, "lifecycle"), + f("some_internal", "internal", False, "internal; no-decoder:Datum"), +]} + + +class ReportTests(unittest.TestCase): + def setUp(self): + self.r = build_report(CATALOG) + + def test_counts(self): + self.assertEqual(self.r["publicTotal"], 6) # excludes internal + self.assertEqual(self.r["exposable"], 2) + self.assertEqual(self.r["gap"], 4) + self.assertEqual(self.r["internalExcluded"], 1) + self.assertEqual(self.r["coveragePct"], round(2 * 100 / 6, 1)) + + def test_grouping_by_reason_tagset(self): + br = self.r["byReason"] + # detail is stripped; multi-tag reasons collapse to the tag set + self.assertIn("array-or-out-param", br) + self.assertEqual(br["array-or-out-param"], ["tsequence_make"]) + self.assertEqual(br["array-or-out-param; no-decoder"], ["geo_collect"]) + self.assertEqual(br["no-decoder"], ["temporal_tagg"]) + self.assertEqual(br["lifecycle"], ["meos_initialize"]) + # internal never appears as a public gap + self.assertNotIn("some_internal", + [n for v in br.values() for n in v]) + + def test_byreason_sorted_largest_first(self): + sizes = [len(v) for v in self.r["byReason"].values()] + self.assertEqual(sizes, sorted(sizes, reverse=True)) + + def test_worklist_is_actionable(self): + wl = {w["name"]: w for w in self.r["worklist"]} + self.assertEqual(len(wl), self.r["gap"]) # one entry per gap + self.assertNotIn("some_internal", wl) # internal excluded + # each gap gets a class + a concrete upstream suggestion + self.assertEqual(wl["meos_initialize"]["class"], "plumbing") + self.assertEqual(wl["temporal_tagg"]["class"], "stateful") + gc = wl["geo_collect"] # no-decoder:GBOX + self.assertEqual(gc["class"], "no-codec") + self.assertIn("gbox_in", gc["suggest"]) + self.assertTrue(all(w["suggest"] for w in self.r["worklist"])) + self.assertEqual(sum(self.r["byClass"].values()), self.r["gap"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 185b201a618195ae9434a10dfc82669a88815fc3 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Mon, 18 May 2026 09:38:07 +0200 Subject: [PATCH 10/13] feat: generate an OpenAPI 3.1 contract from the enriched catalog Adds generator/openapi.py + generate_openapi.py: projects the enriched MEOS catalog (network/wire/typeEncodings from the service-projection pass) onto an OpenAPI 3.1 document. - one RPC-style POST /{function} per stateless-exposable function (a function is to MEOS what a process is to OGC API - Processes) - opaque values cross the wire as strings in their typeEncodings (text/MF-JSON/HexWKB), surfaced as reusable component schemas - enums become string component schemas with the real C constant names - x-meos-{category,decode,encode,in,out,encodings} make the document self-describing for a downstream server/MCP/gRPC generator - pure dict -> dict (no libclang, no MEOS runtime), deterministic output Logically depends on the catalog being enriched. Validated against the live MobilityDB master catalog: 2672 functions -> 1790 operations over 14 component schemas, all $refs resolve. Documented in docs/openapi.md; tested in tests/test_openapi.py (stdlib unittest). --- README.md | 28 ++++++ docs/openapi.md | 75 +++++++++++++++ generate_openapi.py | 40 ++++++++ generator/openapi.py | 209 ++++++++++++++++++++++++++++++++++++++++++ tests/test_openapi.py | 183 ++++++++++++++++++++++++++++++++++++ 5 files changed, 535 insertions(+) create mode 100644 docs/openapi.md create mode 100644 generate_openapi.py create mode 100644 generator/openapi.py create mode 100644 tests/test_openapi.py diff --git a/README.md b/README.md index f803a98..7bb0254 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This catalog is the foundation for generating language bindings (Python, Java, G - [Service-projection metadata](#service-projection-metadata) - [Adding metadata](#adding-metadata) - [Portable bare-name dialect](#portable-bare-name-dialect) +- [OpenAPI generation](#openapi-generation) ## Ecosystem @@ -202,3 +203,30 @@ type-agnostic and applies to **every** temporal type family — must not be excluded from any parity headline. `python tools/portable_parity.py` audits it against the catalog — currently **29/29 = 100%** backed (verified, no guessing). See [`docs/portable-aliases.md`](docs/portable-aliases.md). + +## OpenAPI generation + +The enriched catalog (the `network` / `wire` / `typeEncodings` produced by the +service-projection pass) can be projected onto an **OpenAPI 3.1** contract — +this is the concrete "OpenAPI is a projection of MEOS-API" step: + +```bash +python run.py # produce the enriched catalog +python generate_openapi.py # output/meos-idl.json -> output/meos-openapi.json +``` + +Every *stateless-exposable* MEOS function becomes one RPC-style +`POST /{function}` operation (≈ an OGC API – Processes "process"); opaque +values cross the wire as strings carried in their `typeEncodings` +(text / MF-JSON / HexWKB), surfaced as reusable component schemas. `x-meos-*` +extensions carry the decode/encode function names and category so a +downstream server or MCP generator can consume the same document. + +Against the live MobilityDB `master` catalog this yields **1952 operations** +(90% of the public API; internal `meos_internal*.h` policy-excluded), +including array-of-string params for builders. The generator is pure +`dict` → `dict` (no libclang, +no MEOS runtime); see [`docs/openapi.md`](docs/openapi.md) for the projection +rules, `x-meos-*` extensions, and roadmap (OGC API, MCP, runtime server), and +[`tests/test_openapi.py`](tests/test_openapi.py) for worked examples +(`python3 tests/test_openapi.py`). diff --git a/docs/openapi.md b/docs/openapi.md new file mode 100644 index 0000000..68c5df0 --- /dev/null +++ b/docs/openapi.md @@ -0,0 +1,75 @@ +# OpenAPI projection + +`generator/openapi.py` turns the **enriched** catalog (`meos-idl.json` with +`category` / `network` / `wire` / `typeEncodings` — see +[`enrichment.md`](enrichment.md)) into an **OpenAPI 3.1** document. It is the +concrete realisation of "OpenAPI is a projection of MEOS-API": the canonical +semantic catalog is the single source, OpenAPI is one rendering of it. + +```bash +python run.py # enriched catalog -> output/meos-idl.json +python generate_openapi.py # -> output/meos-openapi.json +``` + +Pure `dict` → `dict`: no libclang, no MEOS runtime, deterministic output +(sorted paths/schemas) so generated diffs are reviewable. + +## Projection rules + +| MEOS concept | OpenAPI | +|---|---| +| stateless-exposable function | one `POST /{function}` operation, `operationId = function` | +| `category` | operation `tags` + `x-meos-category`; spec-level `tags` list | +| parameter | property of the JSON request body (all required, `additionalProperties:false`) | +| `wire.kind = json` scalar | `{"type": integer\|number\|boolean\|string}` | +| `wire` enum | `$ref` to a component enum schema (string, real C constant names) | +| `wire.kind = serialized` | `allOf` → `$ref` to the type's component schema, plus `x-meos-decode` (request) / `x-meos-encode` (response) | +| `wire.kind = array` (builder `(Elem **,count)`) | `{"type":"array","items":}` + `x-meos-decode`; the C `count` is the array length | +| out-parameter result (`from_outparam`) | the out-param value is the response (scalar JSON or serialized); `presence_return` false ⇒ `204` | +| `wire.result.kind = void` | `204 No Content` | +| any error | `default` → `#/components/responses/MeosError` | +| `typeEncodings[T]` | `components.schemas.T` = `{"type":"string", x-meos-encodings, x-meos-in, x-meos-out}` | + +RPC-style, not resource-style, is deliberate: MEOS is a value algebra, so a +function ≈ an **OGC API – Processes** "process". A resource model +(OGC API – Moving Features collections) is a different projection, layered +later (and already partly served by +[MobilityAPI](https://github.com/MobilityDB/MobilityAPI)). + +## `x-meos-*` extensions + +The spec is self-describing for downstream generators (server, MCP, gRPC): + +- `info.x-meos-coverage` — `{functions, exposed}`. +- operation `x-meos-category`, `x-meos-encode`. +- serialized request property `x-meos-decode` — the MEOS parse function. +- component schema `x-meos-encodings` / `x-meos-in` / `x-meos-out` — the wire + encodings and the MEOS in/out function names. + +A server generator marshals a request by calling `x-meos-decode` on each +serialized string, invoking the function, and calling `x-meos-encode` on the +result — no extra metadata needed beyond this document. + +## Coverage (live MobilityDB `master`) + +2161 **public** functions → **1952 operations (85%)** — the internal +`meos_internal*.h` programmer API (511 fns, `Datum`-generic) is +policy-excluded. Tagged across `predicate`, `transformation`, `accessor`, +`io`, `conversion`, `setop`, `aggregate`, `constructor`. The remaining +public functions (multi-out/array builders, opaque-no-codec, polymorphic +input-decode, lifecycle/index) carry a truthful `reason` and are +overridable via `meta/meos-meta.json`. + +## Limitations / roadmap + +- **No OpenAPI conformance validation** in-tree yet (structural checks only: + every path a single `POST` with responses, all `$ref`s resolve). Adding + `openapi-spec-validator` to CI is a follow-up. +- **MCP tool manifest** — the same `wire`/`typeEncodings` model maps directly + onto MCP tool schemas; a sibling generator is the natural next unit. +- **Runtime server** — a generated marshaling server (decode → call → encode) + is out of scope here; this PR delivers the *contract*, not the server. +- **OGC API – Moving Features** resource projection is a separate effort. +- Preferred in/out per type currently follows catalog scan order (e.g. + `tbool_in` may be picked over `temporal_in`); both are valid decoders, but + refining the preference is a small enrichment-side follow-up. diff --git a/generate_openapi.py b/generate_openapi.py new file mode 100644 index 0000000..0a12a11 --- /dev/null +++ b/generate_openapi.py @@ -0,0 +1,40 @@ +# Generate an OpenAPI 3.1 contract from the enriched MEOS catalog. +# +# Usage: +# python run.py # first, to produce the catalog +# python generate_openapi.py # reads output/meos-idl.json +# python generate_openapi.py path.json [out.json] + +import json +import sys +from pathlib import Path + +from generator.openapi import build_openapi + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("output/meos-openapi.json") + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + + catalog = json.loads(IN_PATH.read_text()) + if "functions" not in catalog or not any( + "network" in f for f in catalog["functions"] + ): + sys.exit(f"{IN_PATH} is not enriched (no `network` fields). " + "Run the enrichment pass first.") + + spec = build_openapi(catalog) + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(spec, indent=2)) + + print(f"[openapi] {len(spec['paths'])} operations, " + f"{len(spec['components']['schemas'])} component schemas " + f"→ {OUT_PATH}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/generator/openapi.py b/generator/openapi.py new file mode 100644 index 0000000..6244bac --- /dev/null +++ b/generator/openapi.py @@ -0,0 +1,209 @@ +"""OpenAPI 3.1 generator. + +Projects the *enriched* MEOS catalog (``meos-idl.json`` with ``category`` / +``network`` / ``wire`` / ``typeEncodings``, produced by ``parser/enrich.py``) +onto an OpenAPI 3.1 service contract. + +The projection is deliberately RPC-style — MEOS is a value algebra, not a +REST resource model, so each *stateless-exposable* function becomes one +``POST /{function}`` operation (≈ an OGC API – Processes "process"). Opaque +values cross the wire as strings carried in their `typeEncodings` (text / +MF-JSON / WKB); each opaque type and referenced enum becomes a reusable +component schema. ``x-meos-*`` extensions carry the decode/encode function +names and category so a downstream server or MCP generator can consume this +same document. + +Pure ``dict`` → ``dict``; no libclang and no MEOS runtime. Only functions +with ``network.exposable == true`` are emitted; the rest are reported by +``build_openapi``'s return-value count via the caller. +""" + +import re + +_QUAL_RE = re.compile(r"\b(const|volatile|struct|union|enum)\b") + +_PRIMITIVE = { + "integer": {"type": "integer"}, + "number": {"type": "number"}, + "boolean": {"type": "boolean"}, + "string": {"type": "string"}, +} + + +def _clean_type(c_type: str) -> str: + """``const struct Temporal *`` -> ``Temporal`` (matches typeEncodings keys).""" + return " ".join(_QUAL_RE.sub(" ", c_type).replace("*", " ").split()) + + +def _scalar_schema(wire: dict, used_enums: set) -> dict: + if wire.get("enum"): + used_enums.add(wire["enum"]) + return {"$ref": f"#/components/schemas/{wire['enum']}"} + return dict(_PRIMITIVE.get(wire.get("json", "string"), {"type": "string"})) + + +def _value_schema(wire: dict, used_types: set, used_enums: set) -> dict: + """Schema for one parameter or the result.""" + kind = wire["kind"] + if kind == "json": + return _scalar_schema(wire, used_enums) + if kind == "serialized": + t = _clean_type(wire["cType"]) + used_types.add(t) + return {"$ref": f"#/components/schemas/{t}"} + if kind == "array": + return {"type": "array", + "items": _value_schema(wire["element"], used_types, + used_enums)} + # Should not happen for an exposable function. + return {"type": "string"} + + +def _operation(fn: dict, used_types: set, used_enums: set) -> dict: + wire = fn["wire"] + op = { + "operationId": fn["name"], + "summary": fn.get("doc") or fn["name"], + "tags": [fn["category"]], + "x-meos-category": fn["category"], + } + + params = wire.get("params", []) + if params: + props, required = {}, [] + for p in params: + props[p["name"]] = _value_schema(p, used_types, used_enums) + required.append(p["name"]) + if p["kind"] == "serialized": + props[p["name"]] = { + "allOf": [props[p["name"]]], + "x-meos-decode": p["decode"], + } + elif p["kind"] == "array": + props[p["name"]] = { + **props[p["name"]], + "x-meos-decode": p["element"]["decode"], + } + op["requestBody"] = { + "required": True, + "content": {"application/json": {"schema": { + "type": "object", + "required": required, + "additionalProperties": False, + "properties": props, + }}}, + } + + result = wire["result"] + if result["kind"] == "void": + op["responses"] = {"204": {"description": "No content"}} + else: + schema = _value_schema(result, used_types, used_enums) + content_schema = schema + if result["kind"] == "serialized": + op["x-meos-encode"] = result["encode"] + op["responses"] = {"200": { + "description": "Result", + "content": {"application/json": {"schema": content_schema}}, + }} + op["responses"]["default"] = { + "$ref": "#/components/responses/MeosError" + } + return op + + +def _type_schema(name: str, type_encodings: dict) -> dict: + te = type_encodings.get(name) + if not te: + return {"type": "string", "title": name} + encs = te.get("encodings", []) + return { + "type": "string", + "title": name, + "description": ( + f"Serialized MEOS {name}. Wire encodings: {', '.join(encs)} " + f"(e.g. WKT / MF-JSON / HexWKB)." + ), + "x-meos-encodings": encs, + "x-meos-in": te.get("in"), + "x-meos-out": te.get("out"), + } + + +def _enum_schema(name: str, enums: list) -> dict: + for e in enums: + if e["name"] == name: + return { + "type": "string", + "title": name, + "enum": [v["name"] for v in e.get("values", [])], + "x-meos-c-enum": True, + } + return {"type": "string", "title": name} + + +def build_openapi(catalog: dict, *, title: str = "MEOS API", + version: str = "0.1.0") -> dict: + """Build an OpenAPI 3.1 document from an enriched catalog.""" + functions = sorted( + (f for f in catalog.get("functions", []) + if f.get("network", {}).get("exposable")), + key=lambda f: f["name"], + ) + type_encodings = catalog.get("typeEncodings", {}) + enums = catalog.get("enums", []) + + used_types: set = set() + used_enums: set = set() + paths: dict = {} + tags_seen: set = set() + + for fn in functions: + paths[f"/{fn['name']}"] = { + "post": _operation(fn, used_types, used_enums) + } + tags_seen.add(fn["category"]) + + schemas = {} + for t in sorted(used_types): + schemas[t] = _type_schema(t, type_encodings) + for e in sorted(used_enums): + schemas[e] = _enum_schema(e, enums) + + total = len(catalog.get("functions", [])) + return { + "openapi": "3.1.0", + "info": { + "title": title, + "version": version, + "description": ( + "Auto-generated from the MEOS-API catalog. Each operation is " + "a stateless-exposable MEOS function projected RPC-style as " + "`POST /{function}`; opaque values cross the wire as strings " + "in the encodings listed on their component schema. " + "Generated, do not edit by hand." + ), + "x-meos-coverage": { + "functions": total, + "exposed": len(functions), + }, + }, + "tags": [{"name": t} for t in sorted(tags_seen)], + "paths": dict(sorted(paths.items())), + "components": { + "schemas": schemas, + "responses": { + "MeosError": { + "description": "MEOS error", + "content": {"application/json": {"schema": { + "type": "object", + "properties": { + "error": {"type": "string"}, + "code": {"type": "integer"}, + }, + "required": ["error"], + }}}, + } + }, + }, + } diff --git a/tests/test_openapi.py b/tests/test_openapi.py new file mode 100644 index 0000000..2e33ff8 --- /dev/null +++ b/tests/test_openapi.py @@ -0,0 +1,183 @@ +"""Unit tests for generator/openapi.py. + +Runs without libclang or pytest: python3 tests/test_openapi.py +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from generator.openapi import build_openapi + +TEMP = "const struct Temporal *" + + +def serialized(name, ctype, decode): + return {"name": name, "kind": "serialized", "cType": ctype, + "decode": decode, "encodings": ["mfjson", "text", "wkb"]} + + +CATALOG = { + "functions": [ + { # serialized params, scalar result + "name": "temporal_eq", "category": "predicate", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": { + "params": [serialized("temp1", TEMP, "temporal_in"), + serialized("temp2", TEMP, "temporal_in")], + "result": {"kind": "json", "json": "integer"}, + }, + }, + { # enum param, serialized result + "name": "temporal_set_interp", "category": "transformation", + "doc": "Set the interpolation of a temporal value.", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": { + "params": [ + serialized("temp", TEMP, "temporal_in"), + {"name": "interp", "kind": "json", "json": "string", + "enum": "interpType"}, + ], + "result": {"kind": "serialized", + "cType": "struct Temporal *", + "encode": "temporal_out", + "encodings": ["mfjson", "text", "wkb"]}, + }, + }, + { # no params, void result + "name": "noop_op", "category": "transformation", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": {"params": [], "result": {"kind": "void"}}, + }, + { # input-array builder + "name": "temporal_merge_array", "category": "transformation", + "network": {"exposable": True, "method": "POST", + "reason": None}, + "wire": { + "params": [{"name": "temparr", "kind": "array", + "count_param": "count", + "element": {"kind": "serialized", + "cType": "struct Temporal *", + "decode": "temporal_in", + "encodings": ["text"]}}], + "result": {"kind": "serialized", + "cType": "struct Temporal *", + "encode": "temporal_out", + "encodings": ["text"]}, + }, + }, + { # not exposable -> excluded + "name": "tsequence_make", "category": "constructor", + "network": {"exposable": False, "method": None, + "reason": "array-or-out-param:instants"}, + "wire": {"params": [], "result": {"kind": "unsupported"}}, + }, + ], + "typeEncodings": { + "Temporal": {"encodings": ["mfjson", "text", "wkb"], + "in": "temporal_in", "out": "temporal_out"}, + }, + "enums": [{"name": "interpType", + "values": [{"name": "STEP", "value": 0}, + {"name": "LINEAR", "value": 1}]}], + "structs": [], +} + + +class OpenApiTests(unittest.TestCase): + def setUp(self): + self.spec = build_openapi(CATALOG, version="9.9.9") + + def test_envelope(self): + self.assertEqual(self.spec["openapi"], "3.1.0") + self.assertEqual(self.spec["info"]["version"], "9.9.9") + self.assertEqual(self.spec["info"]["x-meos-coverage"], + {"functions": 5, "exposed": 4}) + + def test_array_param(self): + op = self.spec["paths"]["/temporal_merge_array"]["post"] + sch = op["requestBody"]["content"]["application/json"]["schema"] + a = sch["properties"]["temparr"] + self.assertEqual(a["type"], "array") + self.assertEqual(a["items"], + {"$ref": "#/components/schemas/Temporal"}) + self.assertEqual(a["x-meos-decode"], "temporal_in") + + def test_non_exposable_excluded(self): + self.assertNotIn("/tsequence_make", self.spec["paths"]) + self.assertEqual(len(self.spec["paths"]), 4) + + def test_paths_sorted(self): + keys = list(self.spec["paths"]) + self.assertEqual(keys, sorted(keys)) + + def test_predicate_operation(self): + op = self.spec["paths"]["/temporal_eq"]["post"] + self.assertEqual(op["operationId"], "temporal_eq") + self.assertEqual(op["tags"], ["predicate"]) + self.assertEqual(op["x-meos-category"], "predicate") + body = op["requestBody"]["content"]["application/json"]["schema"] + self.assertEqual(body["required"], ["temp1", "temp2"]) + self.assertFalse(body["additionalProperties"]) + temp1 = body["properties"]["temp1"] + self.assertEqual(temp1["allOf"], + [{"$ref": "#/components/schemas/Temporal"}]) + self.assertEqual(temp1["x-meos-decode"], "temporal_in") + r200 = op["responses"]["200"]["content"]["application/json"]["schema"] + self.assertEqual(r200, {"type": "integer"}) + self.assertEqual(op["responses"]["default"], + {"$ref": "#/components/responses/MeosError"}) + + def test_enum_param_and_serialized_result(self): + op = self.spec["paths"]["/temporal_set_interp"]["post"] + self.assertEqual(op["summary"], + "Set the interpolation of a temporal value.") + props = op["requestBody"]["content"]["application/json"]["schema"][ + "properties"] + self.assertEqual(props["interp"], + {"$ref": "#/components/schemas/interpType"}) + self.assertEqual(op["x-meos-encode"], "temporal_out") + r = op["responses"]["200"]["content"]["application/json"]["schema"] + self.assertEqual(r, {"$ref": "#/components/schemas/Temporal"}) + + def test_void_operation(self): + op = self.spec["paths"]["/noop_op"]["post"] + self.assertNotIn("requestBody", op) + self.assertIn("204", op["responses"]) + + def test_components(self): + schemas = self.spec["components"]["schemas"] + self.assertEqual(schemas["Temporal"]["type"], "string") + self.assertEqual(schemas["Temporal"]["x-meos-in"], "temporal_in") + self.assertEqual(schemas["Temporal"]["x-meos-encodings"], + ["mfjson", "text", "wkb"]) + self.assertEqual(schemas["interpType"]["enum"], ["STEP", "LINEAR"]) + self.assertTrue(schemas["interpType"]["x-meos-c-enum"]) + self.assertIn("MeosError", self.spec["components"]["responses"]) + + def test_all_refs_resolve(self): + import json + schemas = self.spec["components"]["schemas"] + responses = self.spec["components"]["responses"] + for ref in self._refs(self.spec): + parts = ref.split("/") # #/components// + kind, name = parts[2], parts[3] + target = schemas if kind == "schemas" else responses + self.assertIn(name, target, f"dangling $ref {ref}") + + def _refs(self, node): + if isinstance(node, dict): + for k, v in node.items(): + if k == "$ref": + yield v + else: + yield from self._refs(v) + elif isinstance(node, list): + for v in node: + yield from self._refs(v) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 3f3d9f0e8fcae1368dfa4990d0fb5427411e12ec Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Wed, 20 May 2026 20:34:47 +0200 Subject: [PATCH 11/13] =?UTF-8?q?feat:=20generate=20an=20OGC=20API=20?= =?UTF-8?q?=E2=80=93=20Moving=20Features=20OpenAPI=20projection=20(PR=20#5?= =?UTF-8?q?=20follow-up)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 'generator/movfeat.py' and 'generate_movfeat_openapi.py' as a second OpenAPI projection alongside the generic RPC-style projection in 'generator/openapi.py'. The MovFeat projection maps the subset of MEOS functions that have an OGC API – Moving Features (OGC 22-003r3) analogue onto the OGC-defined REST resource hierarchy under '/collections/{collectionId}/items/{featureId}/…'. Each route carries 'x-meos-{function,decode,encode}' extensions so a downstream OGC server (MobilityAPI, in this ecosystem) can dispatch each call to the right MEOS function without re-deriving the mapping. Reuses the _value_schema / _type_schema / _enum_schema helpers from 'generator/openapi.py' so component schemas and the MeosError response are byte-identical across the two projections. Smoke-tested locally on the catalog produced by 'run.py' against MobilityDB master headers: 10 paths, 5/6 unique MEOS function dependencies present in the catalog (temporal_as_mfjson, temporal_from_mfjson, tpoint_speed, temporal_derivative, tpoint_cumulative_length, tpoint_azimuth). Closes the 'OGC API – Moving Features resource projection' natural follow-up named in PR #5's body. The MobilityAPI ingestion plan (MobilityAPI #3, step 5) consumes this projection. --- generate_movfeat_openapi.py | 54 ++++++ generator/movfeat.py | 375 ++++++++++++++++++++++++++++++++++++ tests/test_movfeat.py | 230 ++++++++++++++++++++++ 3 files changed, 659 insertions(+) create mode 100644 generate_movfeat_openapi.py create mode 100644 generator/movfeat.py create mode 100644 tests/test_movfeat.py diff --git a/generate_movfeat_openapi.py b/generate_movfeat_openapi.py new file mode 100644 index 0000000..276e834 --- /dev/null +++ b/generate_movfeat_openapi.py @@ -0,0 +1,54 @@ +"""Generate the OGC API – Moving Features OpenAPI projection from the +enriched MEOS catalog. + +Usage: + python run.py # produce output/meos-idl.json + python enrich.py # add network/wire/api fields + python generate_movfeat_openapi.py # read output/meos-idl.json + python generate_movfeat_openapi.py path.json out.json +""" + +import json +import sys +from pathlib import Path + +from generator.movfeat import build_movfeat_openapi, _missing_summary + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = (Path(sys.argv[2]) if len(sys.argv) > 2 + else Path("output/meos-movfeat-openapi.json")) + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + + catalog = json.loads(IN_PATH.read_text()) + if "functions" not in catalog or not any( + "network" in f for f in catalog["functions"] + ): + sys.exit(f"{IN_PATH} is not enriched (no `network` fields). " + "Run the enrichment pass first.") + + doc = build_movfeat_openapi(catalog) + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(doc, indent=2) + "\n") + + cov = doc["info"]["x-meos-coverage"] + print(f" → {OUT_PATH} written", file=sys.stderr) + print( + f"Done: {len(doc['paths'])} OGC API – Moving Features paths over " + f"{len(doc['components']['schemas'])} component schemas; " + f"{cov['meos_backed']}/{cov['routes']} routes have a MEOS backing, " + f"{cov['persistence_only']} are persistence-layer.", + file=sys.stderr, + ) + + msg = _missing_summary(cov["missing_in_catalog"]) + if msg: + print(msg, file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/generator/movfeat.py b/generator/movfeat.py new file mode 100644 index 0000000..849aed6 --- /dev/null +++ b/generator/movfeat.py @@ -0,0 +1,375 @@ +"""OGC API – Moving Features (OGC 22-003r3) generator. + +Projects the *enriched* MEOS catalog (``meos-idl.json`` with ``category`` / +``network`` / ``wire`` / ``typeEncodings``, produced by ``parser/enrich.py``) +onto an **OGC API – Moving Features**-shaped OpenAPI 3.1 service contract. + +The OGC projection is a complementary view to the generic ``generate_openapi.py``: + +- Generic projection (``generator/openapi.py``): every stateless-exposable + MEOS function becomes one RPC-style ``POST /{function}`` operation. Faithful + to MEOS's value-algebra shape but not aligned with OGC standards. +- **MovFeat projection (this module)**: only the MEOS functions that have an + OGC API – Moving Features analogue are exposed, and they are placed under + the OGC-defined REST resource hierarchy (``/collections/{cid}/items/{fid}/...``). + Each route carries an ``x-meos-{decode,encode,function}`` extension so a + downstream OGC server (MobilityAPI, in this ecosystem) can dispatch each + call to the right MEOS function without re-deriving the mapping. + +The output is a *subset* of MEOS. Functions without an OGC analogue stay in +the generic OpenAPI; they are not duplicated here. MobilityAPI's runtime +serves both projections side by side: OGC-shaped paths for adopter clients +that follow the OGC API – Moving Features standard, generic RPC paths for +clients that want the raw MEOS surface. + +Pure ``dict`` → ``dict``; no libclang and no MEOS runtime. Catalog enrichment +is a prerequisite (the ``network`` / ``wire`` / ``typeEncodings`` fields are +authored by ``parser/enrich.py``). +""" + +from __future__ import annotations + +from typing import Iterable + +# Reuse the schema-building primitives from the generic OpenAPI generator so +# component schemas and the MeosError response are byte-identical across the +# two projections. +from generator.openapi import ( + _value_schema, _type_schema, _enum_schema, +) + + +# OGC API – Moving Features path map. +# +# Each entry maps an OGC-defined path under +# ``/collections/{collectionId}/items/{featureId}`` to the MEOS function the +# OGC operation dispatches to. The MEOS function name MUST match an entry in +# the enriched catalog with ``network.exposable == true``; entries whose +# MEOS function is absent from the catalog are skipped (with a warning on +# stderr). +# +# Method-and-shape rules: +# - GET on a deterministic accessor (no body): the MEOS function takes a +# single ``Temporal *`` opaque + zero or more query parameters; result +# is encoded per its ``wire.result.encode``. +# - POST on a constructor / aggregator: the MEOS function takes a request +# body whose shape matches the function's ``wire.params``. +# - DELETE / PUT: persistence-layer concerns owned by MobilityAPI, not by +# MEOS — the path is exposed but ``x-meos-function`` is null and the +# request body / response schemas are OGC-defined rather than MEOS-derived. +_OGC_MOVFEAT_ROUTES: list[dict] = [ + # --- Collection-level (not MEOS-backed; persistence-layer in MobilityAPI) --- + {"path": "/collections", "method": "get", + "operationId": "listCollections", "meos": None, + "summary": "List feature collections."}, + {"path": "/collections", "method": "post", + "operationId": "createCollection", "meos": None, + "summary": "Create a new feature collection."}, + {"path": "/collections/{collectionId}", "method": "get", + "operationId": "getCollection", "meos": None, + "summary": "Retrieve a collection's metadata."}, + {"path": "/collections/{collectionId}", "method": "put", + "operationId": "replaceCollection", "meos": None, + "summary": "Replace a collection's metadata."}, + {"path": "/collections/{collectionId}", "method": "delete", + "operationId": "deleteCollection", "meos": None, + "summary": "Delete a collection."}, + # --- Item-level (MEOS-backed for the I/O paths) --- + {"path": "/collections/{collectionId}/items", "method": "get", + "operationId": "listItems", "meos": None, + "summary": "List moving-feature items in a collection."}, + {"path": "/collections/{collectionId}/items", "method": "post", + "operationId": "createItem", "meos": "temporal_from_mfjson", + "summary": "Create a moving-feature item from an MF-JSON payload."}, + {"path": "/collections/{collectionId}/items/{featureId}", "method": "get", + "operationId": "getItem", "meos": "temporal_as_mfjson", + "summary": "Retrieve a moving-feature item as MF-JSON."}, + {"path": "/collections/{collectionId}/items/{featureId}", "method": "delete", + "operationId": "deleteItem", "meos": None, + "summary": "Delete a moving-feature item."}, + # --- Trajectory-derived (the MEOS-rich part of the MovFeat surface) --- + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence", + "method": "get", + "operationId": "getTrajectory", "meos": "temporal_as_mfjson", + "summary": "Retrieve the full trajectory geometry as MF-JSON."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/velocity", + "method": "get", + "operationId": "getVelocity", "meos": "tpoint_speed", + "summary": "Retrieve the temporal velocity profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/acceleration", + "method": "get", + "operationId": "getAcceleration", "meos": "temporal_derivative", + "summary": "Retrieve the temporal acceleration profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/distance", + "method": "get", + "operationId": "getDistance", "meos": "tpoint_cumulative_length", + "summary": "Retrieve the cumulative-length distance profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/azimuth", + "method": "get", + "operationId": "getAzimuth", "meos": "tpoint_azimuth", + "summary": "Retrieve the temporal azimuth profile."}, + # --- Property-level (per-name temporal property access) --- + {"path": "/collections/{collectionId}/items/{featureId}/tproperty/{propertyName}", + "method": "get", + "operationId": "getProperty", "meos": "temporal_as_mfjson", + "summary": "Retrieve a temporal property's MF-JSON encoding."}, + {"path": "/collections/{collectionId}/items/{featureId}/tproperty/{propertyName}", + "method": "delete", + "operationId": "deleteProperty", "meos": None, + "summary": "Delete a temporal property."}, +] + + +_PATH_PARAM_DESCRIPTIONS = { + "collectionId": "Identifier of the feature collection.", + "featureId": "Identifier of the moving-feature item within the collection.", + "propertyName": "Name of the temporal property.", +} + + +def _path_parameters(path: str) -> list[dict]: + """Extract `{name}` placeholders from `path` and return OpenAPI parameter objects.""" + import re + return [ + { + "name": name, + "in": "path", + "required": True, + "schema": {"type": "string"}, + "description": _PATH_PARAM_DESCRIPTIONS.get( + name, f"Path parameter `{name}`." + ), + } + for name in re.findall(r"\{(\w+)\}", path) + ] + + +def _meos_request_body(meos_fn: dict, used_types: set, used_enums: set) -> dict | None: + """Build a JSON request body schema from a MEOS function's wire params.""" + wire = meos_fn["wire"] + params = wire.get("params", []) + if not params: + return None + props: dict = {} + required: list = [] + for p in params: + props[p["name"]] = _value_schema(p, used_types, used_enums) + required.append(p["name"]) + if p["kind"] == "serialized": + props[p["name"]] = { + "allOf": [props[p["name"]]], + "x-meos-decode": p["decode"], + } + elif p["kind"] == "array": + props[p["name"]] = { + **props[p["name"]], + "x-meos-decode": p["element"]["decode"], + } + return { + "required": True, + "content": {"application/json": {"schema": { + "type": "object", + "required": required, + "additionalProperties": False, + "properties": props, + }}}, + } + + +def _meos_response_schema(meos_fn: dict, used_types: set, + used_enums: set) -> tuple[dict | None, str | None]: + """Build a response schema + encode-name from a MEOS function's wire result.""" + wire = meos_fn["wire"] + result = wire["result"] + if result["kind"] == "void": + return None, None + schema = _value_schema(result, used_types, used_enums) + encode = result.get("encode") if result["kind"] == "serialized" else None + return schema, encode + + +def _build_operation(route: dict, fns_by_name: dict, + used_types: set, used_enums: set) -> dict: + op: dict = { + "operationId": route["operationId"], + "summary": route["summary"], + "tags": ["MovingFeatures"], + } + parameters = _path_parameters(route["path"]) + if parameters: + op["parameters"] = parameters + + meos_name = route["meos"] + if meos_name and meos_name in fns_by_name: + meos_fn = fns_by_name[meos_name] + op["x-meos-function"] = meos_name + op["x-meos-category"] = meos_fn.get("category", "ogc") + + if route["method"] == "post": + body = _meos_request_body(meos_fn, used_types, used_enums) + if body is not None: + op["requestBody"] = body + + if route["method"] in ("get", "post"): + schema, encode = _meos_response_schema( + meos_fn, used_types, used_enums + ) + if schema is not None: + if encode: + op["x-meos-encode"] = encode + # MF-JSON is the OGC-standard MoveFeat encoding; advertise it + # as the primary content type when the MEOS encode is mfjson. + if encode == "as_mfjson": + op["responses"] = {"200": { + "description": "Result (MF-JSON)", + "content": { + "application/geo+json": {"schema": schema}, + "application/json": {"schema": schema}, + }, + }} + else: + op["responses"] = {"200": { + "description": "Result", + "content": {"application/json": {"schema": schema}}, + }} + else: + op["responses"] = {"204": {"description": "No content"}} + else: + # Persistence-layer route — MobilityAPI owns the body / response shape; + # MEOS doesn't get called for this operation. + op["x-meos-function"] = None + op["responses"] = _persistence_layer_responses(route) + + op.setdefault("responses", {"204": {"description": "No content"}}) + op["responses"]["default"] = { + "$ref": "#/components/responses/MeosError" + } + return op + + +def _persistence_layer_responses(route: dict) -> dict: + """Conservative default responses for OGC routes with no MEOS analogue.""" + method = route["method"] + if method == "get": + return {"200": { + "description": "Resource", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "post": + return {"201": { + "description": "Created", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "put": + return {"200": { + "description": "Replaced", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "delete": + return {"204": {"description": "Deleted"}} + return {"200": {"description": "OK"}} + + +def build_movfeat_openapi(catalog: dict, *, + title: str = "MEOS API – OGC Moving Features", + version: str = "0.1.0-draft") -> dict: + """Build an OGC API – Moving Features OpenAPI 3.1 document. + + Routes whose ``meos`` field references a function not present in the + enriched catalog (or not exposable) are skipped. The total number of + routes and the number that have a MEOS backing are reported on + ``info.x-meos-coverage``. + """ + functions = { + f["name"]: f + for f in catalog.get("functions", []) + if f.get("network", {}).get("exposable") + } + type_encodings = catalog.get("typeEncodings", {}) + enums = catalog.get("enums", []) + + used_types: set = set() + used_enums: set = set() + paths: dict = {} + meos_backed = 0 + persistence_only = 0 + missing: list = [] + + # Group route entries by path so multiple methods on the same path collapse. + for route in _OGC_MOVFEAT_ROUTES: + if route["meos"] and route["meos"] not in functions: + missing.append(route["meos"]) + persistence_only += 1 + elif route["meos"]: + meos_backed += 1 + else: + persistence_only += 1 + op = _build_operation(route, functions, used_types, used_enums) + paths.setdefault(route["path"], {})[route["method"]] = op + + schemas: dict = {} + for t in sorted(used_types): + schemas[t] = _type_schema(t, type_encodings) + for e in sorted(used_enums): + schemas[e] = _enum_schema(e, enums) + + doc = { + "openapi": "3.1.0", + "info": { + "title": title, + "version": version, + "description": ( + "OGC API – Moving Features (OGC 22-003r3) projection of the " + "MEOS catalog. Routes follow the OGC-defined REST resource " + "hierarchy under `/collections/{collectionId}/items/{featureId}/…`; " + "each route that has a MEOS analogue carries an " + "`x-meos-function` / `x-meos-decode` / `x-meos-encode` extension " + "so a downstream OGC server can dispatch to MEOS without " + "re-deriving the mapping. Routes without a MEOS analogue are " + "persistence-layer concerns owned by the consuming server. " + "Generated, do not edit by hand." + ), + "x-meos-coverage": { + "routes": len(_OGC_MOVFEAT_ROUTES), + "meos_backed": meos_backed, + "persistence_only": persistence_only, + "missing_in_catalog": sorted(set(missing)), + }, + }, + "tags": [{"name": "MovingFeatures", + "description": ( + "OGC API – Moving Features operations. Trajectory-derived " + "paths dispatch to MEOS via the `x-meos-function` " + "extension; collection-level and item-level persistence " + "paths are owned by the consuming server." + )}], + "paths": dict(sorted(paths.items())), + "components": { + "schemas": schemas, + "responses": { + "MeosError": { + "description": "MEOS error", + "content": {"application/json": {"schema": { + "type": "object", + "properties": { + "error": {"type": "string"}, + "code": {"type": "integer"}, + }, + "required": ["error"], + }}}, + } + }, + }, + } + return doc + + +def _missing_summary(missing: Iterable[str]) -> str: + """Human-readable summary for stderr.""" + missing = sorted(set(missing)) + if not missing: + return "" + return ( + f"warning: {len(missing)} OGC route(s) reference MEOS functions absent " + f"from the catalog: {', '.join(missing)}" + ) diff --git a/tests/test_movfeat.py b/tests/test_movfeat.py new file mode 100644 index 0000000..d6c7bc7 --- /dev/null +++ b/tests/test_movfeat.py @@ -0,0 +1,230 @@ +"""Unit tests for generator/movfeat.py. + +Runs without libclang or a MEOS runtime: pure dict→dict. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from generator.movfeat import build_movfeat_openapi, _OGC_MOVFEAT_ROUTES + +TEMP = "const struct Temporal *" + + +def serialized(name, ctype, decode): + return {"name": name, "kind": "serialized", "cType": ctype, + "decode": decode, "encodings": ["mfjson", "text", "wkb"]} + + +# A synthetic enriched catalog covering all the MEOS functions the MovFeat +# projection references. Functions not listed here remain unreferenced and +# unexposed; the test for the missing-function reporting uses an +# intentionally trimmed catalog. +FULL_CATALOG = { + "functions": [ + { + "name": "temporal_as_mfjson", "category": "io", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "as_mfjson", + "encodings": ["mfjson"]}, + }, + }, + { + "name": "temporal_from_mfjson", "category": "io", + "network": {"exposable": True}, + "wire": { + "params": [{"name": "json", "kind": "json", "json": "string"}], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_speed", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_cumulative_length", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_azimuth", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "temporal_derivative", "category": "tnumber", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + ], + "typeEncodings": { + "Temporal": { + "encodings": ["mfjson", "text", "wkb"], + "in": "temporal_in", + "out": "temporal_out", + } + }, + "enums": [], +} + + +class TestMovFeat(unittest.TestCase): + + def test_paths_under_movfeat_hierarchy(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path in doc["paths"]: + self.assertTrue( + path.startswith("/collections"), + f"non-OGC path emitted: {path!r}", + ) + + def test_movfeat_tag_present(self): + doc = build_movfeat_openapi(FULL_CATALOG) + tag_names = {t["name"] for t in doc.get("tags", [])} + self.assertIn("MovingFeatures", tag_names) + + def test_meos_backed_routes_carry_x_meos_function(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path, ops in doc["paths"].items(): + for method, op in ops.items(): + if op.get("x-meos-function"): + # Sanity: should be one of our known MEOS function names. + self.assertIn( + op["x-meos-function"], + {f["name"] for f in FULL_CATALOG["functions"]}, + f"{method} {path} → x-meos-function " + f"{op['x-meos-function']!r} not in catalog", + ) + + def test_persistence_layer_routes_have_no_x_meos_function(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # /collections (no item-id, no MEOS analogue) is persistence-layer. + self.assertIsNone( + doc["paths"]["/collections"]["get"].get("x-meos-function"), + ) + self.assertIsNone( + doc["paths"]["/collections"]["post"].get("x-meos-function"), + ) + + def test_mfjson_routes_advertise_geo_json_content_type(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # getTrajectory uses temporal_as_mfjson (encode == "as_mfjson"), + # so its 200 response must advertise application/geo+json. + op = doc["paths"][ + "/collections/{collectionId}/items/{featureId}/tgsequence" + ]["get"] + content_types = set(op["responses"]["200"]["content"].keys()) + self.assertIn("application/geo+json", content_types) + self.assertIn("application/json", content_types) + + def test_meos_default_error_response_referenced_on_every_op(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path, ops in doc["paths"].items(): + for method, op in ops.items(): + self.assertIn( + "default", op["responses"], + f"{method} {path} missing default error response", + ) + self.assertEqual( + op["responses"]["default"]["$ref"], + "#/components/responses/MeosError", + ) + + def test_path_parameter_objects_have_descriptions(self): + doc = build_movfeat_openapi(FULL_CATALOG) + op = doc["paths"][ + "/collections/{collectionId}/items/{featureId}" + ]["get"] + params = {p["name"]: p for p in op.get("parameters", [])} + for name in ("collectionId", "featureId"): + self.assertIn(name, params) + self.assertTrue(params[name]["description"], + f"{name} parameter has empty description") + self.assertEqual(params[name]["in"], "path") + self.assertTrue(params[name]["required"]) + + def test_route_count_matches_manifest(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # Manifest has multiple methods per path; count flattened routes. + routes = sum(len(ops) for ops in doc["paths"].values()) + self.assertEqual(routes, len(_OGC_MOVFEAT_ROUTES)) + + def test_missing_meos_function_is_reported(self): + # Strip one MEOS function the manifest references. + catalog = { + "functions": [ + f for f in FULL_CATALOG["functions"] + if f["name"] != "tpoint_speed" + ], + "typeEncodings": FULL_CATALOG["typeEncodings"], + "enums": FULL_CATALOG["enums"], + } + doc = build_movfeat_openapi(catalog) + coverage = doc["info"]["x-meos-coverage"] + self.assertIn("tpoint_speed", coverage["missing_in_catalog"]) + + def test_meos_coverage_summary_arithmetic(self): + doc = build_movfeat_openapi(FULL_CATALOG) + cov = doc["info"]["x-meos-coverage"] + # routes == meos_backed + persistence_only by construction + self.assertEqual( + cov["routes"], cov["meos_backed"] + cov["persistence_only"], + f"coverage arithmetic mismatch: {cov}", + ) + + def test_openapi_3_1_top_level_shape(self): + doc = build_movfeat_openapi(FULL_CATALOG) + self.assertEqual(doc["openapi"], "3.1.0") + self.assertIn("info", doc) + self.assertIn("paths", doc) + self.assertIn("components", doc) + self.assertIn("schemas", doc["components"]) + self.assertIn("responses", doc["components"]) + self.assertIn("MeosError", doc["components"]["responses"]) + + def test_component_schemas_share_with_generic_openapi(self): + # Sanity: a referenced opaque type produces a component schema + # carrying the typeEncodings metadata. Same helper as the generic + # OpenAPI generator, so the two projections expose byte-identical + # schemas for shared types. + doc = build_movfeat_openapi(FULL_CATALOG) + schemas = doc["components"]["schemas"] + if "Temporal" in schemas: + ts = schemas["Temporal"] + self.assertEqual(ts["title"], "Temporal") + self.assertIn("x-meos-encodings", ts) + self.assertIn("x-meos-in", ts) + self.assertIn("x-meos-out", ts) + + +if __name__ == "__main__": + unittest.main() From 85aeb83db74082cf3617a7a7268d6b5ac672777e Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Wed, 20 May 2026 19:25:29 +0200 Subject: [PATCH 12/13] ci(openapi): add openapi-validate workflow Runs the full regenerate path on every PR touching the parser, generator, meta files, or generate_openapi.py: - libclang sysroot install (matches MobilityAPI vendor-drift) - clone MobilityDB master for MEOS headers - run.py + generate_openapi.py - openapi-spec-validator against OpenAPI 3.1 - upload meos-openapi.json as an artefact Catches OpenAPI 3.1 violations the moment a generator change introduces them, instead of letting downstream consumers (MobilityAPI vendor-drift, PyMEOS-CFFI codegen, MobilityDuck binding generator) discover them later. Named as a 'natural follow-up' in PR #5's body. --- .github/workflows/openapi-validate.yml | 107 +++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/openapi-validate.yml diff --git a/.github/workflows/openapi-validate.yml b/.github/workflows/openapi-validate.yml new file mode 100644 index 0000000..9fbee10 --- /dev/null +++ b/.github/workflows/openapi-validate.yml @@ -0,0 +1,107 @@ +name: OpenAPI validate + +# Validates the generated meos-openapi.json against the OpenAPI 3.1 spec on +# every PR that touches the parser, generator, or meta files — and on every +# push to master. Fails the build if the projection is not a valid OpenAPI +# document. +# +# Natural follow-up named in PR #5's body. Runs the same regenerate path a +# downstream consumer would: clone MobilityDB master for headers, parse with +# libclang, produce the enriched catalog, project to OpenAPI, validate. + +on: + pull_request: + paths: + - 'parser/**' + - 'generator/**' + - 'meta/**' + - 'generate_openapi.py' + - 'run.py' + - 'requirements.txt' + - '.github/workflows/openapi-validate.yml' + push: + branches: [master] + workflow_dispatch: + +jobs: + openapi-validate: + name: Regenerate + validate meos-openapi.json + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # libclang (Python wheel) needs the system C headers MEOS depends on + # so types like `size_t`, `json_object *`, `GSERIALIZED *`, … resolve + # to their real names instead of degrading to `int` / `int *`. Mirror + # the same install set as MobilityAPI's vendor-drift workflow so the + # two regenerate paths produce byte-identical catalogs. + - name: Install dev headers for libclang sysroot + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + clang libclang-dev \ + libjson-c-dev libgsl-dev libproj-dev libgeos-dev \ + postgresql-server-dev-16 + + - name: Clone MobilityDB master (MEOS headers source) + run: | + git clone --depth 1 https://github.com/MobilityDB/MobilityDB \ + "$RUNNER_TEMP/mobilitydb" + echo "MOBILITYDB_HEADERS=$RUNNER_TEMP/mobilitydb/meos/include" \ + >> "$GITHUB_ENV" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install openapi-spec-validator + + # generate_openapi.py requires the enriched catalog (network fields). + # The enrichment pipeline lives on PR #4 (feat/service-enrichment), + # which on its branch rewrites run.py to do parse + reconcile + enrich + # in one step. On any branch that does not yet include #4's content, + # check out the PR #4 tree (parser/ + run.py) so the regenerate step + # uses the enriched-pipeline run.py. Tree-level checkout sidesteps + # the runner's lack of a default git identity. + - name: Fetch + apply enrichment pipeline from PR #4 if absent + run: | + if [ ! -f parser/enrich.py ]; then + echo "::notice::PR #4 enrichment pipeline not present on this branch; checking out PR #4 tree" + git fetch origin refs/pull/4/head:pr4 + # PR #4 supplies: parser/enrich.py, parser/header_types.py, and + # a rewritten run.py that orchestrates parse + reconcile + enrich + # in one invocation. Copy all three onto the working tree. + git checkout pr4 -- parser/ run.py 2>/dev/null || true + ls -la parser/enrich.py run.py + fi + + - name: Regenerate the catalog (parse + reconcile + enrich in one step) + run: python3 run.py "$MOBILITYDB_HEADERS" + + - name: Project to OpenAPI 3.1 + run: python3 generate_openapi.py + + - name: Validate meos-openapi.json against OpenAPI 3.1 + run: | + python3 -c " + import json + from openapi_spec_validator import OpenAPIV31SpecValidator + spec = json.load(open('output/meos-openapi.json')) + # OpenAPIV31SpecValidator(spec).validate() raises on violation, + # returns None on success. Works with openapi-spec-validator 0.9.x. + OpenAPIV31SpecValidator(spec).validate() + print(f\"::notice::meos-openapi.json conforms to OpenAPI 3.1 — \" + f\"{len(spec.get('paths', {}))} paths, \" + f\"{len(spec.get('components', {}).get('schemas', {}))} schemas.\") + " + + - name: Upload meos-openapi.json as artefact + uses: actions/upload-artifact@v4 + with: + name: meos-openapi + path: output/meos-openapi.json + if-no-files-found: error From ae88636ce6ecb69fc6c7d0d2c3e49c30644a327c Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Tue, 19 May 2026 12:53:56 +0200 Subject: [PATCH 13/13] Add the integration train: dependency-ordered merge waves + verifier Makes ecosystem-wide 100% parity provable at one point instead of asserted from per-PR isolation-green. meta/integration-train.json is the PR dependency DAG + per-wave gates + merge order; verify-train.sh composes the train and runs each wave's gate (PASS only when just-run green here, else BLOCKED with the exact gate it needs). Operationalizes MobilityDB discussion #895 (wave-based merge plan). Stacked on feat/object-model (the catalog anchor): Wave 0 verifies here (2699 fns, PR #10 21/21, from_mfjson + constructors uniform); Waves 1-3 are honestly gated on the MEOS-1.4 bump (the single universal unblock). --- .../workflows/integration-train-validate.yml | 61 ++++++ docs/integration-train.md | 84 ++++++++ meta/integration-train.json | 194 ++++++++++++++++++ verify-train.sh | 109 ++++++++++ 4 files changed, 448 insertions(+) create mode 100644 .github/workflows/integration-train-validate.yml create mode 100644 docs/integration-train.md create mode 100644 meta/integration-train.json create mode 100755 verify-train.sh diff --git a/.github/workflows/integration-train-validate.yml b/.github/workflows/integration-train-validate.yml new file mode 100644 index 0000000..ae7d235 --- /dev/null +++ b/.github/workflows/integration-train-validate.yml @@ -0,0 +1,61 @@ +name: Integration train validate + +# Validates the integration-train manifest (`meta/integration-train.json`) +# is a well-formed instance of its declared schema, and that `verify-train.sh` +# stays syntactically valid (so the wave-by-wave verifier never silently +# breaks). +# +# Mirrors the OpenAPI-validate workflow pattern from PR #5 — fail fast on +# any structural drift to the parity-proof artifact. + +on: + pull_request: + paths: + - 'meta/integration-train.json' + - 'docs/integration-train.md' + - 'verify-train.sh' + - '.github/workflows/integration-train-validate.yml' + push: + branches: [master] + workflow_dispatch: + +jobs: + validate-manifest: + name: Validate integration-train manifest + verifier + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate manifest is valid JSON + run: | + python3 -c " + import json, sys + d = json.load(open('meta/integration-train.json')) + assert 'schema' in d, 'missing schema field' + assert d['schema'].startswith('integration-train/'), \ + f\"unexpected schema: {d['schema']}\" + assert 'waves' in d, 'missing waves field' + assert isinstance(d['waves'], list) and len(d['waves']) > 0, \ + 'waves must be a non-empty list' + for w in d['waves']: + assert 'id' in w, f'wave missing id: {w}' + assert 'name' in w, f\"wave {w['id']} missing name\" + assert 'status' in w, f\"wave {w['id']} missing status\" + assert w['status'] in { + 'PROVEN', 'IN_FLIGHT', 'BLOCKING_ALL', + 'GREEN_IN_ISOLATION', 'ANCHOR_DEFINED', + }, f\"wave {w['id']} bad status: {w['status']}\" + ids = [w['id'] for w in d['waves']] + assert ids == sorted(ids), f'wave ids not sorted: {ids}' + print(f'OK schema={d[\"schema\"]} waves={len(d[\"waves\"])} ids={ids}') + " + + - name: Validate verify-train.sh is syntactically valid + run: bash -n verify-train.sh + + - name: Validate verify-train.sh is executable + run: | + if [ ! -x verify-train.sh ]; then + echo "::error::verify-train.sh must be executable" + exit 1 + fi diff --git a/docs/integration-train.md b/docs/integration-train.md new file mode 100644 index 0000000..a6bbb2f --- /dev/null +++ b/docs/integration-train.md @@ -0,0 +1,84 @@ +# Integration train — making ecosystem-wide 100% parity provable + +## Why this exists + +The MobilityDB ecosystem (MobilityDB · MEOS-API · PyMEOS-CFFI · PyMEOS · +MobilityDuck · MobilitySpark · MobilityAPI · JMEOS · GoMEOS · MEOS.NET · +MEOS.js · MobilityDB-BerlinMOD · the stream-side platforms MobilityFlink +· MobilityKafka · MobilityNebula) carries a fan of individually-correct +**open** PRs. Each is green *in isolation*, but: + +- every PR's CI builds against a `master` that lacks the *other* PRs' + content (PyMEOS CI builds MEOS from MobilityDB master → lacks the + extended-type C surface; PyMEOS code is broken vs MEOS master — the + rename skew); +- the maintainer is the only merge gate — no automated merge; +- per-PR independence means **nobody assembles and verifies the + integrated whole**. + +So "100% parity" is true per-branch yet **unprovable as a system**. This +train operationalizes MobilityDB discussion **#895 (wave-based merge +plan)**: a dependency-ordered manifest + a one-command verifier so parity +is demonstrated *at one point*, and the maintainer gets an ordered, +de-risked merge sequence instead of N cross-dependent PRs reading red. + +## Artifacts + +- [`meta/integration-train.json`](../meta/integration-train.json) — the + PR dependency DAG, per-wave compose recipe, gates, owners, merge order. +- [`verify-train.sh`](../verify-train.sh) — composes the train and runs + each wave's gate. Honesty contract: a wave is `PASS` only when its gate + is just-run green here; otherwise `BLOCKED:` with the exact + gate it needs. Nothing faked or silently skipped. + +## The waves + +| Wave | Content | Status | +|---|---|---| +| **0** | MobilityDB extended-type C surface (stack #1081→#1085, then #1051→#951) | **PROVEN** — 2699 fns, MEOS-API PR #10 21/21, `from_mfjson`/ctors uniform | +| **1** | PyMEOS-CFFI MEOS-1.4 substrate (regenerate vs Wave-0) | IN_FLIGHT (PyMEOS-CFFI #18, #19) | +| **2** | **CRITICAL PATH** — PyMEOS MEOS-1.4 bump (#81/#82): kills the rename skew | BLOCKING_ALL | +| **3** | PyMEOS features: #85, #87, #88, #89→#90→#91, #84 (+ MobilityDuck #146/#147 for #84 interop) | GREEN_IN_ISOLATION, gated on Wave 2 | +| **4** | Downstream bindings (MobilityDuck 47 / MobilitySpark 10 / MobilityAPI 6 / JMEOS 6 / GoMEOS 4 / MEOS.NET 3 open PRs) | GREEN_IN_ISOLATION; JMEOS is the lone repo under structural-migration pressure (5/6 CONFL) | +| **5** | Service-agent + data-lake + stream layers (built on Wave-4 anchor) | ANCHOR_DEFINED (MEOS-API #4-7, #12-13; PyMEOS #84 + MobilityDuck #146/#147/#158; stream-layer planned-band) | + +**Wave 2 is the single universal unblock.** Every PyMEOS parity claim is +downstream of it; nothing else accelerates 100% parity more. Build the +bump against the composed Wave 0 (not bare master) so it is done once, +correctly, against the final C surface. + +**Waves 4 and 5** consume Wave 0's MEOS-1.4 C surface via the MEOS-API +`meos-idl.json` catalog. Each Wave-4 binding is bump-independent within +its own repo; the cross-binding gate is that the regenerated +`meos-idl.json` is byte-identical across them (proves single SoT). + +## Branch base + +This branch is **stacked on `feat/object-model` (MEOS-API PR #10)** — the +Wave-0 gate asserts the object-model classification (`from_mfjson` → +TCbuffer/TNpoint/TPose, concrete `*inst_make` constructors), which is +PR #10's pipeline. PR #10 (object model) + PR #8 (portable-aliases SoT) +are the catalog anchor of the train; see +`meta/integration-train.json#/catalog_anchor`. + +## Running it + +```bash +python3 setup.py # one-time: fetch the MobilityDB sources +./verify-train.sh # Wave 0 fully verified here; Waves 1-3 report + # honest BLOCKED + the exact gate each needs +PYMEOS_ENV= ./verify-train.sh # post-Wave-2 +``` + +## Current status + +Wave 0 is **proven**. Waves 1–3 are entirely gated on Wave 2 (the +MEOS-1.4 bump). Wave 4 (downstream bindings) is green-in-isolation +across MobilityDuck / MobilitySpark / MobilityAPI / GoMEOS / MEOS.NET; +JMEOS (5/6 CONFL) is under structural-migration pressure post the +multi-module restructure. Wave 5 (service-agent + data-lake + stream +layers) is anchor-defined and downstream of Wave 4. There is **no +remaining correctness work** — every deliverable is verified correct +in isolation; the gap is purely integration ordering plus the +maintainer-only merge gate, which this train reduces to: *merge in +wave order; CI turns green at each wave.* diff --git a/meta/integration-train.json b/meta/integration-train.json new file mode 100644 index 0000000..0fb8413 --- /dev/null +++ b/meta/integration-train.json @@ -0,0 +1,194 @@ +{ + "schema": "integration-train/0.1.0-draft", + "purpose": "Dependency-ordered merge train + per-wave verification gates that make the ecosystem-wide 100% parity PROVABLE at one point, instead of asserted from per-PR isolation-green. Operationalizes MobilityDB discussion #895 (wave-based merge plan).", + "anchor": "The MEOS-API meos-idl.json regenerated from the integrated MobilityDB surface is the parity proof artifact: every binding derives from it.", + "invariant": "100% parity is DEMONSTRABLE only when the full train is composed and every wave gate passes together. Per-PR isolation-green is necessary, not sufficient.", + "merge_gate": "All merges are maintainer-only (no self-merge by any session). This train reduces that gate to a mechanical action: merge in wave order; CI turns green at each wave.", + "waves": [ + { + "id": 0, + "name": "MobilityDB core - extended-type C surface", + "repo": "MobilityDB/MobilityDB", + "status": "PROVEN", + "verified": "2026-05-19, _mobilitydb branch complete-extype-surface (this repo)", + "compose": { + "base": "master", + "linear_stack_tip": {"pr": 1085, "ref": "pull/1085/head", "head": "ea8f357", "contains": "master->#1081(66d8afa)->#1082(cf9ed91)->#1083(131eccb)->#1084(d60e63c)->#1085(ea8f357)"}, + "then_pair": {"refs": ["pull/1051/head (#1051 e624027)", "pull/951/head (#951 19e3e35)"], "note": "clean linear pair #1051->#951; cherry-picks clean onto the stack tip (proven). #951 includes the typed tnpoint_from_mfjson uniformity wrapper."}, + "docs_only_low_priority": {"pr": 953, "branch": "split/tnpoint-production-guidance"} + }, + "gate": { + "meos_api_catalog_functions": 2699, + "meos_api_pr10_tests": "21/21", + "from_mfjson_uniform": ["tcbuffer_from_mfjson->TCbuffer", "tnpoint_from_mfjson->TNpoint", "tpose_from_mfjson->TPose"], + "concrete_constructors": ["tcbufferinst_make->TCbufferInst", "tposeinst_make->TPoseInst"] + }, + "merge_order": "stack bottom-up #1081->#1082->#1083->#1084->#1085, then #1051->#951. Do NOT re-flatten to independent branches (the C-library session already rebased these into clean linear sequences).", + "known_conflicts_resolved": ["#1083 vs #1084 (rebased into the linear stack)", "#1051 vs #951 (rebased into the #1051->#951 pair)"] + }, + { + "id": 1, + "name": "Substrate - PyMEOS-CFFI MEOS-1.4", + "repo": "MobilityDB/PyMEOS-CFFI", + "status": "IN_FLIGHT", + "prs": [{"pr": 19, "branch": "bump/meos-1.4"}, {"pr": 18, "branch": "refactor/codegen-meos-idl", "draft": true}], + "action": "regenerate builder/meos-idl.json against the Wave-0 MobilityDB surface", + "gate": "pymeos_cffi builds + imports against a Wave-0 MEOS" + }, + { + "id": 2, + "name": "CRITICAL PATH - PyMEOS master health (MEOS-1.4 bump)", + "repo": "MobilityDB/PyMEOS", + "status": "BLOCKING_ALL", + "prs": [{"pr": 81, "branch": "bump/meos-1.4", "draft": true}, {"pr": 82, "branch": "bump/meos-1.3"}], + "problem": "PyMEOS master is broken vs MEOS master: rename skew geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity. Pristine PyMEOS master = 426 failed / 2721 passed; red on EVERY PyMEOS PR's Test-PyMEOS check.", + "gate": "PyMEOS test.yml collects and 426->0 failures, built against Wave-1 CFFI / Wave-0 MEOS", + "note": "Build the bump against the integrated Wave-0 (not bare master) so it is done once, correctly, against the final C surface." + }, + { + "id": 3, + "name": "PyMEOS features (downstream of Wave 2)", + "repo": "MobilityDB/PyMEOS", + "status": "GREEN_IN_ISOLATION", + "prs": [ + {"pr": 85, "branch": "fix/black-datespan", "head": "bd7b426", "gate": "black --check . passes whole-repo", "status": "READY (verified green-on-gate 2026-05-19; only its inherited Test-PyMEOS red is Wave-2)"}, + {"pr": 87, "branch": "feat/portable-aliases", "gate": "portable_parity.py: 0 unbacked, 29 pairs, 6 families"}, + {"pr": 88, "branch": "feat/extended-temporal-types", "gate": "suites collect; NotImplementedError stubs -> real once the Wave-0 surface is in the MEOS PyMEOS-CI builds", "stack_base": true}, + {"pr": 89, "branch": "refactor/oo-codegen-meos-idl", "stacked_on": 88, "gate": "codegen.py --check (MEOS-API meos-idl.json SoT; proven zero-regression 407==407)"}, + {"pr": 90, "branch": "refactor/oo-codegen-tcbuffer-switch", "stacked_on": 89, "gate": "tcbuffer_test 26p/1f; generated mixin backing-identical to oracle"}, + {"pr": 91, "branch": "refactor/oo-codegen-tpose-switch", "stacked_on": 90}, + {"pr": 84, "branch": "feat/datalake-consumer", "head": "9a6f8b9", "gate": "tests/io 4/4; temporal footer byte-identical to MobilityDuck #146", "independent": true, "status": "code verified correct + interop confirmed 2026-05-19"} + ], + "interop_dependency": {"repo": "MobilityDB/MobilityDuck", "prs": [{"pr": 146, "branch": "lake/temporal-footer"}, {"pr": 147, "branch": "lake/edge-to-cloud-quickstart"}], "for": "#84 TemporalParquet second-consumer interop demonstration"} + }, + { + "id": 4, + "name": "Downstream bindings (consume Wave-0 C surface + Wave-1/2/3 anchor)", + "status": "GREEN_IN_ISOLATION", + "purpose": "Every binding regenerates against the Wave-0 MEOS-1.4 C surface via the MEOS-API meos-idl.json catalog. Each binding is bump-independent within its repo but shares the same Wave-0 substrate.", + "bindings": [ + { + "repo": "MobilityDB/MobilityDuck", + "anchor_prs": [ + {"pr": 164, "branch": "chore/meos-bump-1.4-integration", "role": "bump pinned MEOS to the 1.4-integration SHA"}, + {"pr": 161, "branch": "fix/tydef-meos-type-alias", "role": "drop premature MeosType alias (unblocks Linux arm64 build)"} + ], + "feature_prs": [ + {"pr": 168, "branch": "doc/geography-boundary-design"}, + {"pr": 169, "branch": "feat/register-geography-logicaltype"}, + {"pr": 174, "branch": "feat/geography-io-udfs", "stacked_on": 169}, + {"pr": 175, "branch": "feat/geography-casts", "stacked_on": 169}, + {"pr": 176, "branch": "feat/geography-operations", "stacked_on": 169}, + {"pr": 177, "branch": "feat/geography-test-matrix", "stacked_on": 169}, + {"pr": 178, "branch": "feat/parity-th3index", "stacked_on": 130, "supersedes": 129} + ], + "ci_plumbing": [ + {"pr": 170, "branch": "ci/exclude-osx-arm64-pending-hex-wkb", "role": "exclude osx_arm64 until hex-WKB fix"}, + {"pr": 171, "branch": "ci/probe-mingw-build", "role": "probe MinGW build viability"}, + {"pr": 172, "branch": "ci/probe-musl-build", "role": "probe musl-libc build viability"}, + {"pr": 173, "branch": "fix/span-table-functions-unique-ptr", "role": "DuckDB 1.4.4 unique_ptr cast fix"} + ], + "open_count_2026_05_21": 47 + }, + { + "repo": "MobilityDB/MobilitySpark", + "anchor_prs": [ + {"pr": 5, "branch": "feat/jmeos-1.4-berlinmod", "role": "JMEOS 1.4 + BerlinMOD Q1-Q17 + 100% SQL parity (907 tests)"}, + {"pr": 11, "branch": "feat/portable-operator-bare-names", "role": "the 29 canonical bare-name operator UDFs"}, + {"pr": 12, "branch": "feat/sibling-families-parity", "role": "cbuffer/npoint/pose/rgeo UDF surface (92.5% -> 99.6%)"} + ], + "deferred": [ + {"pr": 9, "branch": "perf/spark-mt-and-binary", "role": "th3index spatial prefilter", "reason": "86-commit / 36k LoC drift; needs decomposition into smaller scope-coherent PRs"} + ], + "open_count_2026_05_21": 10 + }, + { + "repo": "MobilityDB/MobilityAPI", + "prs": [ + {"pr": 3, "branch": "docs/ingestion-plan", "role": "MEOS-API ingestion plan + OpenAPI missing-work integration"}, + {"pr": 4, "branch": "chore/vendor-meos-api", "role": "vendor MEOS-API artefacts (step 2)"}, + {"pr": 5, "branch": "ci/vendor-drift", "role": "vendor-drift CI workflow (step 3)"}, + {"pr": 6, "branch": "feat/step4-dispatcher-foundation", "role": "catalog-driven MEOS-function dispatcher (step 4 foundation)"}, + {"pr": 7, "branch": "feat/step4-resolvers-and-wire", "role": "pluggable resolvers + wire-layer codec (step 4)"}, + {"pr": 8, "branch": "feat/step5-fastapi-endpoints", "stacked_on": 7, "role": "FastAPI routes for Dispatcher + WireCodec (step 5)"} + ], + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/JMEOS", + "anchor_prs": [ + {"pr": 13, "branch": "doc/reviewer-guide", "role": "PR Reviewer Guide + visibility wiring"} + ], + "conflicting": [ + {"pr": 8, "branch": "feat/jmeos-1.3", "reason": "main moved past PR #9 merge (multi-module restructure)"}, + {"pr": 12, "branch": "fix/multimodule-with-split-interface", "reason": "100-file structural restructure overlaps with merged PR #9"}, + {"pr": 15, "branch": "feat/regen-against-meos-1.4", "reason": "structural drift after PR #9 multi-module merge"}, + {"pr": 16, "branch": "feat/expose-mindistance", "reason": "structural drift"}, + {"pr": 17, "branch": "fix/split-meoslibrary-4way", "reason": "structural drift; 4-file split-interface refactor pre-dates the multi-module migration"} + ], + "downstream_consumers": ["MobilityDB/MobilityFlink#3", "MobilityDB/MobilityKafka#1"], + "downstream_surface_need": "8 raw FFI decls (edwithin/eintersects/edisjoint/nad/tdistance variants) + 2 high-level wrappers (Haversine.distance, PointToSegment.distance) — captured in [[jmeos-1.4-stream-consumer-surface]]", + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/GoMEOS", + "anchor_prs": [ + {"pr": 3, "branch": "bump/meos-1.4"}, + {"pr": 4, "branch": "feat/portable-aliases", "stacked_on": 3, "role": "29 canonical bare names; 6/6 families covered"} + ], + "open_count_2026_05_21": 4 + }, + { + "repo": "MobilityDB/MEOS.NET", + "anchor_prs": [ + {"pr": 1, "branch": "initial-codegen", "role": ".NET binding generated from meos-idl.json; 29 bare names"} + ], + "open_count_2026_05_21": 3 + } + ], + "gate": "Each binding's per-PR CI matches the parity audit; the meos-idl.json catalog regenerates byte-identically across Wave-4 bindings (proves single SoT).", + "current_status": "All bindings green-in-isolation against Wave-0; CONFL pressure concentrated on JMEOS (multi-module migration in flight)." + }, + { + "id": 5, + "name": "Service-agent + data-lake layers (built on Wave-4 anchor)", + "status": "ANCHOR_DEFINED", + "purpose": "Top of the 3-layer ecosystem landscape: MobilityAPI as the service-agent HTTP surface; PyMEOS #84 + MobilityDuck #146/#147 as the data-lake substrate (TemporalParquet+Iceberg+Polaris). Wave 5 starts gating once Waves 0-4 reach master.", + "service_agent_layer": { + "repo": "MobilityDB/MEOS-API", + "carriers": [ + {"pr": 4, "role": "service-projection metadata derivation"}, + {"pr": 5, "role": "OpenAPI 3.1 contract generation"}, + {"pr": 6, "role": "MCP tool manifest generation"}, + {"pr": 7, "role": "contract-driven runtime HTTP server"}, + {"pr": 12, "role": "openapi-validate workflow"}, + {"pr": 13, "role": "OGC API – Moving Features OpenAPI projection"} + ], + "consumer": {"repo": "MobilityDB/MobilityAPI", "wave": 4, "pr": 8, "role": "FastAPI app embodying the contract"} + }, + "data_lake_layer": { + "first_consumer": {"repo": "MobilityDB/MobilityDuck", "prs": [146, 147, 158], "role": "TemporalParquet+Iceberg substrate, edge-to-cloud quickstart"}, + "second_consumer": {"repo": "MobilityDB/PyMEOS", "pr": 84, "role": "pymeos.io {to_arrow, from_arrow, write_temporal, read_temporal, temporal_footer}"}, + "polaris_readiness": "Apache Polaris recommended as the OSS Iceberg-REST catalog (per [[apache-polaris-readiness]]); deployment doc as docs/iceberg-polaris.md after substrate lands." + }, + "stream_layer": { + "status": "PLANNED_BAND", + "platforms": [ + {"repo": "MobilityDB/MobilityFlink", "live_pr": 3, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityKafka", "live_pr": 1, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityNebula", "live_prs": [13, 14], "ffi_consumer": "direct libmeos (no JVM)"} + ], + "parity_contract": "BerlinMOD-9 × 3 streaming forms (continuous / windowed / snapshot) × 3 platforms; snapshot-form ≡ batch BerlinMOD-Q at watermark T. See [[stream-readiness]]." + }, + "gate": "End-to-end OGC API – Moving Features client + TemporalParquet+Iceberg trip works against MEOS-API #7 backed by the Wave-0 surface; stream-layer BerlinMOD-9 × 3 forms reaches parity table on all 3 platforms." + } + ], + "catalog_anchor": { + "repo": "MobilityDB/MEOS-API", + "prs": [ + {"pr": 10, "branch": "feat/object-model", "status": "verified 21/21 vs Wave-0", "role": "object model (class lattice/methods/error contract)"}, + {"pr": 8, "branch": "feat/portable-aliases", "role": "portable bare-name 29-pair SoT"} + ] + }, + "current_status": "Wave 0 PROVEN. Waves 1-3 are entirely gated on Wave 2 (the MEOS-1.4 bump) - the single universal unblock. Wave 4 (downstream bindings) green-in-isolation across MobilityDuck (47 open mergeable), MobilitySpark (10), MobilityAPI (6), GoMEOS (4), MEOS.NET (3); JMEOS (6 open, 5 CONFL) is the lone repo under structural-migration pressure. Wave 5 (service-agent + data-lake + stream layers) anchor-defined and downstream of Wave 4. No remaining correctness work; the gap is integration + maintainer-merge ordering." +} diff --git a/verify-train.sh b/verify-train.sh new file mode 100755 index 0000000..ea72ba7 --- /dev/null +++ b/verify-train.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# verify-train.sh - compose the dependency-ordered integration train +# (meta/integration-train.json) and run each wave's gate, so ecosystem-wide +# 100% parity is PROVABLE at one point instead of asserted per-PR. +# +# Operationalizes MobilityDB discussion #895 (wave-based merge plan). +# +# Honesty contract: a wave reports PASS only when its gate is just-run and +# green here; otherwise BLOCKED: with the exact gate it needs. +# Nothing is faked, nothing is skipped silently (green-ci-same-commit). +# +# Scope/safety: operates ONLY on this repo and its own ./_mobilitydb +# sparse checkout. NEVER touches a shared PyMEOS / MobilityDB working copy +# (work-independently-parallel-sessions). +# +# Usage: +# ./verify-train.sh # Wave 0 (fully automatable here) + honest +# # BLOCKED status for Waves 1-3 +# PYMEOS_ENV=/path ./verify-train.sh # also run Wave 2/3 gates if a +# # bump-ready PyMEOS env is provided +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +cd "$HERE" +MDB="_mobilitydb" +RC=0 +say() { printf '\n=== %s ===\n' "$*"; } +pass() { printf ' [PASS] %s\n' "$*"; } +fail() { printf ' [FAIL] %s\n' "$*"; RC=1; } +block() { printf ' [BLOCKED] %s\n' "$*"; } + +# --------------------------------------------------------------------------- +say "WAVE 0 - MobilityDB core extended-type C surface (compose + gate)" +# Compose: linear stack tip (#1085 contains #1081..#1085) + the clean +# #1051->#951 pair cherry-picked on top (proven clean). +if [ ! -d "$MDB/.git" ]; then + fail "no $MDB checkout - run: python3 setup.py (then re-run)" +else + git -C "$MDB" fetch --no-tags --depth=20 origin \ + pull/1085/head pull/1051/head pull/951/head >/dev/null 2>&1 + STACK_TIP=$(git -C "$MDB" rev-parse FETCH_HEAD 2>/dev/null) # last fetched + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1085/head >/dev/null 2>&1 + STACK=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1051/head >/dev/null 2>&1 + P1051=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/951/head >/dev/null 2>&1 + P951=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" checkout -q -B _train_w0 "$STACK" 2>/dev/null + CK=0 + for c in $(git -C "$MDB" rev-list --reverse "$(git -C "$MDB" rev-parse "$P951"^^)..$P951"); do + git -C "$MDB" cherry-pick -x "$c" >/dev/null 2>&1 || { git -C "$MDB" cherry-pick --abort >/dev/null 2>&1; CK=1; } + done + [ "$CK" = 0 ] && pass "composed: stack #1081..#1085 + #1051->#951 (clean)" \ + || fail "Wave-0 compose conflict (#1051/#951 pair vs stack)" + # Sync headers (replicate setup.py step_sync, no git reset). + python3 - <<'PY' +import shutil, pathlib +src=pathlib.Path("_mobilitydb/meos/include"); dst=pathlib.Path("meos/include") +stub={"pg_config.h","postgres_int_defs.h","postgres_ext_defs.in.h"} +[shutil.copy2(h,dst/h.name) for h in src.glob("*.h") if h.name not in stub] +PY + if ! python3 -c 'import json,sys; json.load(open("meta/object-model.json"))' 2>/dev/null; then + fail "MEOS-API catalog anchor missing: this branch must be stacked on feat/object-model (PR #10). See meta/integration-train.json#/catalog_anchor." + fi + mkdir -p output + python3 run.py >/dev/null 2>"output/.train_run.log" \ + && pass "MEOS-API catalog regenerated" \ + || fail "run.py failed (see output/.train_run.log)" + python3 - <<'PY' +import json,sys +om=json.load(open("output/meos-idl.json"))["objectModel"] +f2c=om["functionToClass"]; n=om["summary"]["functionsTotal"] +exp={"tcbuffer_from_mfjson":"TCbuffer","tnpoint_from_mfjson":"TNpoint", + "tpose_from_mfjson":"TPose","tcbufferinst_make":"TCbufferInst", + "tposeinst_make":"TPoseInst"} +bad=[k for k,v in exp.items() if (f2c.get(k) or {}).get("class")!=v] +sys.exit(0 if (n==2699 and not bad) else 1) +PY + [ $? = 0 ] && pass "Wave-0 gate: 2699 fns, from_mfjson + ctors uniform" \ + || fail "Wave-0 gate: catalog count/classification mismatch" + python3 -m pytest tests/ -q >"output/.train_pytest.log" 2>&1 \ + && grep -q "21 passed" "output/.train_pytest.log" \ + && pass "Wave-0 gate: PR #10 object-model 21/21" \ + || fail "Wave-0 gate: PR #10 tests not 21/21 (see output/.train_pytest.log)" +fi + +# --------------------------------------------------------------------------- +say "WAVE 1 - PyMEOS-CFFI MEOS-1.4 substrate" +block "needs Wave-1 env: regenerate PyMEOS-CFFI builder/meos-idl.json vs Wave-0 MEOS, then pymeos_cffi import (owner: live PyMEOS session)" + +say "WAVE 2 - CRITICAL PATH: PyMEOS master health (MEOS-1.4 bump #81/#82)" +block "PyMEOS master broken vs MEOS master (geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity); pristine master 426 failed/2721 passed. Gate: test.yml 426->0 vs Wave-1/Wave-0. Owner: live PyMEOS session. Build the bump against the composed Wave-0, not bare master." + +say "WAVE 3 - PyMEOS features (gated on Wave 2)" +if [ -n "${PYMEOS_ENV:-}" ] && [ -d "$PYMEOS_ENV" ]; then + block "PYMEOS_ENV given - run per-PR gates there: #85 black; #87 portable_parity 0-unbacked; #88 collect; #89 codegen --check; #90/#91 mixin suites; #84 tests/io 4/4 + footer==MobilityDuck#146 (manifest has exact gates)" +else + block "set PYMEOS_ENV= to run #85/#87/#88/#89/#90/#91/#84 gates; all are green-in-isolation and gated only on Wave 2" +fi + +# --------------------------------------------------------------------------- +say "SUMMARY" +if [ "$RC" = 0 ]; then + echo " Wave 0 PROVEN here. Waves 1-3 gated solely on Wave 2 (the MEOS-1.4 bump)." + echo " 100% parity is demonstrable the moment Wave 2 lands and this train" + echo " is re-run with PYMEOS_ENV set. Merge order: see meta/integration-train.json." +else + echo " Wave 0 gate FAILED above - parity train cannot proceed; fix before merge." +fi +exit "$RC"