diff --git a/.github/workflows/integration-train-validate.yml b/.github/workflows/integration-train-validate.yml new file mode 100644 index 0000000..ae7d235 --- /dev/null +++ b/.github/workflows/integration-train-validate.yml @@ -0,0 +1,61 @@ +name: Integration train validate + +# Validates the integration-train manifest (`meta/integration-train.json`) +# is a well-formed instance of its declared schema, and that `verify-train.sh` +# stays syntactically valid (so the wave-by-wave verifier never silently +# breaks). +# +# Mirrors the OpenAPI-validate workflow pattern from PR #5 — fail fast on +# any structural drift to the parity-proof artifact. + +on: + pull_request: + paths: + - 'meta/integration-train.json' + - 'docs/integration-train.md' + - 'verify-train.sh' + - '.github/workflows/integration-train-validate.yml' + push: + branches: [master] + workflow_dispatch: + +jobs: + validate-manifest: + name: Validate integration-train manifest + verifier + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate manifest is valid JSON + run: | + python3 -c " + import json, sys + d = json.load(open('meta/integration-train.json')) + assert 'schema' in d, 'missing schema field' + assert d['schema'].startswith('integration-train/'), \ + f\"unexpected schema: {d['schema']}\" + assert 'waves' in d, 'missing waves field' + assert isinstance(d['waves'], list) and len(d['waves']) > 0, \ + 'waves must be a non-empty list' + for w in d['waves']: + assert 'id' in w, f'wave missing id: {w}' + assert 'name' in w, f\"wave {w['id']} missing name\" + assert 'status' in w, f\"wave {w['id']} missing status\" + assert w['status'] in { + 'PROVEN', 'IN_FLIGHT', 'BLOCKING_ALL', + 'GREEN_IN_ISOLATION', 'ANCHOR_DEFINED', + }, f\"wave {w['id']} bad status: {w['status']}\" + ids = [w['id'] for w in d['waves']] + assert ids == sorted(ids), f'wave ids not sorted: {ids}' + print(f'OK schema={d[\"schema\"]} waves={len(d[\"waves\"])} ids={ids}') + " + + - name: Validate verify-train.sh is syntactically valid + run: bash -n verify-train.sh + + - name: Validate verify-train.sh is executable + run: | + if [ ! -x verify-train.sh ]; then + echo "::error::verify-train.sh must be executable" + exit 1 + fi diff --git a/.github/workflows/openapi-validate.yml b/.github/workflows/openapi-validate.yml new file mode 100644 index 0000000..9fbee10 --- /dev/null +++ b/.github/workflows/openapi-validate.yml @@ -0,0 +1,107 @@ +name: OpenAPI validate + +# Validates the generated meos-openapi.json against the OpenAPI 3.1 spec on +# every PR that touches the parser, generator, or meta files — and on every +# push to master. Fails the build if the projection is not a valid OpenAPI +# document. +# +# Natural follow-up named in PR #5's body. Runs the same regenerate path a +# downstream consumer would: clone MobilityDB master for headers, parse with +# libclang, produce the enriched catalog, project to OpenAPI, validate. + +on: + pull_request: + paths: + - 'parser/**' + - 'generator/**' + - 'meta/**' + - 'generate_openapi.py' + - 'run.py' + - 'requirements.txt' + - '.github/workflows/openapi-validate.yml' + push: + branches: [master] + workflow_dispatch: + +jobs: + openapi-validate: + name: Regenerate + validate meos-openapi.json + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # libclang (Python wheel) needs the system C headers MEOS depends on + # so types like `size_t`, `json_object *`, `GSERIALIZED *`, … resolve + # to their real names instead of degrading to `int` / `int *`. Mirror + # the same install set as MobilityAPI's vendor-drift workflow so the + # two regenerate paths produce byte-identical catalogs. + - name: Install dev headers for libclang sysroot + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + clang libclang-dev \ + libjson-c-dev libgsl-dev libproj-dev libgeos-dev \ + postgresql-server-dev-16 + + - name: Clone MobilityDB master (MEOS headers source) + run: | + git clone --depth 1 https://github.com/MobilityDB/MobilityDB \ + "$RUNNER_TEMP/mobilitydb" + echo "MOBILITYDB_HEADERS=$RUNNER_TEMP/mobilitydb/meos/include" \ + >> "$GITHUB_ENV" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install openapi-spec-validator + + # generate_openapi.py requires the enriched catalog (network fields). + # The enrichment pipeline lives on PR #4 (feat/service-enrichment), + # which on its branch rewrites run.py to do parse + reconcile + enrich + # in one step. On any branch that does not yet include #4's content, + # check out the PR #4 tree (parser/ + run.py) so the regenerate step + # uses the enriched-pipeline run.py. Tree-level checkout sidesteps + # the runner's lack of a default git identity. + - name: Fetch + apply enrichment pipeline from PR #4 if absent + run: | + if [ ! -f parser/enrich.py ]; then + echo "::notice::PR #4 enrichment pipeline not present on this branch; checking out PR #4 tree" + git fetch origin refs/pull/4/head:pr4 + # PR #4 supplies: parser/enrich.py, parser/header_types.py, and + # a rewritten run.py that orchestrates parse + reconcile + enrich + # in one invocation. Copy all three onto the working tree. + git checkout pr4 -- parser/ run.py 2>/dev/null || true + ls -la parser/enrich.py run.py + fi + + - name: Regenerate the catalog (parse + reconcile + enrich in one step) + run: python3 run.py "$MOBILITYDB_HEADERS" + + - name: Project to OpenAPI 3.1 + run: python3 generate_openapi.py + + - name: Validate meos-openapi.json against OpenAPI 3.1 + run: | + python3 -c " + import json + from openapi_spec_validator import OpenAPIV31SpecValidator + spec = json.load(open('output/meos-openapi.json')) + # OpenAPIV31SpecValidator(spec).validate() raises on violation, + # returns None on success. Works with openapi-spec-validator 0.9.x. + OpenAPIV31SpecValidator(spec).validate() + print(f\"::notice::meos-openapi.json conforms to OpenAPI 3.1 — \" + f\"{len(spec.get('paths', {}))} paths, \" + f\"{len(spec.get('components', {}).get('schemas', {}))} schemas.\") + " + + - name: Upload meos-openapi.json as artefact + uses: actions/upload-artifact@v4 + with: + name: meos-openapi + path: output/meos-openapi.json + if-no-files-found: error diff --git a/README.md b/README.md index 18da57d..7bb0254 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,10 @@ This catalog is the foundation for generating language bindings (Python, Java, G - [How it works](#how-it-works) - [Getting started](#getting-started) - [Output format](#output-format) +- [Service-projection metadata](#service-projection-metadata) - [Adding metadata](#adding-metadata) - [Portable bare-name dialect](#portable-bare-name-dialect) +- [OpenAPI generation](#openapi-generation) ## Ecosystem @@ -60,10 +62,12 @@ service contracts generated from the same model: ## How it works -The pipeline runs in two steps: +The pipeline runs in four steps: 1. **Parser** — scans the MEOS `.h` header files using libclang and extracts every function signature, struct, and enum into structured JSON. -2. **Merger** — enriches the parser output with manual annotations from `meta/meos-meta.json`, such as documentation and memory ownership rules. +2. **Reconcile** — restores opaque types the PostgreSQL stub headers `#define` to `int` (`Interval`, `text`, …) from the header source, so they are not mistaken for `int *` out-parameters. +3. **Enrich** — derives the service-projection metadata (`category` / `typeEncodings` / `network` / `wire`). +4. **Merger** — applies manual annotations from `meta/meos-meta.json` (documentation, ownership, overrides) on top. ## Getting started @@ -127,9 +131,60 @@ A typical function entry looks like this: } ``` +## Service-projection metadata + +C headers describe *signatures*; they do not say what a function **is**, how an +opaque type crosses the wire, or whether an operation can be served +*statelessly*. A second pass (`parser/enrich.py`) derives that — the metadata a +service generator (OpenAPI, MCP, gRPC, …) needs to project MEOS onto a network +API. It runs **before** the merge, so every derived field is overridable from +`meta/meos-meta.json`. + +Each function gains a `category`, a `network` verdict, and a `wire` mapping: + +```json +{ + "name": "temporal_eq", + "returnType": { "c": "bool", "canonical": "int" }, + "params": [ { "name": "temp1", "canonical": "const struct Temporal *" }, + { "name": "temp2", "canonical": "const struct Temporal *" } ], + "category": "predicate", + "network": { "exposable": true, "method": "POST", "reason": null }, + "wire": { + "params": [ + { "name": "temp1", "kind": "serialized", "cType": "const struct Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] }, + { "name": "temp2", "kind": "serialized", "cType": "const struct Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] } + ], + "result": { "kind": "json", "json": "integer" } + } +} +``` + +(MEOS predicates return `int`, and libclang emits canonical spellings such as +`const struct Temporal *` — the enrichment matches those.) + +```text +Live coverage (MobilityDB master): 2161 public + 511 internal functions. +The service projects the public user API; internal (meos_internal*.h, +Datum-generic) is policy-excluded. + 1963 / 2161 = 91% of the public API stateless-exposable (verified). +``` + +The catalog also gains a top-level `typeEncodings` map (opaque type → its +in/out functions) and an `enrichment` summary (category counts, exposable +count) for coverage tracking. Non-exposable functions carry a precise +`reason` (`array-or-out-param:…`, `no-encoder:…`, `lifecycle`, `index`, …) so +generators can report exactly what they can and cannot emit. + +See [`docs/enrichment.md`](docs/enrichment.md) for the full contract and +[`tests/test_enrich.py`](tests/test_enrich.py) for worked examples on real +MEOS signatures (run: `python3 tests/test_enrich.py`). + ## Adding metadata -Manual annotations (ownership rules, additional documentation, deprecation flags, etc.) live in `meta/meos-meta.json`. The merger applies them on top of the libclang-parsed structure when generating the final catalog. +Manual annotations (ownership rules, additional documentation, deprecation flags, etc.) live in `meta/meos-meta.json`. The merger applies them on top of the libclang-parsed structure when generating the final catalog — including any field derived by the service-projection pass (e.g. correcting a `category` or forcing `network.exposable`). ## Portable bare-name dialect @@ -148,3 +203,30 @@ type-agnostic and applies to **every** temporal type family — must not be excluded from any parity headline. `python tools/portable_parity.py` audits it against the catalog — currently **29/29 = 100%** backed (verified, no guessing). See [`docs/portable-aliases.md`](docs/portable-aliases.md). + +## OpenAPI generation + +The enriched catalog (the `network` / `wire` / `typeEncodings` produced by the +service-projection pass) can be projected onto an **OpenAPI 3.1** contract — +this is the concrete "OpenAPI is a projection of MEOS-API" step: + +```bash +python run.py # produce the enriched catalog +python generate_openapi.py # output/meos-idl.json -> output/meos-openapi.json +``` + +Every *stateless-exposable* MEOS function becomes one RPC-style +`POST /{function}` operation (≈ an OGC API – Processes "process"); opaque +values cross the wire as strings carried in their `typeEncodings` +(text / MF-JSON / HexWKB), surfaced as reusable component schemas. `x-meos-*` +extensions carry the decode/encode function names and category so a +downstream server or MCP generator can consume the same document. + +Against the live MobilityDB `master` catalog this yields **1952 operations** +(90% of the public API; internal `meos_internal*.h` policy-excluded), +including array-of-string params for builders. The generator is pure +`dict` → `dict` (no libclang, +no MEOS runtime); see [`docs/openapi.md`](docs/openapi.md) for the projection +rules, `x-meos-*` extensions, and roadmap (OGC API, MCP, runtime server), and +[`tests/test_openapi.py`](tests/test_openapi.py) for worked examples +(`python3 tests/test_openapi.py`). diff --git a/docs/enrichment.md b/docs/enrichment.md new file mode 100644 index 0000000..388cf09 --- /dev/null +++ b/docs/enrichment.md @@ -0,0 +1,251 @@ +# Service-projection enrichment + +The libclang parser can only report what is written in the C headers: +function signatures, structs, enums. Projecting MEOS onto a network service +(OpenAPI / MCP / gRPC) needs information the headers do **not** contain — what +each function *is*, how each opaque type crosses the wire, and whether an +operation can be served *statelessly*. + +`parser/enrich.py` derives that metadata from the parsed catalog. It runs +**before** `merge_meta`, so every field below can be overridden per +function/type from `meta/meos-meta.json`. + +> All values here are heuristic defaults. They are intended to be *curated* +> over time through `meta/meos-meta.json`; the heuristics give a correct-by- +> default starting point and keep coverage measurable. + +## 1. Function `category` + +Each function gets one `category` (first matching rule wins): + +| category | rule (by name / signature) | +|------------------|-------------------------------------------------------------| +| `lifecycle` | name starts `meos_` (init/finalize/configuration) | +| `index` | name starts `rtree_` | +| `io` | name matches an in/out encoding pattern (`_in`, `_out`, `_from_mfjson`, `_as_hexwkb`, `_from_wkb`, …) | +| `aggregate` | name ends `_transfn`/`_combinefn`/`_finalfn`, or `_tagg`/`_collect` | +| `predicate` | comparison/topological/temporal predicate by name (`*_eq`, `*_lt`, `contains_*`, `intersects_*`, `dwithin_*`, `ever_*`, `always_*`, `t{eq,contains,…}_*`, …) or returns `bool`. MEOS predicates return `int`, so name patterns — not the return type — drive this. | +| `constructor` | name ends `_make`/`_copy` | +| `setop` | name starts `union_`/`intersection_`/`minus_`/`difference_` | +| `conversion` | name contains `_to_`/`_from_`/`_from_base`/`_as_` | +| `accessor` | name ends with a component/property pattern (`_value`, `_srid`, `_duration`, `_num_*`, …) | +| `transformation` | default: value → value of the same family | + +## 2. `typeEncodings` + +A top-level map: opaque C type → how it round-trips to the wire. Built by +scanning the catalog for the type's own in/out functions. + +```json +"typeEncodings": { + "Temporal": { + "encodings": ["mfjson", "text", "wkb"], + "decoders": { "text": "temporal_in", "mfjson": "temporal_from_mfjson" }, + "encoders": { "text": "temporal_out", "wkb": "temporal_as_hexwkb" }, + "in": "temporal_in", + "out": "temporal_out" + } +} +``` + +- **decoder** — `const char * (+ aux) → T *` (`*_in`, `*_from_mfjson`, …) +- **encoder** — `const T * (+ aux) → char *` (`*_out`, `*_as_mfjson`, …) +- `in`/`out` — the preferred decoder/encoder, `text` > `mfjson` > `wkb`; + among candidates the **generic root** (`_in`/`_out`) is preferred + (so `temporal_out` serialises *every* subtype), else a deterministic + alphabetical pick. `in_aux`/`out_aux` carry the trailing args. + +> **Auxiliary arguments.** Real MEOS in/out wrappers (the public functions +> in the `*_meos.c` files) are not pure `(str)->T` / `(T)->str`: they take +> trailing *formatting* scalars — `temporal_out(temp, int maxdd)`, +> `*_as_mfjson(temp, with_bbox, flags, precision, srs)`. Those are safe to +> default (`maxdd`/`precision` → 15, flags/bbox → 0, `srs` → NULL), so the +> wrapper still satisfies the stateless contract; the defaults are recorded +> in `in_aux`/`out_aux` for the runtime to pass. A trailing arg that is +> *not* a defaultable formatting scalar disqualifies the wrapper: a +> semantic `*type` tag (`temporal_in`'s `temptype` — tagged +> `@ingroup meos_internal` in MEOS) or a pointer/array (`*_as_wkb`'s +> `size_out`). So polymorphic `Temporal` *decoding* resolves to a typed +> wrapper (`tbool_in`, …) — subtype-narrow on input; carrying the subtype +> on the wire for a universal decode is future work. *Encoding* is already +> universal via the generic `temporal_out`. + +The same data is folded onto each `structs[*]` entry as `serialization`. + +## 3. `network` and `wire` + +For every function: + +```json +"category": "predicate", +"network": { "exposable": true, "method": "POST", "reason": null }, +"wire": { + "params": [ + { "name": "temp1", "kind": "serialized", "cType": "Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] }, + { "name": "temp2", "kind": "serialized", "cType": "Temporal *", + "decode": "temporal_in", "encodings": ["mfjson","text","wkb"] } + ], + "result": { "kind": "json", "json": "boolean" } +} +``` + +`wire` element `kind`: + +| kind | meaning | JSON Schema hint | +|--------------|------------------------------------------------------|------------------| +| `json` | scalar; `json` ∈ integer/number/boolean/string. Enum types add `"enum": ""` | direct | +| `serialized` | opaque value carried as a string in its `encodings`; `decode`/`encode` names the MEOS function | `{"type":"string"}` + media type | +| `array` | JSON array of `element` (an `Elem` builder param, or an `Elem **`+count return) | `{"type":"array","items":…}` | +| `void` | no return value | `204` | +| `unsupported`| cannot be represented in a stateless request/response| — | + +> **Canonical spellings.** libclang emits canonical C, not source aliases: +> `struct Temporal *` (not `Temporal *`), `unsigned char` (not `uint8_t`), +> `long` (not `int64_t`), and MEOS uses `int` for booleans. The heuristics +> match canonical spellings; `struct`/`union`/`enum` qualifiers are stripped +> so `typeEncodings` keys are clean (`Temporal`, not `struct Temporal`). +> `enum` parameters are scalars; only declared structs become `serialized`. + +### Exposability + +`network.exposable` is `true` iff **every** parameter is `json` or +`serialized` **and** the result is `json`, `serialized`, or `void`. +Otherwise `exposable` is `false` and `reason` lists the blockers +(deduplicated, `;`-joined): + +| reason | cause | +|------------------------------|-------------------------------------------------------------| +| `lifecycle` / `index` | library plumbing, not a domain operation | +| `array-or-out-param:` | parameter is a pointer to a scalar/array or an out-parameter (`T **`, `int *`, `double *`, …) | +| `no-decoder:` | opaque parameter type has no parser function | +| `no-encoder:` | opaque return type has no serializer function | +| `unsupported-return:` | return cannot be represented on the wire | + +> **Out-parameters.** A common MEOS accessor shape — +> `bool f(.., T *result)` (or `T **result`) — returns its value through a +> trailing out-parameter, with the `bool`/`int` return acting as a presence +> flag (`void` = always present). Two safe shapes are recognised: a +> **scalar** `T *result` becomes the JSON `result`; an **opaque** +> `T **result` becomes a `serialized` result via the type's encoder. In +> both cases `from_outparam`/`out_ctype`/`presence_return` annotate the +> wire result, the function is `exposable`, and a false presence return +> maps to *no value* (HTTP 204). This recovers the public `*_value_n`, +> box-bound, and `*_value_at`/geo-accessor families. + +> **Input-array builders.** A builder taking `(Elem **arr, int count)` +> becomes one wire param of `kind: "array"` (element = the serialized +> `Elem`); the `count` is implicit (the JSON array length). This recovers +> `*_make` / `*_merge_array` / `*arr_to_*` builders whose element type is +> decodable. + +> **Array returns.** An accessor `Elem **f(.., int *count)` returning a +> freshly-allocated element array becomes a `result` of `kind: "array"` +> (element = the serialized `Elem`, `count_outparam` names the byref +> length). Recovers `temporal_instants`/`segments`/`sequences`, +> `tgeo_values`, `geo_pointarr`, … whose element type is encodable. + +This is the precise, machine-checkable boundary of what a generator can emit +today. Functions still blocked only by `array-or-out-param` (multi- or +array out-params, builder `T **`+count) are the candidates for the next, +hand-designed composite-endpoint unit; everything `exposable` can be +generated mechanically. + +## 4. Overriding + +`merge_meta` applies `meta/meos-meta.json` *after* enrichment, so any derived +field can be corrected by hand: + +```json +{ + "functions": { + "temporal_at_value": { "category": "transformation" }, + "some_fn": { "network": { "exposable": false, "reason": "side-effect" } } + } +} +``` + +## 5. Catalog summary + +`enrich_idl` adds an `enrichment` block for coverage tracking. Run against +the live MobilityDB `master` catalog (2672 functions, 47 structs, 6 enums): + +```json +"enrichment": { + "categoryCounts": { ... }, + "publicFunctions": 2161, + "internalFunctions": 511, + "exposableFunctions": 1963 +} +``` + +MEOS has two API surfaces: the **public user API** (`meos.h` + the public +type headers, 2161 functions) and the **internal programmer API** +(`meos_internal*.h`, 511 functions — type-erased `Datum`-generic, +undocumented for end users). A network service projects the *user* API, so +internal functions are **policy-excluded** (`reason: internal`, like +`lifecycle`/`index`); `132/133` `Datum` functions are internal and never +belonged in the parity denominator. + +So **1963 / 2161 = 91% of the public API** projects onto a stateless +endpoint as-is — verified by a strict invariant (every exposable function +has only scalar/enum/serialized/array params with a real decoder, and an +encodable result; 0 violations; 0 internal leaks). The 209-function public +remainder is dominated by **irregular** signatures with no clean stateless +shape (mixed/odd `array-or-out-param`, raw-array `unsupported-return`, +out-params of codec-less types) plus `SkipList` aggregate state and +`lifecycle`/`index` plumbing — all excluded with a truthful `reason`, +never silently mis-called. + +`report.py` emits `output/meos-coverage.json`: an **actionable worklist**. +Every non-exposable public function carries a `class` and a concrete +`suggest` — the precise upstream regularization that closes it — and +`byClass` ranks classes by leverage. Live (`master`, gap 198): + +| class | n | upstream action | +|---|---|---| +| `out-param-naming` | 76 | rename the lone out-parameter to `result` (one-liner each) | +| `plumbing` | 37 | none — `lifecycle`/`index`, intentionally not exposed | +| `stateful` | 30 | none — aggregate state; needs a stateful endpoint, not a stateless RPC | +| `array-return-shape` | 21 | add a trailing `int *count` (+ element encoder) | +| `multi-out` | 13 | return a struct, or split into single-result accessors | +| `other` / `no-codec` / `array-shape` / `internal-generic` | 11+7+3+1 | add a 1-arg `T_in`/`T_out`; keep `Datum`-generic internal | + +**Honest ceiling.** ~91% is the *safe principled* maximum for this layer. +The `out-param-naming` 76 are **not** closed on this side on purpose: +a name-agnostic "trailing pointer ⇒ out-parameter" rule would misread +genuine pointer *inputs* as out-parameters — silently wrong answers. +Correctness-over-coverage makes these a cheap **upstream** rename, now +enumerated. Everything remaining is therefore upstream (precisely +specified above) or definitional (`stateful`/`plumbing` — correct, +labelled exclusions, never silent). The catalog regenerates from headers, +so upstream fixes flow in automatically on the next run (no coupling). + +### Compensation register (retirement path) + +Each heuristic exists only to absorb a current irregularity and is +**deleted** once the irregularity is uniformized upstream: + +| compensation | absorbs | retire when upstream… | +|---|---|---| +| `header_types.reconcile` / `_preserved_opaque` | stub `#define`s erasing `Interval`/`text`/`Datum` to `int` | the public headers no longer route opaque types through `int` stubs | +| `_aux_specs` (default `maxdd`/mfjson flags) | in/out wrappers carrying formatting args | I/O wrappers are pure `(str)->T`/`(T)->str` | +| typed-decode (`tbool_in` for polymorphic `Temporal`) | generic `temporal_in` needing a `meosType` tag | a tag-free polymorphic decoder exists | +| out-param / array-builder / array-return shapes | irregular out/array signatures | signatures follow the canonical shapes in the worklist | + +`tests/test_coverage_gate.py` asserts coverage does not regress. + +> **Type reconciliation.** The PostgreSQL stub headers `#define` +> `Interval`/`text`/`TimestampTz`/… to `int` *before* libclang parses, so +> those opaque pointers reached the catalog as `int *` — +> indistinguishable from a real `int *` out-parameter. `parser/ +> header_types.py` re-scans the header *source* and restores the true +> named type wherever libclang produced a bare scalar but the source +> declares a distinct named pointer (scalar typedefs like `TimestampTz` +> are deliberately left resolved). This is primarily a *correctness* fix — +> `add_timestamptz_interval`'s `interv` is now honestly `Interval`, not a +> phantom `int *`. `text *` is then treated as a JSON string (it *is* one), +> and the opaque-codec gate spans *any* named non-scalar pointer type (not +> just parsed structs), so reconciled types register their own in/out +> (`Interval` ↔ `pg_interval_in`/`interval_out`). Together this lifts +> verified public coverage to 1963/2161 (91%). diff --git a/docs/integration-train.md b/docs/integration-train.md new file mode 100644 index 0000000..a6bbb2f --- /dev/null +++ b/docs/integration-train.md @@ -0,0 +1,84 @@ +# Integration train — making ecosystem-wide 100% parity provable + +## Why this exists + +The MobilityDB ecosystem (MobilityDB · MEOS-API · PyMEOS-CFFI · PyMEOS · +MobilityDuck · MobilitySpark · MobilityAPI · JMEOS · GoMEOS · MEOS.NET · +MEOS.js · MobilityDB-BerlinMOD · the stream-side platforms MobilityFlink +· MobilityKafka · MobilityNebula) carries a fan of individually-correct +**open** PRs. Each is green *in isolation*, but: + +- every PR's CI builds against a `master` that lacks the *other* PRs' + content (PyMEOS CI builds MEOS from MobilityDB master → lacks the + extended-type C surface; PyMEOS code is broken vs MEOS master — the + rename skew); +- the maintainer is the only merge gate — no automated merge; +- per-PR independence means **nobody assembles and verifies the + integrated whole**. + +So "100% parity" is true per-branch yet **unprovable as a system**. This +train operationalizes MobilityDB discussion **#895 (wave-based merge +plan)**: a dependency-ordered manifest + a one-command verifier so parity +is demonstrated *at one point*, and the maintainer gets an ordered, +de-risked merge sequence instead of N cross-dependent PRs reading red. + +## Artifacts + +- [`meta/integration-train.json`](../meta/integration-train.json) — the + PR dependency DAG, per-wave compose recipe, gates, owners, merge order. +- [`verify-train.sh`](../verify-train.sh) — composes the train and runs + each wave's gate. Honesty contract: a wave is `PASS` only when its gate + is just-run green here; otherwise `BLOCKED:` with the exact + gate it needs. Nothing faked or silently skipped. + +## The waves + +| Wave | Content | Status | +|---|---|---| +| **0** | MobilityDB extended-type C surface (stack #1081→#1085, then #1051→#951) | **PROVEN** — 2699 fns, MEOS-API PR #10 21/21, `from_mfjson`/ctors uniform | +| **1** | PyMEOS-CFFI MEOS-1.4 substrate (regenerate vs Wave-0) | IN_FLIGHT (PyMEOS-CFFI #18, #19) | +| **2** | **CRITICAL PATH** — PyMEOS MEOS-1.4 bump (#81/#82): kills the rename skew | BLOCKING_ALL | +| **3** | PyMEOS features: #85, #87, #88, #89→#90→#91, #84 (+ MobilityDuck #146/#147 for #84 interop) | GREEN_IN_ISOLATION, gated on Wave 2 | +| **4** | Downstream bindings (MobilityDuck 47 / MobilitySpark 10 / MobilityAPI 6 / JMEOS 6 / GoMEOS 4 / MEOS.NET 3 open PRs) | GREEN_IN_ISOLATION; JMEOS is the lone repo under structural-migration pressure (5/6 CONFL) | +| **5** | Service-agent + data-lake + stream layers (built on Wave-4 anchor) | ANCHOR_DEFINED (MEOS-API #4-7, #12-13; PyMEOS #84 + MobilityDuck #146/#147/#158; stream-layer planned-band) | + +**Wave 2 is the single universal unblock.** Every PyMEOS parity claim is +downstream of it; nothing else accelerates 100% parity more. Build the +bump against the composed Wave 0 (not bare master) so it is done once, +correctly, against the final C surface. + +**Waves 4 and 5** consume Wave 0's MEOS-1.4 C surface via the MEOS-API +`meos-idl.json` catalog. Each Wave-4 binding is bump-independent within +its own repo; the cross-binding gate is that the regenerated +`meos-idl.json` is byte-identical across them (proves single SoT). + +## Branch base + +This branch is **stacked on `feat/object-model` (MEOS-API PR #10)** — the +Wave-0 gate asserts the object-model classification (`from_mfjson` → +TCbuffer/TNpoint/TPose, concrete `*inst_make` constructors), which is +PR #10's pipeline. PR #10 (object model) + PR #8 (portable-aliases SoT) +are the catalog anchor of the train; see +`meta/integration-train.json#/catalog_anchor`. + +## Running it + +```bash +python3 setup.py # one-time: fetch the MobilityDB sources +./verify-train.sh # Wave 0 fully verified here; Waves 1-3 report + # honest BLOCKED + the exact gate each needs +PYMEOS_ENV= ./verify-train.sh # post-Wave-2 +``` + +## Current status + +Wave 0 is **proven**. Waves 1–3 are entirely gated on Wave 2 (the +MEOS-1.4 bump). Wave 4 (downstream bindings) is green-in-isolation +across MobilityDuck / MobilitySpark / MobilityAPI / GoMEOS / MEOS.NET; +JMEOS (5/6 CONFL) is under structural-migration pressure post the +multi-module restructure. Wave 5 (service-agent + data-lake + stream +layers) is anchor-defined and downstream of Wave 4. There is **no +remaining correctness work** — every deliverable is verified correct +in isolation; the gap is purely integration ordering plus the +maintainer-only merge gate, which this train reduces to: *merge in +wave order; CI turns green at each wave.* diff --git a/docs/openapi.md b/docs/openapi.md new file mode 100644 index 0000000..68c5df0 --- /dev/null +++ b/docs/openapi.md @@ -0,0 +1,75 @@ +# OpenAPI projection + +`generator/openapi.py` turns the **enriched** catalog (`meos-idl.json` with +`category` / `network` / `wire` / `typeEncodings` — see +[`enrichment.md`](enrichment.md)) into an **OpenAPI 3.1** document. It is the +concrete realisation of "OpenAPI is a projection of MEOS-API": the canonical +semantic catalog is the single source, OpenAPI is one rendering of it. + +```bash +python run.py # enriched catalog -> output/meos-idl.json +python generate_openapi.py # -> output/meos-openapi.json +``` + +Pure `dict` → `dict`: no libclang, no MEOS runtime, deterministic output +(sorted paths/schemas) so generated diffs are reviewable. + +## Projection rules + +| MEOS concept | OpenAPI | +|---|---| +| stateless-exposable function | one `POST /{function}` operation, `operationId = function` | +| `category` | operation `tags` + `x-meos-category`; spec-level `tags` list | +| parameter | property of the JSON request body (all required, `additionalProperties:false`) | +| `wire.kind = json` scalar | `{"type": integer\|number\|boolean\|string}` | +| `wire` enum | `$ref` to a component enum schema (string, real C constant names) | +| `wire.kind = serialized` | `allOf` → `$ref` to the type's component schema, plus `x-meos-decode` (request) / `x-meos-encode` (response) | +| `wire.kind = array` (builder `(Elem **,count)`) | `{"type":"array","items":}` + `x-meos-decode`; the C `count` is the array length | +| out-parameter result (`from_outparam`) | the out-param value is the response (scalar JSON or serialized); `presence_return` false ⇒ `204` | +| `wire.result.kind = void` | `204 No Content` | +| any error | `default` → `#/components/responses/MeosError` | +| `typeEncodings[T]` | `components.schemas.T` = `{"type":"string", x-meos-encodings, x-meos-in, x-meos-out}` | + +RPC-style, not resource-style, is deliberate: MEOS is a value algebra, so a +function ≈ an **OGC API – Processes** "process". A resource model +(OGC API – Moving Features collections) is a different projection, layered +later (and already partly served by +[MobilityAPI](https://github.com/MobilityDB/MobilityAPI)). + +## `x-meos-*` extensions + +The spec is self-describing for downstream generators (server, MCP, gRPC): + +- `info.x-meos-coverage` — `{functions, exposed}`. +- operation `x-meos-category`, `x-meos-encode`. +- serialized request property `x-meos-decode` — the MEOS parse function. +- component schema `x-meos-encodings` / `x-meos-in` / `x-meos-out` — the wire + encodings and the MEOS in/out function names. + +A server generator marshals a request by calling `x-meos-decode` on each +serialized string, invoking the function, and calling `x-meos-encode` on the +result — no extra metadata needed beyond this document. + +## Coverage (live MobilityDB `master`) + +2161 **public** functions → **1952 operations (85%)** — the internal +`meos_internal*.h` programmer API (511 fns, `Datum`-generic) is +policy-excluded. Tagged across `predicate`, `transformation`, `accessor`, +`io`, `conversion`, `setop`, `aggregate`, `constructor`. The remaining +public functions (multi-out/array builders, opaque-no-codec, polymorphic +input-decode, lifecycle/index) carry a truthful `reason` and are +overridable via `meta/meos-meta.json`. + +## Limitations / roadmap + +- **No OpenAPI conformance validation** in-tree yet (structural checks only: + every path a single `POST` with responses, all `$ref`s resolve). Adding + `openapi-spec-validator` to CI is a follow-up. +- **MCP tool manifest** — the same `wire`/`typeEncodings` model maps directly + onto MCP tool schemas; a sibling generator is the natural next unit. +- **Runtime server** — a generated marshaling server (decode → call → encode) + is out of scope here; this PR delivers the *contract*, not the server. +- **OGC API – Moving Features** resource projection is a separate effort. +- Preferred in/out per type currently follows catalog scan order (e.g. + `tbool_in` may be picked over `temporal_in`); both are valid decoders, but + refining the preference is a small enrichment-side follow-up. diff --git a/generate_movfeat_openapi.py b/generate_movfeat_openapi.py new file mode 100644 index 0000000..276e834 --- /dev/null +++ b/generate_movfeat_openapi.py @@ -0,0 +1,54 @@ +"""Generate the OGC API – Moving Features OpenAPI projection from the +enriched MEOS catalog. + +Usage: + python run.py # produce output/meos-idl.json + python enrich.py # add network/wire/api fields + python generate_movfeat_openapi.py # read output/meos-idl.json + python generate_movfeat_openapi.py path.json out.json +""" + +import json +import sys +from pathlib import Path + +from generator.movfeat import build_movfeat_openapi, _missing_summary + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = (Path(sys.argv[2]) if len(sys.argv) > 2 + else Path("output/meos-movfeat-openapi.json")) + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + + catalog = json.loads(IN_PATH.read_text()) + if "functions" not in catalog or not any( + "network" in f for f in catalog["functions"] + ): + sys.exit(f"{IN_PATH} is not enriched (no `network` fields). " + "Run the enrichment pass first.") + + doc = build_movfeat_openapi(catalog) + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(doc, indent=2) + "\n") + + cov = doc["info"]["x-meos-coverage"] + print(f" → {OUT_PATH} written", file=sys.stderr) + print( + f"Done: {len(doc['paths'])} OGC API – Moving Features paths over " + f"{len(doc['components']['schemas'])} component schemas; " + f"{cov['meos_backed']}/{cov['routes']} routes have a MEOS backing, " + f"{cov['persistence_only']} are persistence-layer.", + file=sys.stderr, + ) + + msg = _missing_summary(cov["missing_in_catalog"]) + if msg: + print(msg, file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/generate_openapi.py b/generate_openapi.py new file mode 100644 index 0000000..0a12a11 --- /dev/null +++ b/generate_openapi.py @@ -0,0 +1,40 @@ +# Generate an OpenAPI 3.1 contract from the enriched MEOS catalog. +# +# Usage: +# python run.py # first, to produce the catalog +# python generate_openapi.py # reads output/meos-idl.json +# python generate_openapi.py path.json [out.json] + +import json +import sys +from pathlib import Path + +from generator.openapi import build_openapi + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("output/meos-openapi.json") + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + + catalog = json.loads(IN_PATH.read_text()) + if "functions" not in catalog or not any( + "network" in f for f in catalog["functions"] + ): + sys.exit(f"{IN_PATH} is not enriched (no `network` fields). " + "Run the enrichment pass first.") + + spec = build_openapi(catalog) + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(spec, indent=2)) + + print(f"[openapi] {len(spec['paths'])} operations, " + f"{len(spec['components']['schemas'])} component schemas " + f"→ {OUT_PATH}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/generator/movfeat.py b/generator/movfeat.py new file mode 100644 index 0000000..849aed6 --- /dev/null +++ b/generator/movfeat.py @@ -0,0 +1,375 @@ +"""OGC API – Moving Features (OGC 22-003r3) generator. + +Projects the *enriched* MEOS catalog (``meos-idl.json`` with ``category`` / +``network`` / ``wire`` / ``typeEncodings``, produced by ``parser/enrich.py``) +onto an **OGC API – Moving Features**-shaped OpenAPI 3.1 service contract. + +The OGC projection is a complementary view to the generic ``generate_openapi.py``: + +- Generic projection (``generator/openapi.py``): every stateless-exposable + MEOS function becomes one RPC-style ``POST /{function}`` operation. Faithful + to MEOS's value-algebra shape but not aligned with OGC standards. +- **MovFeat projection (this module)**: only the MEOS functions that have an + OGC API – Moving Features analogue are exposed, and they are placed under + the OGC-defined REST resource hierarchy (``/collections/{cid}/items/{fid}/...``). + Each route carries an ``x-meos-{decode,encode,function}`` extension so a + downstream OGC server (MobilityAPI, in this ecosystem) can dispatch each + call to the right MEOS function without re-deriving the mapping. + +The output is a *subset* of MEOS. Functions without an OGC analogue stay in +the generic OpenAPI; they are not duplicated here. MobilityAPI's runtime +serves both projections side by side: OGC-shaped paths for adopter clients +that follow the OGC API – Moving Features standard, generic RPC paths for +clients that want the raw MEOS surface. + +Pure ``dict`` → ``dict``; no libclang and no MEOS runtime. Catalog enrichment +is a prerequisite (the ``network`` / ``wire`` / ``typeEncodings`` fields are +authored by ``parser/enrich.py``). +""" + +from __future__ import annotations + +from typing import Iterable + +# Reuse the schema-building primitives from the generic OpenAPI generator so +# component schemas and the MeosError response are byte-identical across the +# two projections. +from generator.openapi import ( + _value_schema, _type_schema, _enum_schema, +) + + +# OGC API – Moving Features path map. +# +# Each entry maps an OGC-defined path under +# ``/collections/{collectionId}/items/{featureId}`` to the MEOS function the +# OGC operation dispatches to. The MEOS function name MUST match an entry in +# the enriched catalog with ``network.exposable == true``; entries whose +# MEOS function is absent from the catalog are skipped (with a warning on +# stderr). +# +# Method-and-shape rules: +# - GET on a deterministic accessor (no body): the MEOS function takes a +# single ``Temporal *`` opaque + zero or more query parameters; result +# is encoded per its ``wire.result.encode``. +# - POST on a constructor / aggregator: the MEOS function takes a request +# body whose shape matches the function's ``wire.params``. +# - DELETE / PUT: persistence-layer concerns owned by MobilityAPI, not by +# MEOS — the path is exposed but ``x-meos-function`` is null and the +# request body / response schemas are OGC-defined rather than MEOS-derived. +_OGC_MOVFEAT_ROUTES: list[dict] = [ + # --- Collection-level (not MEOS-backed; persistence-layer in MobilityAPI) --- + {"path": "/collections", "method": "get", + "operationId": "listCollections", "meos": None, + "summary": "List feature collections."}, + {"path": "/collections", "method": "post", + "operationId": "createCollection", "meos": None, + "summary": "Create a new feature collection."}, + {"path": "/collections/{collectionId}", "method": "get", + "operationId": "getCollection", "meos": None, + "summary": "Retrieve a collection's metadata."}, + {"path": "/collections/{collectionId}", "method": "put", + "operationId": "replaceCollection", "meos": None, + "summary": "Replace a collection's metadata."}, + {"path": "/collections/{collectionId}", "method": "delete", + "operationId": "deleteCollection", "meos": None, + "summary": "Delete a collection."}, + # --- Item-level (MEOS-backed for the I/O paths) --- + {"path": "/collections/{collectionId}/items", "method": "get", + "operationId": "listItems", "meos": None, + "summary": "List moving-feature items in a collection."}, + {"path": "/collections/{collectionId}/items", "method": "post", + "operationId": "createItem", "meos": "temporal_from_mfjson", + "summary": "Create a moving-feature item from an MF-JSON payload."}, + {"path": "/collections/{collectionId}/items/{featureId}", "method": "get", + "operationId": "getItem", "meos": "temporal_as_mfjson", + "summary": "Retrieve a moving-feature item as MF-JSON."}, + {"path": "/collections/{collectionId}/items/{featureId}", "method": "delete", + "operationId": "deleteItem", "meos": None, + "summary": "Delete a moving-feature item."}, + # --- Trajectory-derived (the MEOS-rich part of the MovFeat surface) --- + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence", + "method": "get", + "operationId": "getTrajectory", "meos": "temporal_as_mfjson", + "summary": "Retrieve the full trajectory geometry as MF-JSON."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/velocity", + "method": "get", + "operationId": "getVelocity", "meos": "tpoint_speed", + "summary": "Retrieve the temporal velocity profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/acceleration", + "method": "get", + "operationId": "getAcceleration", "meos": "temporal_derivative", + "summary": "Retrieve the temporal acceleration profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/distance", + "method": "get", + "operationId": "getDistance", "meos": "tpoint_cumulative_length", + "summary": "Retrieve the cumulative-length distance profile."}, + {"path": "/collections/{collectionId}/items/{featureId}/tgsequence/azimuth", + "method": "get", + "operationId": "getAzimuth", "meos": "tpoint_azimuth", + "summary": "Retrieve the temporal azimuth profile."}, + # --- Property-level (per-name temporal property access) --- + {"path": "/collections/{collectionId}/items/{featureId}/tproperty/{propertyName}", + "method": "get", + "operationId": "getProperty", "meos": "temporal_as_mfjson", + "summary": "Retrieve a temporal property's MF-JSON encoding."}, + {"path": "/collections/{collectionId}/items/{featureId}/tproperty/{propertyName}", + "method": "delete", + "operationId": "deleteProperty", "meos": None, + "summary": "Delete a temporal property."}, +] + + +_PATH_PARAM_DESCRIPTIONS = { + "collectionId": "Identifier of the feature collection.", + "featureId": "Identifier of the moving-feature item within the collection.", + "propertyName": "Name of the temporal property.", +} + + +def _path_parameters(path: str) -> list[dict]: + """Extract `{name}` placeholders from `path` and return OpenAPI parameter objects.""" + import re + return [ + { + "name": name, + "in": "path", + "required": True, + "schema": {"type": "string"}, + "description": _PATH_PARAM_DESCRIPTIONS.get( + name, f"Path parameter `{name}`." + ), + } + for name in re.findall(r"\{(\w+)\}", path) + ] + + +def _meos_request_body(meos_fn: dict, used_types: set, used_enums: set) -> dict | None: + """Build a JSON request body schema from a MEOS function's wire params.""" + wire = meos_fn["wire"] + params = wire.get("params", []) + if not params: + return None + props: dict = {} + required: list = [] + for p in params: + props[p["name"]] = _value_schema(p, used_types, used_enums) + required.append(p["name"]) + if p["kind"] == "serialized": + props[p["name"]] = { + "allOf": [props[p["name"]]], + "x-meos-decode": p["decode"], + } + elif p["kind"] == "array": + props[p["name"]] = { + **props[p["name"]], + "x-meos-decode": p["element"]["decode"], + } + return { + "required": True, + "content": {"application/json": {"schema": { + "type": "object", + "required": required, + "additionalProperties": False, + "properties": props, + }}}, + } + + +def _meos_response_schema(meos_fn: dict, used_types: set, + used_enums: set) -> tuple[dict | None, str | None]: + """Build a response schema + encode-name from a MEOS function's wire result.""" + wire = meos_fn["wire"] + result = wire["result"] + if result["kind"] == "void": + return None, None + schema = _value_schema(result, used_types, used_enums) + encode = result.get("encode") if result["kind"] == "serialized" else None + return schema, encode + + +def _build_operation(route: dict, fns_by_name: dict, + used_types: set, used_enums: set) -> dict: + op: dict = { + "operationId": route["operationId"], + "summary": route["summary"], + "tags": ["MovingFeatures"], + } + parameters = _path_parameters(route["path"]) + if parameters: + op["parameters"] = parameters + + meos_name = route["meos"] + if meos_name and meos_name in fns_by_name: + meos_fn = fns_by_name[meos_name] + op["x-meos-function"] = meos_name + op["x-meos-category"] = meos_fn.get("category", "ogc") + + if route["method"] == "post": + body = _meos_request_body(meos_fn, used_types, used_enums) + if body is not None: + op["requestBody"] = body + + if route["method"] in ("get", "post"): + schema, encode = _meos_response_schema( + meos_fn, used_types, used_enums + ) + if schema is not None: + if encode: + op["x-meos-encode"] = encode + # MF-JSON is the OGC-standard MoveFeat encoding; advertise it + # as the primary content type when the MEOS encode is mfjson. + if encode == "as_mfjson": + op["responses"] = {"200": { + "description": "Result (MF-JSON)", + "content": { + "application/geo+json": {"schema": schema}, + "application/json": {"schema": schema}, + }, + }} + else: + op["responses"] = {"200": { + "description": "Result", + "content": {"application/json": {"schema": schema}}, + }} + else: + op["responses"] = {"204": {"description": "No content"}} + else: + # Persistence-layer route — MobilityAPI owns the body / response shape; + # MEOS doesn't get called for this operation. + op["x-meos-function"] = None + op["responses"] = _persistence_layer_responses(route) + + op.setdefault("responses", {"204": {"description": "No content"}}) + op["responses"]["default"] = { + "$ref": "#/components/responses/MeosError" + } + return op + + +def _persistence_layer_responses(route: dict) -> dict: + """Conservative default responses for OGC routes with no MEOS analogue.""" + method = route["method"] + if method == "get": + return {"200": { + "description": "Resource", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "post": + return {"201": { + "description": "Created", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "put": + return {"200": { + "description": "Replaced", + "content": {"application/json": {"schema": {"type": "object"}}}, + }} + if method == "delete": + return {"204": {"description": "Deleted"}} + return {"200": {"description": "OK"}} + + +def build_movfeat_openapi(catalog: dict, *, + title: str = "MEOS API – OGC Moving Features", + version: str = "0.1.0-draft") -> dict: + """Build an OGC API – Moving Features OpenAPI 3.1 document. + + Routes whose ``meos`` field references a function not present in the + enriched catalog (or not exposable) are skipped. The total number of + routes and the number that have a MEOS backing are reported on + ``info.x-meos-coverage``. + """ + functions = { + f["name"]: f + for f in catalog.get("functions", []) + if f.get("network", {}).get("exposable") + } + type_encodings = catalog.get("typeEncodings", {}) + enums = catalog.get("enums", []) + + used_types: set = set() + used_enums: set = set() + paths: dict = {} + meos_backed = 0 + persistence_only = 0 + missing: list = [] + + # Group route entries by path so multiple methods on the same path collapse. + for route in _OGC_MOVFEAT_ROUTES: + if route["meos"] and route["meos"] not in functions: + missing.append(route["meos"]) + persistence_only += 1 + elif route["meos"]: + meos_backed += 1 + else: + persistence_only += 1 + op = _build_operation(route, functions, used_types, used_enums) + paths.setdefault(route["path"], {})[route["method"]] = op + + schemas: dict = {} + for t in sorted(used_types): + schemas[t] = _type_schema(t, type_encodings) + for e in sorted(used_enums): + schemas[e] = _enum_schema(e, enums) + + doc = { + "openapi": "3.1.0", + "info": { + "title": title, + "version": version, + "description": ( + "OGC API – Moving Features (OGC 22-003r3) projection of the " + "MEOS catalog. Routes follow the OGC-defined REST resource " + "hierarchy under `/collections/{collectionId}/items/{featureId}/…`; " + "each route that has a MEOS analogue carries an " + "`x-meos-function` / `x-meos-decode` / `x-meos-encode` extension " + "so a downstream OGC server can dispatch to MEOS without " + "re-deriving the mapping. Routes without a MEOS analogue are " + "persistence-layer concerns owned by the consuming server. " + "Generated, do not edit by hand." + ), + "x-meos-coverage": { + "routes": len(_OGC_MOVFEAT_ROUTES), + "meos_backed": meos_backed, + "persistence_only": persistence_only, + "missing_in_catalog": sorted(set(missing)), + }, + }, + "tags": [{"name": "MovingFeatures", + "description": ( + "OGC API – Moving Features operations. Trajectory-derived " + "paths dispatch to MEOS via the `x-meos-function` " + "extension; collection-level and item-level persistence " + "paths are owned by the consuming server." + )}], + "paths": dict(sorted(paths.items())), + "components": { + "schemas": schemas, + "responses": { + "MeosError": { + "description": "MEOS error", + "content": {"application/json": {"schema": { + "type": "object", + "properties": { + "error": {"type": "string"}, + "code": {"type": "integer"}, + }, + "required": ["error"], + }}}, + } + }, + }, + } + return doc + + +def _missing_summary(missing: Iterable[str]) -> str: + """Human-readable summary for stderr.""" + missing = sorted(set(missing)) + if not missing: + return "" + return ( + f"warning: {len(missing)} OGC route(s) reference MEOS functions absent " + f"from the catalog: {', '.join(missing)}" + ) diff --git a/generator/openapi.py b/generator/openapi.py new file mode 100644 index 0000000..6244bac --- /dev/null +++ b/generator/openapi.py @@ -0,0 +1,209 @@ +"""OpenAPI 3.1 generator. + +Projects the *enriched* MEOS catalog (``meos-idl.json`` with ``category`` / +``network`` / ``wire`` / ``typeEncodings``, produced by ``parser/enrich.py``) +onto an OpenAPI 3.1 service contract. + +The projection is deliberately RPC-style — MEOS is a value algebra, not a +REST resource model, so each *stateless-exposable* function becomes one +``POST /{function}`` operation (≈ an OGC API – Processes "process"). Opaque +values cross the wire as strings carried in their `typeEncodings` (text / +MF-JSON / WKB); each opaque type and referenced enum becomes a reusable +component schema. ``x-meos-*`` extensions carry the decode/encode function +names and category so a downstream server or MCP generator can consume this +same document. + +Pure ``dict`` → ``dict``; no libclang and no MEOS runtime. Only functions +with ``network.exposable == true`` are emitted; the rest are reported by +``build_openapi``'s return-value count via the caller. +""" + +import re + +_QUAL_RE = re.compile(r"\b(const|volatile|struct|union|enum)\b") + +_PRIMITIVE = { + "integer": {"type": "integer"}, + "number": {"type": "number"}, + "boolean": {"type": "boolean"}, + "string": {"type": "string"}, +} + + +def _clean_type(c_type: str) -> str: + """``const struct Temporal *`` -> ``Temporal`` (matches typeEncodings keys).""" + return " ".join(_QUAL_RE.sub(" ", c_type).replace("*", " ").split()) + + +def _scalar_schema(wire: dict, used_enums: set) -> dict: + if wire.get("enum"): + used_enums.add(wire["enum"]) + return {"$ref": f"#/components/schemas/{wire['enum']}"} + return dict(_PRIMITIVE.get(wire.get("json", "string"), {"type": "string"})) + + +def _value_schema(wire: dict, used_types: set, used_enums: set) -> dict: + """Schema for one parameter or the result.""" + kind = wire["kind"] + if kind == "json": + return _scalar_schema(wire, used_enums) + if kind == "serialized": + t = _clean_type(wire["cType"]) + used_types.add(t) + return {"$ref": f"#/components/schemas/{t}"} + if kind == "array": + return {"type": "array", + "items": _value_schema(wire["element"], used_types, + used_enums)} + # Should not happen for an exposable function. + return {"type": "string"} + + +def _operation(fn: dict, used_types: set, used_enums: set) -> dict: + wire = fn["wire"] + op = { + "operationId": fn["name"], + "summary": fn.get("doc") or fn["name"], + "tags": [fn["category"]], + "x-meos-category": fn["category"], + } + + params = wire.get("params", []) + if params: + props, required = {}, [] + for p in params: + props[p["name"]] = _value_schema(p, used_types, used_enums) + required.append(p["name"]) + if p["kind"] == "serialized": + props[p["name"]] = { + "allOf": [props[p["name"]]], + "x-meos-decode": p["decode"], + } + elif p["kind"] == "array": + props[p["name"]] = { + **props[p["name"]], + "x-meos-decode": p["element"]["decode"], + } + op["requestBody"] = { + "required": True, + "content": {"application/json": {"schema": { + "type": "object", + "required": required, + "additionalProperties": False, + "properties": props, + }}}, + } + + result = wire["result"] + if result["kind"] == "void": + op["responses"] = {"204": {"description": "No content"}} + else: + schema = _value_schema(result, used_types, used_enums) + content_schema = schema + if result["kind"] == "serialized": + op["x-meos-encode"] = result["encode"] + op["responses"] = {"200": { + "description": "Result", + "content": {"application/json": {"schema": content_schema}}, + }} + op["responses"]["default"] = { + "$ref": "#/components/responses/MeosError" + } + return op + + +def _type_schema(name: str, type_encodings: dict) -> dict: + te = type_encodings.get(name) + if not te: + return {"type": "string", "title": name} + encs = te.get("encodings", []) + return { + "type": "string", + "title": name, + "description": ( + f"Serialized MEOS {name}. Wire encodings: {', '.join(encs)} " + f"(e.g. WKT / MF-JSON / HexWKB)." + ), + "x-meos-encodings": encs, + "x-meos-in": te.get("in"), + "x-meos-out": te.get("out"), + } + + +def _enum_schema(name: str, enums: list) -> dict: + for e in enums: + if e["name"] == name: + return { + "type": "string", + "title": name, + "enum": [v["name"] for v in e.get("values", [])], + "x-meos-c-enum": True, + } + return {"type": "string", "title": name} + + +def build_openapi(catalog: dict, *, title: str = "MEOS API", + version: str = "0.1.0") -> dict: + """Build an OpenAPI 3.1 document from an enriched catalog.""" + functions = sorted( + (f for f in catalog.get("functions", []) + if f.get("network", {}).get("exposable")), + key=lambda f: f["name"], + ) + type_encodings = catalog.get("typeEncodings", {}) + enums = catalog.get("enums", []) + + used_types: set = set() + used_enums: set = set() + paths: dict = {} + tags_seen: set = set() + + for fn in functions: + paths[f"/{fn['name']}"] = { + "post": _operation(fn, used_types, used_enums) + } + tags_seen.add(fn["category"]) + + schemas = {} + for t in sorted(used_types): + schemas[t] = _type_schema(t, type_encodings) + for e in sorted(used_enums): + schemas[e] = _enum_schema(e, enums) + + total = len(catalog.get("functions", [])) + return { + "openapi": "3.1.0", + "info": { + "title": title, + "version": version, + "description": ( + "Auto-generated from the MEOS-API catalog. Each operation is " + "a stateless-exposable MEOS function projected RPC-style as " + "`POST /{function}`; opaque values cross the wire as strings " + "in the encodings listed on their component schema. " + "Generated, do not edit by hand." + ), + "x-meos-coverage": { + "functions": total, + "exposed": len(functions), + }, + }, + "tags": [{"name": t} for t in sorted(tags_seen)], + "paths": dict(sorted(paths.items())), + "components": { + "schemas": schemas, + "responses": { + "MeosError": { + "description": "MEOS error", + "content": {"application/json": {"schema": { + "type": "object", + "properties": { + "error": {"type": "string"}, + "code": {"type": "integer"}, + }, + "required": ["error"], + }}}, + } + }, + }, + } diff --git a/meta/integration-train.json b/meta/integration-train.json new file mode 100644 index 0000000..0fb8413 --- /dev/null +++ b/meta/integration-train.json @@ -0,0 +1,194 @@ +{ + "schema": "integration-train/0.1.0-draft", + "purpose": "Dependency-ordered merge train + per-wave verification gates that make the ecosystem-wide 100% parity PROVABLE at one point, instead of asserted from per-PR isolation-green. Operationalizes MobilityDB discussion #895 (wave-based merge plan).", + "anchor": "The MEOS-API meos-idl.json regenerated from the integrated MobilityDB surface is the parity proof artifact: every binding derives from it.", + "invariant": "100% parity is DEMONSTRABLE only when the full train is composed and every wave gate passes together. Per-PR isolation-green is necessary, not sufficient.", + "merge_gate": "All merges are maintainer-only (no self-merge by any session). This train reduces that gate to a mechanical action: merge in wave order; CI turns green at each wave.", + "waves": [ + { + "id": 0, + "name": "MobilityDB core - extended-type C surface", + "repo": "MobilityDB/MobilityDB", + "status": "PROVEN", + "verified": "2026-05-19, _mobilitydb branch complete-extype-surface (this repo)", + "compose": { + "base": "master", + "linear_stack_tip": {"pr": 1085, "ref": "pull/1085/head", "head": "ea8f357", "contains": "master->#1081(66d8afa)->#1082(cf9ed91)->#1083(131eccb)->#1084(d60e63c)->#1085(ea8f357)"}, + "then_pair": {"refs": ["pull/1051/head (#1051 e624027)", "pull/951/head (#951 19e3e35)"], "note": "clean linear pair #1051->#951; cherry-picks clean onto the stack tip (proven). #951 includes the typed tnpoint_from_mfjson uniformity wrapper."}, + "docs_only_low_priority": {"pr": 953, "branch": "split/tnpoint-production-guidance"} + }, + "gate": { + "meos_api_catalog_functions": 2699, + "meos_api_pr10_tests": "21/21", + "from_mfjson_uniform": ["tcbuffer_from_mfjson->TCbuffer", "tnpoint_from_mfjson->TNpoint", "tpose_from_mfjson->TPose"], + "concrete_constructors": ["tcbufferinst_make->TCbufferInst", "tposeinst_make->TPoseInst"] + }, + "merge_order": "stack bottom-up #1081->#1082->#1083->#1084->#1085, then #1051->#951. Do NOT re-flatten to independent branches (the C-library session already rebased these into clean linear sequences).", + "known_conflicts_resolved": ["#1083 vs #1084 (rebased into the linear stack)", "#1051 vs #951 (rebased into the #1051->#951 pair)"] + }, + { + "id": 1, + "name": "Substrate - PyMEOS-CFFI MEOS-1.4", + "repo": "MobilityDB/PyMEOS-CFFI", + "status": "IN_FLIGHT", + "prs": [{"pr": 19, "branch": "bump/meos-1.4"}, {"pr": 18, "branch": "refactor/codegen-meos-idl", "draft": true}], + "action": "regenerate builder/meos-idl.json against the Wave-0 MobilityDB surface", + "gate": "pymeos_cffi builds + imports against a Wave-0 MEOS" + }, + { + "id": 2, + "name": "CRITICAL PATH - PyMEOS master health (MEOS-1.4 bump)", + "repo": "MobilityDB/PyMEOS", + "status": "BLOCKING_ALL", + "prs": [{"pr": 81, "branch": "bump/meos-1.4", "draft": true}, {"pr": 82, "branch": "bump/meos-1.3"}], + "problem": "PyMEOS master is broken vs MEOS master: rename skew geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity. Pristine PyMEOS master = 426 failed / 2721 passed; red on EVERY PyMEOS PR's Test-PyMEOS check.", + "gate": "PyMEOS test.yml collects and 426->0 failures, built against Wave-1 CFFI / Wave-0 MEOS", + "note": "Build the bump against the integrated Wave-0 (not bare master) so it is done once, correctly, against the final C surface." + }, + { + "id": 3, + "name": "PyMEOS features (downstream of Wave 2)", + "repo": "MobilityDB/PyMEOS", + "status": "GREEN_IN_ISOLATION", + "prs": [ + {"pr": 85, "branch": "fix/black-datespan", "head": "bd7b426", "gate": "black --check . passes whole-repo", "status": "READY (verified green-on-gate 2026-05-19; only its inherited Test-PyMEOS red is Wave-2)"}, + {"pr": 87, "branch": "feat/portable-aliases", "gate": "portable_parity.py: 0 unbacked, 29 pairs, 6 families"}, + {"pr": 88, "branch": "feat/extended-temporal-types", "gate": "suites collect; NotImplementedError stubs -> real once the Wave-0 surface is in the MEOS PyMEOS-CI builds", "stack_base": true}, + {"pr": 89, "branch": "refactor/oo-codegen-meos-idl", "stacked_on": 88, "gate": "codegen.py --check (MEOS-API meos-idl.json SoT; proven zero-regression 407==407)"}, + {"pr": 90, "branch": "refactor/oo-codegen-tcbuffer-switch", "stacked_on": 89, "gate": "tcbuffer_test 26p/1f; generated mixin backing-identical to oracle"}, + {"pr": 91, "branch": "refactor/oo-codegen-tpose-switch", "stacked_on": 90}, + {"pr": 84, "branch": "feat/datalake-consumer", "head": "9a6f8b9", "gate": "tests/io 4/4; temporal footer byte-identical to MobilityDuck #146", "independent": true, "status": "code verified correct + interop confirmed 2026-05-19"} + ], + "interop_dependency": {"repo": "MobilityDB/MobilityDuck", "prs": [{"pr": 146, "branch": "lake/temporal-footer"}, {"pr": 147, "branch": "lake/edge-to-cloud-quickstart"}], "for": "#84 TemporalParquet second-consumer interop demonstration"} + }, + { + "id": 4, + "name": "Downstream bindings (consume Wave-0 C surface + Wave-1/2/3 anchor)", + "status": "GREEN_IN_ISOLATION", + "purpose": "Every binding regenerates against the Wave-0 MEOS-1.4 C surface via the MEOS-API meos-idl.json catalog. Each binding is bump-independent within its repo but shares the same Wave-0 substrate.", + "bindings": [ + { + "repo": "MobilityDB/MobilityDuck", + "anchor_prs": [ + {"pr": 164, "branch": "chore/meos-bump-1.4-integration", "role": "bump pinned MEOS to the 1.4-integration SHA"}, + {"pr": 161, "branch": "fix/tydef-meos-type-alias", "role": "drop premature MeosType alias (unblocks Linux arm64 build)"} + ], + "feature_prs": [ + {"pr": 168, "branch": "doc/geography-boundary-design"}, + {"pr": 169, "branch": "feat/register-geography-logicaltype"}, + {"pr": 174, "branch": "feat/geography-io-udfs", "stacked_on": 169}, + {"pr": 175, "branch": "feat/geography-casts", "stacked_on": 169}, + {"pr": 176, "branch": "feat/geography-operations", "stacked_on": 169}, + {"pr": 177, "branch": "feat/geography-test-matrix", "stacked_on": 169}, + {"pr": 178, "branch": "feat/parity-th3index", "stacked_on": 130, "supersedes": 129} + ], + "ci_plumbing": [ + {"pr": 170, "branch": "ci/exclude-osx-arm64-pending-hex-wkb", "role": "exclude osx_arm64 until hex-WKB fix"}, + {"pr": 171, "branch": "ci/probe-mingw-build", "role": "probe MinGW build viability"}, + {"pr": 172, "branch": "ci/probe-musl-build", "role": "probe musl-libc build viability"}, + {"pr": 173, "branch": "fix/span-table-functions-unique-ptr", "role": "DuckDB 1.4.4 unique_ptr cast fix"} + ], + "open_count_2026_05_21": 47 + }, + { + "repo": "MobilityDB/MobilitySpark", + "anchor_prs": [ + {"pr": 5, "branch": "feat/jmeos-1.4-berlinmod", "role": "JMEOS 1.4 + BerlinMOD Q1-Q17 + 100% SQL parity (907 tests)"}, + {"pr": 11, "branch": "feat/portable-operator-bare-names", "role": "the 29 canonical bare-name operator UDFs"}, + {"pr": 12, "branch": "feat/sibling-families-parity", "role": "cbuffer/npoint/pose/rgeo UDF surface (92.5% -> 99.6%)"} + ], + "deferred": [ + {"pr": 9, "branch": "perf/spark-mt-and-binary", "role": "th3index spatial prefilter", "reason": "86-commit / 36k LoC drift; needs decomposition into smaller scope-coherent PRs"} + ], + "open_count_2026_05_21": 10 + }, + { + "repo": "MobilityDB/MobilityAPI", + "prs": [ + {"pr": 3, "branch": "docs/ingestion-plan", "role": "MEOS-API ingestion plan + OpenAPI missing-work integration"}, + {"pr": 4, "branch": "chore/vendor-meos-api", "role": "vendor MEOS-API artefacts (step 2)"}, + {"pr": 5, "branch": "ci/vendor-drift", "role": "vendor-drift CI workflow (step 3)"}, + {"pr": 6, "branch": "feat/step4-dispatcher-foundation", "role": "catalog-driven MEOS-function dispatcher (step 4 foundation)"}, + {"pr": 7, "branch": "feat/step4-resolvers-and-wire", "role": "pluggable resolvers + wire-layer codec (step 4)"}, + {"pr": 8, "branch": "feat/step5-fastapi-endpoints", "stacked_on": 7, "role": "FastAPI routes for Dispatcher + WireCodec (step 5)"} + ], + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/JMEOS", + "anchor_prs": [ + {"pr": 13, "branch": "doc/reviewer-guide", "role": "PR Reviewer Guide + visibility wiring"} + ], + "conflicting": [ + {"pr": 8, "branch": "feat/jmeos-1.3", "reason": "main moved past PR #9 merge (multi-module restructure)"}, + {"pr": 12, "branch": "fix/multimodule-with-split-interface", "reason": "100-file structural restructure overlaps with merged PR #9"}, + {"pr": 15, "branch": "feat/regen-against-meos-1.4", "reason": "structural drift after PR #9 multi-module merge"}, + {"pr": 16, "branch": "feat/expose-mindistance", "reason": "structural drift"}, + {"pr": 17, "branch": "fix/split-meoslibrary-4way", "reason": "structural drift; 4-file split-interface refactor pre-dates the multi-module migration"} + ], + "downstream_consumers": ["MobilityDB/MobilityFlink#3", "MobilityDB/MobilityKafka#1"], + "downstream_surface_need": "8 raw FFI decls (edwithin/eintersects/edisjoint/nad/tdistance variants) + 2 high-level wrappers (Haversine.distance, PointToSegment.distance) — captured in [[jmeos-1.4-stream-consumer-surface]]", + "open_count_2026_05_21": 6 + }, + { + "repo": "MobilityDB/GoMEOS", + "anchor_prs": [ + {"pr": 3, "branch": "bump/meos-1.4"}, + {"pr": 4, "branch": "feat/portable-aliases", "stacked_on": 3, "role": "29 canonical bare names; 6/6 families covered"} + ], + "open_count_2026_05_21": 4 + }, + { + "repo": "MobilityDB/MEOS.NET", + "anchor_prs": [ + {"pr": 1, "branch": "initial-codegen", "role": ".NET binding generated from meos-idl.json; 29 bare names"} + ], + "open_count_2026_05_21": 3 + } + ], + "gate": "Each binding's per-PR CI matches the parity audit; the meos-idl.json catalog regenerates byte-identically across Wave-4 bindings (proves single SoT).", + "current_status": "All bindings green-in-isolation against Wave-0; CONFL pressure concentrated on JMEOS (multi-module migration in flight)." + }, + { + "id": 5, + "name": "Service-agent + data-lake layers (built on Wave-4 anchor)", + "status": "ANCHOR_DEFINED", + "purpose": "Top of the 3-layer ecosystem landscape: MobilityAPI as the service-agent HTTP surface; PyMEOS #84 + MobilityDuck #146/#147 as the data-lake substrate (TemporalParquet+Iceberg+Polaris). Wave 5 starts gating once Waves 0-4 reach master.", + "service_agent_layer": { + "repo": "MobilityDB/MEOS-API", + "carriers": [ + {"pr": 4, "role": "service-projection metadata derivation"}, + {"pr": 5, "role": "OpenAPI 3.1 contract generation"}, + {"pr": 6, "role": "MCP tool manifest generation"}, + {"pr": 7, "role": "contract-driven runtime HTTP server"}, + {"pr": 12, "role": "openapi-validate workflow"}, + {"pr": 13, "role": "OGC API – Moving Features OpenAPI projection"} + ], + "consumer": {"repo": "MobilityDB/MobilityAPI", "wave": 4, "pr": 8, "role": "FastAPI app embodying the contract"} + }, + "data_lake_layer": { + "first_consumer": {"repo": "MobilityDB/MobilityDuck", "prs": [146, 147, 158], "role": "TemporalParquet+Iceberg substrate, edge-to-cloud quickstart"}, + "second_consumer": {"repo": "MobilityDB/PyMEOS", "pr": 84, "role": "pymeos.io {to_arrow, from_arrow, write_temporal, read_temporal, temporal_footer}"}, + "polaris_readiness": "Apache Polaris recommended as the OSS Iceberg-REST catalog (per [[apache-polaris-readiness]]); deployment doc as docs/iceberg-polaris.md after substrate lands." + }, + "stream_layer": { + "status": "PLANNED_BAND", + "platforms": [ + {"repo": "MobilityDB/MobilityFlink", "live_pr": 3, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityKafka", "live_pr": 1, "ffi_consumer": "JMEOS"}, + {"repo": "MobilityDB/MobilityNebula", "live_prs": [13, 14], "ffi_consumer": "direct libmeos (no JVM)"} + ], + "parity_contract": "BerlinMOD-9 × 3 streaming forms (continuous / windowed / snapshot) × 3 platforms; snapshot-form ≡ batch BerlinMOD-Q at watermark T. See [[stream-readiness]]." + }, + "gate": "End-to-end OGC API – Moving Features client + TemporalParquet+Iceberg trip works against MEOS-API #7 backed by the Wave-0 surface; stream-layer BerlinMOD-9 × 3 forms reaches parity table on all 3 platforms." + } + ], + "catalog_anchor": { + "repo": "MobilityDB/MEOS-API", + "prs": [ + {"pr": 10, "branch": "feat/object-model", "status": "verified 21/21 vs Wave-0", "role": "object model (class lattice/methods/error contract)"}, + {"pr": 8, "branch": "feat/portable-aliases", "role": "portable bare-name 29-pair SoT"} + ] + }, + "current_status": "Wave 0 PROVEN. Waves 1-3 are entirely gated on Wave 2 (the MEOS-1.4 bump) - the single universal unblock. Wave 4 (downstream bindings) green-in-isolation across MobilityDuck (47 open mergeable), MobilitySpark (10), MobilityAPI (6), GoMEOS (4), MEOS.NET (3); JMEOS (6 open, 5 CONFL) is the lone repo under structural-migration pressure. Wave 5 (service-agent + data-lake + stream layers) anchor-defined and downstream of Wave 4. No remaining correctness work; the gap is integration + maintainer-merge ordering." +} diff --git a/parser/doxygroup.py b/parser/doxygroup.py new file mode 100644 index 0000000..b809d13 --- /dev/null +++ b/parser/doxygroup.py @@ -0,0 +1,44 @@ +"""Attach the doxygen module group (@ingroup) to the MEOS-API catalog. + +Every MEOS-C function carries an `@ingroup meos_` doxygen tag in its +source comment block (meos/src). Those groups ARE the structure of the MEOS +reference manual / doxygen XML — e.g. `meos_setspan_accessor`, +`meos_temporal_comp_ever`, `meos_geo_constructor`. Carrying the group into the +catalog lets every binding organize its generated surface the SAME way the +manual does, so a function is found in the same place across all tools. + +Adds per function (when found): `group`. The `meos_internal_*` groups are +MEOS-internal, not user-facing — they are tagged like any other so a binding +can filter them out, but they are NOT a separate concept here. +""" +import re +from pathlib import Path + +_INGROUP = re.compile(r"@ingroup\s+(meos_\w+)") +# Same shape as sqlfn._FNDEF: after the doxygen close, an optional return-type +# line (no parens/braces/;/=), then `name(`. +_FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") + + +def _name_to_group(meos_src): + """MEOS-C function name -> doxygen @ingroup group (first occurrence wins).""" + out = {} + for cf in Path(meos_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _INGROUP.finditer(text): + grp = m.group(1) + fm = _FNDEF.search(text, m.end()) + if fm: + out.setdefault(fm.group(1), grp) + return out + + +def attach_groups(idl, meos_src): + n2g = _name_to_group(meos_src) + n = 0 + for f in idl["functions"]: + g = n2g.get(f["name"]) + if g: + f["group"] = g + n += 1 + return idl, n diff --git a/parser/enrich.py b/parser/enrich.py new file mode 100644 index 0000000..e699e85 --- /dev/null +++ b/parser/enrich.py @@ -0,0 +1,549 @@ +"""Service-projection enrichment. + +Derives, for every function and type in the parsed catalog, the metadata a +service generator (OpenAPI, MCP, gRPC, ...) needs but that *cannot* be read +from C headers: + +- ``category`` — a coarse semantic class (constructor, predicate, io, ...). +- ``typeEncodings``— for each opaque C type, how it round-trips to the wire + (text / MF-JSON / WKB) and the function names that do it. +- ``network`` — whether the function can be projected onto a *stateless* + endpoint, and if not, why. +- ``wire`` — per-parameter and return value, the concrete request / + response representation a generator should emit. + +Everything here is a heuristic *default*. The pass runs before the manual +merge step, so any field can be overridden per function/type from +``meta/meos-meta.json`` (the merger applies on top). This module is +deliberately free of any libclang dependency: it operates purely on the +parsed ``idl`` dict and is therefore unit-testable on its own. + +Note: libclang emits *canonical* C spellings — ``struct Temporal *`` (not +``Temporal *``), ``unsigned char`` (not ``uint8_t``), ``long`` (not +``int64_t``), and MEOS uses ``int`` for booleans. The heuristics below match +those canonical spellings. See ``docs/enrichment.md`` for the full contract. +""" + +import re + +# Ordered category vocabulary. The first matching rule wins. +CATEGORIES = ( + "lifecycle", # library/process setup, configuration, teardown + "index", # in-memory index objects (RTree, ...) + "io", # parse/serialize between a value and a wire encoding + "aggregate", # aggregate transition/combine/final functions + "predicate", # boolean question about value(s) + "constructor", # build a value (_make, _copy) + "setop", # set algebra (union/intersection/minus/...) + "conversion", # convert one value into another representation + "accessor", # read a component/property of a value + "transformation", # value -> value of the same family + "other", # anything not classified above +) + +# Canonical scalar spellings as emitted by libclang. +_INT_BASES = { + "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", + "long", "unsigned long", "long long", "unsigned long long", +} +_FLOAT_BASES = {"float", "double", "long double"} +_BOOL_BASES = {"bool", "_Bool"} +# char-like pointers carry text or bytes (WKB) — represented as a JSON string. +_STRING_PTR_BASES = {"char", "signed char", "unsigned char"} + +# name suffix -> wire encoding produced (encoder) / consumed (decoder) +_ENCODERS = [ + (re.compile(r"_out$"), "text"), + (re.compile(r"_as_text$"), "text"), + (re.compile(r"_as_e?wkt$"), "text"), + (re.compile(r"_as_mfjson$"), "mfjson"), + (re.compile(r"_as_geojson$"), "mfjson"), + (re.compile(r"_as_hex_?wkb$"), "wkb"), + (re.compile(r"_as_e?wkb$"), "wkb"), +] +_DECODERS = [ + (re.compile(r"_in$"), "text"), + (re.compile(r"_from_e?wkt$"), "text"), + (re.compile(r"_from_text$"), "text"), + (re.compile(r"_from_mfjson$"), "mfjson"), + (re.compile(r"_from_geojson$"), "mfjson"), + (re.compile(r"_from_hex_?wkb$"), "wkb"), + (re.compile(r"_from_e?wkb$"), "wkb"), +] +_IO_RE = [rx for rx, _ in _DECODERS + _ENCODERS] + +_LIFECYCLE_RE = re.compile(r"^meos_") +_INDEX_RE = re.compile(r"^rtree_") +_AGG_RE = re.compile(r"(_transfn|_combinefn|_finalfn)$|_tagg|_collect$") +_CONSTRUCTOR_RE = re.compile(r"(_make|_copy)$") +_SETOP_RE = re.compile(r"^(union|intersection|minus|difference)_") +_CONVERSION_RE = re.compile(r"_to_|_from_base|_from_|_as_") +_ACCESSOR_RE = re.compile( + r"(_values?|_start_value|_end_value|_min_value|_max_value|_srid|_timespan|" + r"_duration|_length|_num_[a-z]+|_n|_lower|_upper|_start_[a-z]+|_end_[a-z]+|" + r"_get_[a-z]+|_value_n)$" +) +# MEOS predicates return `int`, so they must be recognised by name. +_PREDICATE_RE = re.compile( + r"^(ever|always)_|(_eq|_ne|_lt|_le|_gt|_ge|_cmp)$|" + r"^(contains|contained|overlaps|overbefore|overafter|overleft|overright|" + r"overbelow|overabove|overfront|overback|left|right|below|above|front|" + r"back|before|after|adjacent|same|intersects|disjoint|touches|dwithin|" + r"covers|coveredby|equals|crosses|within|relate)_|" + r"^t(eq|ne|lt|le|gt|ge|contains|intersects|disjoint|touches|dwithin)_" +) + +_QUAL_RE = re.compile(r"\b(const|volatile|struct|union|enum)\b") + + +def _base(c_type: str) -> str: + """Bare type token: qualifiers, ``struct``/``union``/``enum`` and ``*`` + stripped (so ``const struct Temporal *`` -> ``Temporal``).""" + return " ".join(_QUAL_RE.sub(" ", c_type).replace("*", " ").split()) + + +def _ptr_depth(c_type: str) -> int: + return c_type.count("*") + + +def _scalar_wire(c_type: str, enums: set): + """Wire descriptor for a non-opaque scalar/string/enum, else ``None``.""" + base, depth = _base(c_type), _ptr_depth(c_type) + if base == "void" and depth == 0: + return {"kind": "void"} + # char-likes and PostgreSQL `text` are just strings on the wire. + if depth == 1 and (base in _STRING_PTR_BASES or base == "text"): + return {"kind": "json", "json": "string"} + if depth == 0: + if base in _BOOL_BASES: + return {"kind": "json", "json": "boolean"} + if base in _FLOAT_BASES: + return {"kind": "json", "json": "number"} + if base in _INT_BASES: + return {"kind": "json", "json": "integer"} + if base in enums: + return {"kind": "json", "json": "string", "enum": base} + return None + + +def _is_scalar_pointer(c_type: str, enums: set) -> bool: + """``int *`` / ``double *`` / ``interpType *`` etc. — an array/out-param, + as opposed to a pointer to an opaque value (``struct Temporal *``).""" + base, depth = _base(c_type), _ptr_depth(c_type) + if depth >= 2: + return True + if depth == 1 and base not in _STRING_PTR_BASES: + return (base in _INT_BASES or base in _FLOAT_BASES + or base in _BOOL_BASES or base in enums) + return False + + +def _aux_specs(params: list): + """Defaults for the trailing args of an in/out helper. + + Real MEOS in/out helpers are not pure ``(str)->T`` / ``(T)->str``: they + take trailing *formatting* scalars — ``temporal_out(temp, int maxdd)``, + ``*_as_mfjson(temp, with_bbox, flags, precision, srs)``. Those are safe + to default, so the helper still satisfies the stateless contract. + + Returns the aux spec list (one ``{name, kind, default}`` per trailing + param), or ``None`` if any trailing parameter is *not* a defaultable + formatting scalar — a semantic ``*type`` tag (``temporal_in``'s + ``temptype``), a pointer/array (``*_as_wkb``'s ``size_out``), etc. — + which disqualifies the helper entirely. + """ + specs = [] + for p in params: + sc = _scalar_wire(p["canonical"], set()) + if sc is None or sc.get("kind") != "json": + return None # pointer / array / opaque aux + nm = p["name"].lower() + if "type" in nm: # temptype/basetype/settype tag + return None + j = sc["json"] + if j == "integer": + default = (15 if any(k in nm for k in + ("maxdd", "decimal", "digit", "precision")) + else 0) + elif j == "number": + default = 0.0 + elif j == "boolean": + default = False + else: # string (e.g. srs) -> NULL + default = None + specs.append({"name": p["name"], "kind": j, "default": default}) + return specs + + +def build_type_encodings(functions: list, structs: set) -> dict: + """Scan the catalog for the in/out functions of every opaque struct. + + A *decoder* turns a wire string into an object (returns ``struct T *``, + first arg a char-like string). An *encoder* turns an object into a wire + string (returns ``char *``, first arg ``const struct T *``). Trailing + *formatting* scalars are allowed and defaulted (see ``_aux_specs``); a + non-defaultable trailing arg disqualifies the helper. Only declared + structs qualify, so primitives never register by accident. + """ + enc: dict[str, dict] = {} + + def slot(b: str) -> dict: + return enc.setdefault(b, {"encodings": set(), "encoders": {}, + "decoders": {}}) + + for fn in functions: + name = fn["name"] + ret = fn["returnType"]["canonical"] + params = fn.get("params", []) + if not params: + continue + p0 = params[0]["canonical"] + rb, rd = _base(ret), _ptr_depth(ret) + pb, pd = _base(p0), _ptr_depth(p0) + aux = _aux_specs(params[1:]) # None => non-defaultable trailing arg + + # Decoder: const char* (+ defaultable scalar aux) -> opaque struct + if (aux is not None and rd >= 1 and rb in structs + and pd == 1 and pb in _STRING_PTR_BASES): + for rx, encoding in _DECODERS: + if rx.search(name): + s = slot(rb) + s["encodings"].add(encoding) + s["decoders"].setdefault(encoding, {})[name] = aux + break + + # Encoder: const struct T* (+ defaultable scalar aux) -> char* + if (aux is not None and rd == 1 and rb in _STRING_PTR_BASES + and pd >= 1 and pb in structs): + for rx, encoding in _ENCODERS: + if rx.search(name): + s = slot(pb) + s["encodings"].add(encoding) + s["encoders"].setdefault(encoding, {})[name] = aux + break + + # Several typed functions can serve one encoding (e.g. tbool_in, + # tint_in, ... all decode a `Temporal *`). Prefer the *generic root* + # (`_in` / `_out`) so the chosen in/out works for every subtype + # (and `temporal_out` correctly serialises any subtype); fall back to a + # deterministic alphabetical pick otherwise. + order = ("text", "mfjson", "wkb") + dec_suffix = {"text": "_in", "mfjson": "_from_mfjson", + "wkb": "_from_hexwkb"} + enc_suffix = {"text": "_out", "mfjson": "_as_mfjson", + "wkb": "_as_hexwkb"} + + def choose(cands: dict, base: str, suffix: str) -> str: + generic = base.lower() + suffix + return generic if generic in cands else sorted(cands)[0] + + out: dict[str, dict] = {} + for base, s in enc.items(): + dec = {e: choose(c, base, dec_suffix[e]) + for e, c in s["decoders"].items()} + encd = {e: choose(c, base, enc_suffix[e]) + for e, c in s["encoders"].items()} + in_e = next((e for e in order if e in dec), None) + out_e = next((e for e in order if e in encd), None) + out[base] = { + "encodings": sorted(s["encodings"]), + "decoders": dec, + "encoders": encd, + "in": dec.get(in_e) if in_e else None, + "out": encd.get(out_e) if out_e else None, + "in_aux": s["decoders"][in_e][dec[in_e]] if in_e else [], + "out_aux": s["encoders"][out_e][encd[out_e]] if out_e else [], + } + return out + + +def classify_category(fn: dict) -> str: + name = fn["name"] + ret = fn["returnType"]["canonical"] + + if _LIFECYCLE_RE.match(name): + return "lifecycle" + if _INDEX_RE.match(name): + return "index" + if any(rx.search(name) for rx in _IO_RE): + return "io" + if _AGG_RE.search(name): + return "aggregate" + if _PREDICATE_RE.search(name) or _base(ret) in _BOOL_BASES: + return "predicate" + if _CONSTRUCTOR_RE.search(name): + return "constructor" + if _SETOP_RE.match(name): + return "setop" + if _CONVERSION_RE.search(name): + return "conversion" + if _ACCESSOR_RE.search(name): + return "accessor" + return "transformation" + + +# MEOS splits its API: the public *user* surface (meos.h + the public type +# headers) and the *internal* programmer surface. The latter is type-erased +# (``Datum``-generic), undocumented for end users, and must not be projected +# onto a network service — it is policy-excluded, like lifecycle/index. +_INTERNAL_FILES = {"meos_internal.h", "meos_internal_geo.h"} + + +def _outparam(fn: dict, enums: set, type_encodings: dict): + """A MEOS accessor of the form ``bool f(.., T *result)`` returns its + value through a trailing out-parameter, with the ``bool``/``int`` return + (or ``void``) as a presence flag. Two safe shapes: + + - ``scalar``: ``T *result`` where ``T`` is a JSON scalar. + - ``opaque``: ``T **result`` where ``T`` has an encoder (serialised). + + Returns ``(leading_params, outparam, mode)`` or ``(None, None, None)``. + """ + ps = fn.get("params", []) + ret = fn["returnType"]["canonical"] + if (not ps or _ptr_depth(ret) != 0 + or _base(ret) not in ("bool", "_Bool", "int", "void")): + return None, None, None + last = ps[-1] + if last["name"] not in ("result", "value"): + return None, None, None + c = last["canonical"] + if _ptr_depth(c) == 1: + pointee = c.replace("const", "").replace("*", "").strip() + sw = _scalar_wire(pointee, enums) + if sw is not None and sw.get("kind") == "json": + return ps[:-1], last, "scalar" + elif _ptr_depth(c) == 2: + te = type_encodings.get(_base(c)) + if te and te.get("out"): + return ps[:-1], last, "opaque" + return None, None, None + + +def _array_param(params: list, type_encodings: dict): + """A MEOS builder takes an element array as a ``(Elem **arr, int count)`` + pair. Detect the first such pair whose element type is decodable, so the + array can be projected as a JSON list (the ``count`` is then implicit). + Returns ``(arr_index, count_index)`` or ``(None, None)``. + """ + for i, p in enumerate(params): + c = p["canonical"] + if (_ptr_depth(c) == 2 and p["name"] not in ("result", "value") + and i + 1 < len(params)): + te = type_encodings.get(_base(c)) + nxt = params[i + 1]["canonical"] + if (te and te.get("in") and _base(nxt) == "int" + and _ptr_depth(nxt) == 0): + return i, i + 1 + return None, None + + +def _array_return(fn: dict, type_encodings: dict): + """A MEOS accessor that returns a freshly-allocated element array as + ``Elem **f(.., int *count)`` (e.g. ``temporal_sequences``). The element + type must be encodable. Returns ``(count_param_name, elem_base)`` or + ``(None, None)``. + """ + ret = fn["returnType"]["canonical"] + if _ptr_depth(ret) != 2: + return None, None + rb = _base(ret) + te = type_encodings.get(rb) + if not (te and te.get("out")): + return None, None + cnt = [p for p in fn.get("params", []) + if _ptr_depth(p["canonical"]) == 1 + and _base(p["canonical"]) in ("int", "long") + and p["name"] in ("count", "n", "nvalues", "size", "npoints")] + if len(cnt) != 1: + return None, None + return cnt[0]["name"], rb + + +def assess(fn: dict, type_encodings: dict, enums: set) -> tuple: + """Return ``(network, wire)`` for one function. + + Exposable over a stateless endpoint iff every parameter can be decoded + from the request and the return can be encoded into the response. + Pointer-to-scalar parameters (arrays / out-params) and opaque types + lacking an in/out function make it non-exposable; the reason is recorded. + """ + reasons: list[str] = [] + wire_params = [] + + out_lead, out_p, out_mode = _outparam(fn, enums, type_encodings) + eff = out_lead if out_p is not None else fn.get("params", []) + arr_i, count_i = _array_param(eff, type_encodings) + ret_count_name, ret_elem = _array_return(fn, type_encodings) + ret_count_i = (next((i for i, p in enumerate(eff) + if p["name"] == ret_count_name), None) + if ret_elem is not None else None) + + for idx, p in enumerate(eff): + if idx == count_i or idx == ret_count_i: + continue # array length is implicit + c = p["canonical"] + if idx == arr_i: + elem = type_encodings[_base(c)] + wire_params.append({ + "name": p["name"], "kind": "array", + "count_param": eff[count_i]["name"], + "element": { + "kind": "serialized", + "cType": " ".join(c.replace("*", " ").split()) + " *", + "decode": elem["in"], + "decode_aux": elem.get("in_aux", []), + "encodings": elem["encodings"], + }, + }) + continue + scalar = _scalar_wire(c, enums) + if scalar is not None: + wire_params.append({"name": p["name"], **scalar}) + continue + if _is_scalar_pointer(c, enums): + reasons.append(f"array-or-out-param:{p['name']}") + wire_params.append({"name": p["name"], "kind": "unsupported"}) + continue + base = _base(c) + te = type_encodings.get(base) + if te and te["in"]: + wire_params.append({ + "name": p["name"], "kind": "serialized", "cType": c, + "decode": te["in"], "decode_aux": te.get("in_aux", []), + "encodings": te["encodings"], + }) + else: + reasons.append(f"no-decoder:{base}") + wire_params.append({"name": p["name"], "kind": "unsupported"}) + + ret = fn["returnType"]["canonical"] + if out_p is not None: + # a bool/int C return is a presence flag; void = always present + presence = _base(ret) in ("bool", "_Bool", "int") + if out_mode == "scalar": + pointee = out_p["canonical"].replace("const", "").replace( + "*", "").strip() + wire_result = { + **_scalar_wire(pointee, enums), # kind:"json", json:… + "from_outparam": out_p["name"], + "out_ctype": out_p["canonical"], + "presence_return": presence, + } + else: # opaque T **result + te = type_encodings[_base(out_p["canonical"])] + wire_result = { + "kind": "serialized", + "cType": out_p["canonical"], + "encode": te["out"], "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"], + "from_outparam": out_p["name"], + "out_ctype": out_p["canonical"], + "presence_return": presence, + } + scalar = "handled" + elif ret_elem is not None: + te = type_encodings[ret_elem] + wire_result = { + "kind": "array", + "element": { + "kind": "serialized", + "cType": " ".join(ret.replace("*", " ").split()) + " *", + "encode": te["out"], "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"], + }, + "count_outparam": ret_count_name, + } + scalar = "handled" + else: + scalar = _scalar_wire(ret, enums) + if scalar == "handled": + pass + elif scalar is not None: + wire_result = scalar + elif _is_scalar_pointer(ret, enums): + reasons.append(f"unsupported-return:{ret}") + wire_result = {"kind": "unsupported"} + else: + base, depth = _base(ret), _ptr_depth(ret) + te = type_encodings.get(base) + if depth == 1 and te and te["out"]: + wire_result = {"kind": "serialized", "cType": ret, + "encode": te["out"], + "encode_aux": te.get("out_aux", []), + "encodings": te["encodings"]} + else: + reasons.append( + f"no-encoder:{base}" if depth == 1 + else f"unsupported-return:{ret}" + ) + wire_result = {"kind": "unsupported"} + + if fn["category"] in ("lifecycle", "index"): + reasons.insert(0, fn["category"]) + if fn.get("api") == "internal": + reasons.insert(0, "internal") + + exposable = not reasons + network = { + "exposable": exposable, + "method": "POST" if exposable else None, + "reason": None if exposable else "; ".join(dict.fromkeys(reasons)), + } + return network, {"params": wire_params, "result": wire_result} + + +def enrich_idl(idl: dict) -> dict: + """Augment ``idl`` in place with service-projection metadata.""" + functions = idl.get("functions", []) + struct_names = {s["name"] for s in idl.get("structs", [])} + enum_names = {e["name"] for e in idl.get("enums", [])} + + # An opaque type is any *named* pointer type that is not a scalar / enum + # / string. Beyond parsed structs this also covers reconciled + # PostgreSQL/PostGIS types (`Interval`, `GBOX`, ...) so their own in/out + # wrappers can register a codec instead of being dead `no-decoder`s. + _scalarish = (_INT_BASES | _FLOAT_BASES | _BOOL_BASES | _STRING_PTR_BASES + | enum_names | {"void", "text"}) + opaque_names = set(struct_names) + for fn in functions: + for c in ([fn["returnType"]["canonical"]] + + [p["canonical"] for p in fn.get("params", [])]): + b = _base(c) + if _ptr_depth(c) >= 1 and b and b not in _scalarish: + opaque_names.add(b) + + type_encodings = build_type_encodings(functions, opaque_names) + + for fn in functions: + fn["api"] = ("internal" if fn.get("file") in _INTERNAL_FILES + else "public") + fn["category"] = classify_category(fn) + network, wire = assess(fn, type_encodings, enum_names) + fn["network"] = network + fn["wire"] = wire + + for struct in idl.get("structs", []): + te = type_encodings.get(struct["name"]) + if te: + struct["serialization"] = { + "encodings": te["encodings"], "in": te["in"], "out": te["out"], + } + idl["typeEncodings"] = type_encodings + + counts: dict[str, int] = {} + for fn in functions: + counts[fn["category"]] = counts.get(fn["category"], 0) + 1 + public = [fn for fn in functions if fn["api"] == "public"] + idl["enrichment"] = { + "categoryCounts": counts, + "publicFunctions": len(public), + "internalFunctions": len(functions) - len(public), + # Internal functions are policy-excluded, so this equals the + # public-exposable count — the meaningful parity numerator. + "exposableFunctions": sum( + 1 for fn in functions if fn["network"]["exposable"] + ), + } + return idl diff --git a/parser/extractors.py b/parser/extractors.py index a5855e9..e67f542 100644 --- a/parser/extractors.py +++ b/parser/extractors.py @@ -31,6 +31,48 @@ def _c_spelling(ty) -> str: return spelling +# Canonical spellings of plain C scalars/builtins. +_SCALAR_CANON = { + "void", "_Bool", "bool", "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", "long", + "unsigned long", "long long", "unsigned long long", + "float", "double", "long double", +} +# Named opaque types that the PostgreSQL *stub* headers collapse to a bare +# scalar even without a pointer (type-erased values). Kept by name so they +# read as themselves, not as the stub's underlying integer. +_EXPLICIT_OPAQUE = {"Datum"} + + +def _strip(s: str) -> str: + return " ".join( + re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", s) + .replace("*", " ").split() + ) + + +def _preserved_opaque(ty) -> str | None: + """Keep the *declared* name of opaque types the PG stubs canonicalise to + a bare scalar (``Interval *`` / ``text *`` -> ``const int *``, ``Datum`` + -> ``unsigned long``). A pointer whose typedef'd pointee resolves to a + plain scalar is, in practice, always a stubbed opaque struct — so the + declared spelling is the truthful one. Genuine scalar pointers + (``int *result``) are unaffected: their pointee is a builtin, not a + distinct typedef name. + """ + if ty.kind == clang.cindex.TypeKind.POINTER: + pointee = ty.get_pointee() + dname = _strip(pointee.spelling) + cname = _strip(pointee.get_canonical().spelling) + if (dname and dname not in _SCALAR_CANON and "(" not in dname + and cname in _SCALAR_CANON and dname != cname): + return ty.spelling.replace("_Bool", "bool") + return None + if _strip(ty.spelling) in _EXPLICIT_OPAQUE: + return _strip(ty.spelling) + return None + + def _canonical_c_spelling(ty) -> str: # Like ``_canonical_spelling`` but normalises boolean types to ``"bool"``. # Handles: @@ -42,6 +84,9 @@ def _canonical_c_spelling(ty) -> str: # Fallback: also catch _Bool reached through other typedef chains if ty.get_canonical().kind == clang.cindex.TypeKind.BOOL: return "bool" + preserved = _preserved_opaque(ty) + if preserved is not None: + return preserved return _canonical_spelling(ty) diff --git a/parser/header_types.py b/parser/header_types.py new file mode 100644 index 0000000..550f3ee --- /dev/null +++ b/parser/header_types.py @@ -0,0 +1,123 @@ +"""Recover opaque parameter/return types from the header *source*. + +The PostgreSQL stub headers ``#define`` several opaque types (``Interval``, +``text``, ``TimestampTz`` …) to ``int`` *before* libclang parses, so the +typedef name is destroyed: a ``const Interval *`` argument reaches the +catalog as ``const int *``, indistinguishable from a real ``int *`` +out-parameter and impossible to project correctly. + +The public MEOS headers, however, declare every function on a regular +``extern ();`` line with the *true* spellings. This +module re-scans that source and reconciles the catalog: where libclang +produced a bare scalar but the header says a distinct **named pointer** +type, the header is the truth. Scalar typedefs (``TimestampTz``, ``int64``, +enums) are deliberately left as their resolved scalar — only mis-rendered +*opaque pointers* are restored. + +Pure text + ``re``; no libclang. +""" + +import re +from pathlib import Path + +_COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.DOTALL) +_DECL = re.compile( + r"\bextern\b\s+(?P[^;{}]+?\([^;{}]*\))\s*;", re.DOTALL) +_FUNC = re.compile(r"^(?P.+?)\b(?P\w+)\s*\((?P.*)\)$", + re.DOTALL) + +_SCALARS = { + "void", "bool", "_Bool", "char", "signed char", "unsigned char", + "short", "unsigned short", "int", "unsigned int", "long", + "unsigned long", "long long", "unsigned long long", "size_t", + "float", "double", "long double", + # scalar typedefs we *want* left resolved to their integer form + "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", + "uint64", "int8_t", "int16_t", "int32_t", "int64_t", "uint8_t", + "uint16_t", "uint32_t", "uint64_t", "TimestampTz", "TimeADT", + "DateADT", "Timestamp", "Datum", "meosType", "interpType", +} + + +def _norm(t: str) -> str: + return " ".join(t.replace("*", " * ").split()) + + +def _base(t: str) -> str: + return " ".join( + re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", t) + .replace("*", " ").split() + ) + + +def _split_params(s: str) -> list: + out, depth, cur = [], 0, "" + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if ch == "," and depth == 0: + out.append(cur) + cur = "" + else: + cur += ch + if cur.strip(): + out.append(cur) + return out + + +def _param_type(p: str) -> str: + p = p.strip() + if p in ("void", ""): + return "void" + p = re.sub(r"\[[^\]]*\]", " *", p) # arr[] -> arr * + m = re.match(r"^(.*?)(\w+)\s*$", p, re.DOTALL) # strip trailing name + return _norm(m.group(1) if m and "*" not in m.group(2) else p) + + +def scan_headers(headers_dir: Path) -> dict: + """``{func_name: {"ret": type, "params": [type, …]}}`` from source.""" + out: dict = {} + for h in sorted(Path(headers_dir).glob("**/*.h")): + text = _COMMENT.sub(" ", h.read_text(errors="replace")) + for d in _DECL.finditer(text): + m = _FUNC.match(" ".join(d.group("sig").split())) + if not m: + continue + params = [_param_type(x) for x in _split_params(m.group("params"))] + out[m.group("name")] = { + "ret": _norm(m.group("ret")), + "params": params if params != ["void"] else [], + } + return out + + +def _restore(decl_canon: str, header_t: str, enums: set) -> str | None: + """Return the header type if ``decl_canon`` is a scalar but the header + says a distinct named *pointer* opaque type, else ``None``.""" + cb = _base(decl_canon) + hb, hd = _base(header_t), header_t.count("*") + if (hd >= 1 and hb and hb not in _SCALARS and hb not in enums + and cb in _SCALARS): + return _norm(header_t) + return None + + +def reconcile(idl: dict, headers_dir: Path) -> dict: + """Restore opaque pointer types the stub headers erased to ``int``.""" + headers = scan_headers(headers_dir) + enums = {e["name"] for e in idl.get("enums", [])} + for fn in idl.get("functions", []): + h = headers.get(fn["name"]) + if not h: + continue + rt = fn["returnType"] + fixed = _restore(rt["canonical"], h["ret"], enums) + if fixed: + rt["c"] = rt["canonical"] = fixed + for p, ht in zip(fn.get("params", []), h["params"]): + fixed = _restore(p["canonical"], ht, enums) + if fixed: + p["cType"] = p["canonical"] = fixed + return idl diff --git a/parser/nullable.py b/parser/nullable.py new file mode 100644 index 0000000..41c411c --- /dev/null +++ b/parser/nullable.py @@ -0,0 +1,68 @@ +"""Extract per-parameter nullability from the MEOS C Doxygen as the SoT. + +A MEOS function parameter accepts NULL iff its Doxygen ``@param`` line says so, +e.g. ``@param[in] srs Spatial reference system, may be `NULL```. This is the +single source of truth the codegens consume — grounded in the C code, keyed by +parameter name, and cross-checked in MobilityDB against the PG layer (a SQL +function declared without ``STRICT`` + the wrapper's ``PG_ARGISNULL`` guards). + +The extractor walks the MEOS sources, pairs each Doxygen block with the function +it documents, and records the params whose description carries a NULL note. The +result feeds ``shape.nullable`` in the IDL so every binding can guard the param. +""" +from __future__ import annotations + +import glob +import re +from pathlib import Path + +# Doxygen block immediately followed by a function definition (``name(...) {``). +_FUNC = re.compile( + r'/\*\*(?P.*?)\*/\s*\n' + r'(?:[A-Za-z_][\w\s\*]*?\n)?' # optional return-type line + r'(?P[a-z][a-z0-9_]*)\s*\(' + r'(?P[^;{]*?)\)\s*\{', + re.S) +# One @param entry: capture the (possibly comma-separated) names + description. +_PARAM = re.compile( + r'@param\[[^\]]*\]\s+(?P\w+(?:\s*,\s*\w+)*)\s+(?P.*?)' + r'(?=\n\s*\*\s*@|\*/|\Z)', re.S) +_NULLISH = re.compile(r'may be\s+`?NULL`?|can be\s+`?NULL`?|`?NULL`?\s+is allowed' + r'|or\s+`?NULL`?', re.I) + + +def extract_nullable(meos_root: str | Path) -> dict[str, list[str]]: + """Return ``{function: [nullable params]}`` from the MEOS C sources under + ``meos_root`` (scans both ``src`` and ``include``).""" + root = Path(meos_root) + out: dict[str, list[str]] = {} + files = glob.glob(str(root / "src/**/*.c"), recursive=True) + files += glob.glob(str(root / "include/**/*.h"), recursive=True) + for f in files: + txt = Path(f).read_text(errors="ignore") + for m in _FUNC.finditer(txt): + name = m.group("name") + for pm in _PARAM.finditer(m.group("doc")): + if not _NULLISH.search(pm.group("desc")): + continue + for p in (n.strip() for n in pm.group("names").split(",")): + out.setdefault(name, []) + if p and p not in out[name]: + out[name].append(p) + return out + + +def merge_nullable(idl: dict, meos_root: str | Path) -> tuple[dict, int]: + """Fold the extracted nullability into each function's ``shape.nullable``.""" + nul = extract_nullable(meos_root) + n = 0 + for func in idl["functions"]: + params = nul.get(func["name"]) + if not params: + continue + present = {p["name"] for p in func.get("params", [])} + keep = [p for p in params if p in present] + if keep: + func.setdefault("shape", {})["nullable"] = keep + n += len(keep) + return idl, n diff --git a/parser/shapeinfer.py b/parser/shapeinfer.py new file mode 100644 index 0000000..598372c --- /dev/null +++ b/parser/shapeinfer.py @@ -0,0 +1,72 @@ +"""Infer per-function output-array *shape* from the C signatures. + +MEOS array-returning functions follow one fixed convention, so the shape the +codegens need is fully derivable from the headers — no hand-maintained table: + + TYPE *f(..., int *count) -> returns an array of ``count`` + TYPE **f(..., TYPE **extra, int *count) -> primary array return PLUS one + or more parallel out-arrays + +The output length is always passed *by pointer* (``int *count``); an *input* +array instead carries its length *by value* (``int count``). That pointer/value +distinction is exactly how a written-back out-array is told apart from a +read-only in-array — e.g. ``temporal_time_split(..., TimestampTz **time_bins, +int *count)`` (out) versus ``tsequence_make(const TInstant **instants, int +count, ...)`` (in). + +This replaces the ``meta/meos-meta.json`` shape entries, which had drifted to a +3-function stub and silently mis-classified every out-array as an input +parameter, breaking the split / space-split / mvtgeom / normalize families in +every binding generated from the IDL. +""" +from __future__ import annotations + + +# Parameters that accept NULL by MEOS convention regardless of the function. +# ``srs`` is the optional spatial-reference string of every ``*_as_*json`` / +# text output function — passing NULL means "no CRS". Nullability is otherwise +# semantic (not signature-derivable), so this stays a narrow, named convention +# rather than a blanket rule; extend only when a binding's tests prove a param +# is passed None. +_NULLABLE_BY_CONVENTION = {"srs"} + + +def _out_count_param(func: dict) -> str | None: + """Return the name of the by-pointer output count param, if the function + has one. This is the marker that the function returns array(s).""" + for p in func.get("params", []): + if p["name"] == "count" and p.get("cType", "").strip() == "int *": + return p["name"] + return None + + +def _is_written_back_array(p: dict) -> bool: + """A non-const double (or higher) pointer parameter the callee allocates + and writes back, i.e. a parallel output array.""" + ct = p.get("cType", "") + return "**" in ct and not ct.lstrip().startswith("const") + + +def infer_shapes(idl: dict) -> tuple[dict, dict]: + """Populate ``func['shape']`` with ``arrayReturn``/``outputArrays`` derived + from the signatures. Returns ``(idl, stats)``. Idempotent and additive: + only the array-output families are touched, everything else is untouched.""" + n_arr = n_oa = 0 + for func in idl["functions"]: + count = _out_count_param(func) + if not count: + continue # not array-returning; nothing to infer + shape = func.setdefault("shape", {}) + # The primary pointer return takes its length from the output count. + ret = func.get("returnType", {}).get("c", "") + if ret.rstrip().endswith("*"): + shape.setdefault("arrayReturn", {})["lengthFrom"] = { + "kind": "param", "name": count} + n_arr += 1 + # Parallel written-back out-arrays (``TYPE **extra`` alongside count). + out = [{"param": p["name"]} for p in func["params"] + if p["name"] != count and _is_written_back_array(p)] + if out: + shape["outputArrays"] = out + n_oa += len(out) + return idl, {"arrayReturn": n_arr, "outputArrays": n_oa} diff --git a/parser/sqlfn.py b/parser/sqlfn.py new file mode 100644 index 0000000..0c0fcf1 --- /dev/null +++ b/parser/sqlfn.py @@ -0,0 +1,158 @@ +"""Attach the SQL-name map (@sqlfn / @sqlop) to the MEOS-API catalog. + +The catalog carries MEOS-C function names + C signatures, but bindings that +emit a SQL/UDF surface (MobilityDB SQL, MobilitySpark UDFs, MobilityDuck, …) +need the user-facing SQL name and operator. Both are machine-extractable from +the doxygen tag chain that already pervades the source: + + MEOS-C fn --@csqlfn #MobilityDB_C()--> MobilityDB-C wrapper + MobilityDB-C wrapper --@sqlfn sqlName() / @sqlop @p --> SQL name + op + +So: in meos/src `@csqlfn #Wrapper()` sits above the MEOS-C function (→ MEOS-C → +Wrapper); in mobilitydb/src `@sqlfn name()` + `@sqlop @p ` sit above +`Datum Wrapper(PG_FUNCTION_ARGS)` (→ Wrapper → name, op). Join on Wrapper. + +Adds per function (when the chain resolves): `sqlfn`, `sqlop`, `mdbC`. +""" +import re +from pathlib import Path + +# A @csqlfn tag carries one OR MORE #Wrapper() references — comma- or +# space-separated, and possibly continued across doxygen lines — because a single +# MEOS function can back several wrappers (the ever/always pair eDisjoint/aDisjoint +# share one ea_* function; the shift/scale/shift_scale trio share one C function). +# The tag value runs from @csqlfn up to the next doxygen tag or the comment close. +_CSQLFN = re.compile(r"@csqlfn\b") +_CSQLFN_REF = re.compile(r"#(\w+)\s*\(\)") +_CSQLFN_END = re.compile(r"@\w|\*/") +# After the doxygen close, the MEOS-C definition: an optional return-type line +# (no parens/braces/;/=), then `name(`. +_FNDEF = re.compile(r"\*/\s*\n(?:[^\n(){};=]+\n)?(\w+)\s*\(") +_SQLFN = re.compile(r"@sqlfn\s+(\w+)\s*\(\)") +_SQLOP = re.compile(r"@sqlop\s+@p\s+(\S+)") +_DATUM = re.compile(r"Datum\s+(\w+)\s*\(\s*PG_FUNCTION_ARGS") + + +def _meos_to_mdb(meos_src): + """MEOS-C function name -> ordered list of MobilityDB-C wrapper names (from + @csqlfn). One MEOS function can back more than one wrapper — the ever/always + pair eDisjoint/aDisjoint share a single ea_* function tagged + `@csqlfn #Edisjoint_…() #Adisjoint_…()` — so each @csqlfn carries one or more + #Wrapper() references; collect them all (mirrors _mdb_to_sql collecting every + @sqlfn rather than the first).""" + out = {} + for cf in Path(meos_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _CSQLFN.finditer(text): + tail = text[m.end():] + end = _CSQLFN_END.search(tail) + value = tail[:end.start()] if end else tail + wrappers = _CSQLFN_REF.findall(value) + if not wrappers: + continue + fm = _FNDEF.search(text, m.end()) + if not fm: + continue + lst = out.setdefault(fm.group(1), []) + for w in wrappers: + if w not in lst: + lst.append(w) + return out + + +def _mdb_to_sql(mdb_src): + """MobilityDB-C wrapper name -> ordered list of (sqlfn, sqlop). + + A shared PG wrapper can carry more than one @sqlfn (e.g. Temporal_derivative + is exposed as both derivative() and speed()), so collect ALL of them rather + than the first — otherwise the mapped SQL name is order-dependent. + """ + out = {} + for cf in Path(mdb_src).rglob("*.c"): + text = cf.read_text(errors="ignore") + for m in _SQLFN.finditer(text): + sqlfn = m.group(1) + # @sqlop lives in the SAME doxygen block (before the closing */). + close = text.find("*/", m.end()) + block = text[m.start():close] if close != -1 else text[m.start():m.start() + 800] + op = _SQLOP.search(block) + dm = _DATUM.search(text, close if close != -1 else m.end()) + if dm: + entry = (sqlfn, op.group(1) if op else None) + lst = out.setdefault(dm.group(1), []) + if entry not in lst: + lst.append(entry) + return out + + +def attach_sqlfn_map(idl, meos_src, mdb_src): + m2d = _meos_to_mdb(meos_src) + d2s = _mdb_to_sql(mdb_src) + n = 0 + for f in idl["functions"]: + wrappers = m2d.get(f["name"]) + if not wrappers: + continue + # A MEOS function can back several wrappers (the ever/always pair), each + # carrying its own @sqlfn; collect the (sqlfn, sqlop) pairs across all of + # them in order, keeping the primary (first) wrapper for back-compat. + pairs = [] + for w in wrappers: + for entry in d2s.get(w, []): + if entry not in pairs: + pairs.append(entry) + if not pairs: + continue + f["mdbC"] = wrappers[0] + f["sqlfn"] = pairs[0][0] + if pairs[0][1]: + f["sqlop"] = pairs[0][1] + # Shared wrapper OR ever/always pair exposing >1 SQL name: record them all. + if len(pairs) > 1: + f["sqlfnAll"] = [s for s, _ in pairs] + if len(wrappers) > 1: + f["mdbCAll"] = wrappers + n += 1 + return idl, n + + +# MEOS-C ever/always spatial-relationship functions are named _...; their +# @csqlfn must point at the matching _... wrapper. A copy-paste @csqlfn in +# meos/src (e.g. eintersects_tgeo_geo tagged #Aintersects_tgeo_geo) silently flips the +# resolved @sqlfn from eX to aX — which then drops the real overload from the eX dispatch +# group and lets a wrong subtype backing be reached (a runtime "must be of type ..." error +# in the bindings). The parser is faithful, so guard the SOURCE here: flag any function +# whose name e/a prefix disagrees with its resolved @sqlfn e/a prefix. +_EA_FAMILY = re.compile( + r"^(e|a)(intersects|disjoint|contains|contained|covers|coveredby|touches|" + r"dwithin|within|equals|crosses|overlaps)_") + + +def lint_ea_sqlfn(idl): + """Return [(meos_c_name, sqlfn)] where the function's ever/always (e/a) name prefix + contradicts its resolved @sqlfn — a source @csqlfn mistag in meos/src.""" + bad = [] + for f in idl["functions"]: + sf = f.get("sqlfn") + m = _EA_FAMILY.match(f["name"]) + if sf and m and re.match(r"^[ea][A-Z]", sf) and sf[0] != m.group(1): + bad.append((f["name"], sf)) + return bad + + +def lint_sqlfn_case_collisions(idl): + """Return [(lower, [spelling, ...])] for @sqlfn names that collide + case-insensitively but differ in case (e.g. tDistance vs tdistance). + + PostgreSQL folds unquoted identifiers to lower case, so the two spell the + SAME SQL function and the clash is invisible in SQL / pg_regress. But the + binding name is taken case-SENSITIVELY, and case-insensitive engines (Spark + SQL, …) register every spelling under one UDF — so one silently shadows the + other. A canonical binding name must have exactly ONE spelling; surface a + casing straggler here before it reaches a binding.""" + by_lower = {} + for f in idl["functions"]: + for sf in [f.get("sqlfn"), *f.get("sqlfnAll", [])]: + if sf: + by_lower.setdefault(sf.lower(), set()).add(sf) + return sorted((lo, sorted(sp)) for lo, sp in by_lower.items() if len(sp) > 1) diff --git a/parser/typerecover.py b/parser/typerecover.py index 01d8153..4acee15 100644 --- a/parser/typerecover.py +++ b/parser/typerecover.py @@ -35,6 +35,7 @@ "Timestamp": "Timestamp", "TimestampTz": "TimestampTz", "H3Index": "uint64_t", + "Quadbin": "uint64_t", "text": "text", "GSERIALIZED": "GSERIALIZED", "Interval": "Interval", @@ -44,6 +45,8 @@ "GBOX": "GBOX", "BOX3D": "BOX3D", "AFFINE": "AFFINE", + "Jsonb": "Jsonb", + "JsonPath": "JsonPath", } _NAMES = "|".join(sorted(_TYPE_MAP, key=len, reverse=True)) @@ -75,7 +78,8 @@ def _recovery(fragment): suffix = (" " + stars) if stars else "" collapsed = f"{const}int{suffix}" recovered = f"{const}{_TYPE_MAP[m.group('base')]}{suffix}" - return collapsed, recovered + original = f"{const}{m.group('base')}{suffix}" + return collapsed, recovered, original def _parse_header_decls(headers_dir): @@ -123,12 +127,17 @@ def _apply(slot, recovery): """Rewrite a return/param slot in place; return 1 if rewritten.""" if not (recovery and isinstance(slot, dict)): return 0 - collapsed, recovered = recovery + collapsed, recovered, original = recovery key = "c" if "c" in slot else "cType" - if _nospace(slot.get(key)) != _nospace(collapsed): + # The base name is either erased to int by the host-collision prefix + # rename (slot spells `collapsed`), or it survives while only the + # canonical collapses (slot spells `original`, e.g. a MobilityDB typedef + # such as Quadbin whose uint64 underlying type was the part that erased). + recoverable = (_nospace(collapsed), _nospace(original)) + if _nospace(slot.get(key)) not in recoverable: return 0 slot[key] = recovered - if _nospace(slot.get("canonical")) == _nospace(collapsed): + if _nospace(slot.get("canonical")) in recoverable: slot["canonical"] = recovered return 1 diff --git a/report.py b/report.py new file mode 100644 index 0000000..29faa70 --- /dev/null +++ b/report.py @@ -0,0 +1,153 @@ +# Emit a coverage / irregularity report from the enriched catalog. +# +# python run.py # produce output/meos-idl.json +# python report.py # -> output/meos-coverage.json (+ stderr summary) +# +# `worklist` is the actionable form: one entry per non-exposable public +# function with a `class` and a concrete `suggest` — the precise upstream +# regularization that would make it stateless-projectable (e.g. "rename the +# out-parameter to `result`", "add a trailing `int *count`", "return a +# struct instead of N out-parameters", "add a single-arg `T_in`/`T_out`"). +# `byClass` ranks the classes by size, so the upstream work is prioritised +# by leverage. Fixing an irregularity upstream removes its entry and lifts +# coverage toward 100%; internal (`meos_internal*.h`) functions and +# inherently-stateful aggregates are reported but are correct exclusions, +# never silent gaps. This is the direct, no-coupling input for the +# cross-repo API-uniformization workstream. + +import json +import re +import sys +from collections import Counter, defaultdict +from pathlib import Path + + +def _base(c: str) -> str: + return " ".join(re.sub(r"\b(const|volatile|struct|union|enum)\b", " ", + c).replace("*", " ").split()) + + +def _dep(c: str) -> int: + return c.count("*") + + +def _classify(fn: dict): + """``(klass, suggestion)`` — the concrete upstream regularization that + would make this function stateless-projectable. Turns the worklist from + *what is broken* into *what to change upstream*. + """ + reason = fn["network"]["reason"] or "" + tags = {p.split(":")[0] for p in reason.split("; ")} + detail = {p.split(":", 1)[1] for p in reason.split("; ") if ":" in p} + params = fn.get("params", []) + ret = fn.get("returnType", {}).get("canonical", "") + + if tags & {"lifecycle", "index"}: + return ("plumbing", + "intentionally not exposed (process/library plumbing)") + if fn.get("category") == "aggregate" or "SkipList" in detail: + return ("stateful", + "stateful aggregation — expose via a stateful endpoint, " + "not a stateless RPC") + if "Datum" in detail or "MeosArray" in detail: + return ("internal-generic", + "Datum/array-generic — expose a typed variant; keep the " + "generic form internal") + + outptrs = [p for p in params if _dep(p["canonical"]) >= 2 + or (_dep(p["canonical"]) == 1 and _base(p["canonical"]) in + ("int", "long", "double", "float", "bool"))] + if "unsupported-return" in tags and _dep(ret) >= 1: + return ("array-return-shape", + f"array return `{ret}` lacks a length: add a trailing " + "`int *count` out-parameter (+ an element encoder)") + if len(outptrs) >= 2: + return ("multi-out", + f"{len(outptrs)} out-parameters: return a struct (or split " + "into separate single-result accessors)") + if len(outptrs) == 1 and outptrs[0]["name"] not in ("result", "value"): + return ("out-param-naming", + f"rename out-parameter `{outptrs[0]['name']}` to `result` " + "(`bool f(.., T *result)` convention)") + nocodec = sorted(d for d in detail if d[:1].isupper()) + if tags & {"no-decoder", "no-encoder"} and nocodec: + t = nocodec[0] + return ("no-codec", + f"type `{t}` has no stateless codec: add a single-argument " + f"`{t.lower()}_in`/`{t.lower()}_out` wrapper, or keep it " + "internal") + if "array-or-out-param" in tags: + return ("array-shape", + "pass element arrays as an adjacent `(Elem **arr, int " + "count)`; use `bool f(.., T *result)` for out-values") + return ("other", "regularize the signature to a stateless shape") + + +IN_PATH = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("output/meos-idl.json") +OUT_PATH = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("output/meos-coverage.json") + + +def build_report(catalog: dict) -> dict: + fns = catalog.get("functions", []) + pub = [f for f in fns if f.get("api") == "public"] + exposable = [f for f in pub if f["network"]["exposable"]] + by_reason: dict = defaultdict(list) + for f in pub: + if f["network"]["exposable"]: + continue + # collapse "tag:detail; tag:detail" to the set of distinct tags + tags = sorted({p.split(":")[0] + for p in (f["network"]["reason"] or "").split("; ")}) + by_reason["; ".join(tags)].append(f["name"]) + + worklist = [] + for f in pub: + if f["network"]["exposable"]: + continue + klass, suggest = _classify(f) + worklist.append({ + "name": f["name"], "file": f.get("file"), + "reason": f["network"]["reason"], + "class": klass, "suggest": suggest, + }) + worklist.sort(key=lambda w: (w["class"], w["name"])) + by_class = Counter(w["class"] for w in worklist) + + total = len(pub) + n_exp = len(exposable) + return { + "publicTotal": total, + "exposable": n_exp, + "coveragePct": round(n_exp * 100 / total, 1) if total else 0, + "internalExcluded": len(fns) - total, + "gap": total - n_exp, + "byClass": dict(by_class.most_common()), + "byReason": {k: sorted(v) + for k, v in sorted(by_reason.items(), + key=lambda kv: -len(kv[1]))}, + # actionable: one upstream-change suggestion per gap function + "worklist": worklist, + } + + +def main() -> None: + if not IN_PATH.exists(): + sys.exit(f"Catalog not found: {IN_PATH} — run `python run.py` first.") + catalog = json.loads(IN_PATH.read_text()) + if not any("network" in f for f in catalog.get("functions", [])): + sys.exit(f"{IN_PATH} is not enriched.") + + rep = build_report(catalog) + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUT_PATH.write_text(json.dumps(rep, indent=2)) + + print(f"[coverage] public {rep['exposable']}/{rep['publicTotal']} " + f"({rep['coveragePct']}%), gap {rep['gap']}, " + f"{rep['internalExcluded']} internal excluded → {OUT_PATH}", + file=sys.stderr) + for klass, n in rep["byClass"].items(): + print(f" {n:4d} {klass}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/run.py b/run.py index 640dd9e..98a447b 100644 --- a/run.py +++ b/run.py @@ -1,3 +1,4 @@ +import os import sys import json from pathlib import Path @@ -5,6 +6,12 @@ from parser.parser import parse_all_headers, merge_meta from parser.portable import attach_portable_aliases from parser.typerecover import recover_collapsed_types +from parser.shapeinfer import infer_shapes +from parser.nullable import merge_nullable +from parser.sqlfn import attach_sqlfn_map, lint_ea_sqlfn, lint_sqlfn_case_collisions +from parser.doxygroup import attach_groups +from parser.header_types import reconcile +from parser.enrich import enrich_idl HEADERS_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("./meos/include") @@ -28,6 +35,25 @@ def main(): print(f" recovered {rec['returns']} return types, " f"{rec['params']} params from collapsed int", file=sys.stderr) + # 1c. Generate the codegen `shape` from the signatures + Doxygen, replacing + # the hand-maintained meta stub. outputArrays/arrayReturn come from the + # parameter forms; nullable comes from the C `@param ... may be NULL` SoT. + idl, sh = infer_shapes(idl) + print(f" inferred shape: {sh['arrayReturn']} array returns, " + f"{sh['outputArrays']} output arrays", file=sys.stderr) + idl, nn = merge_nullable(idl, HEADERS_DIR.parent) + print(f" nullable params from Doxygen `may be NULL`: {nn}", + file=sys.stderr) + + # 1d. Restore opaque types the PG stub headers #define'd away to int. + print(f" reconciling types from header source...", file=sys.stderr) + idl = reconcile(idl, HEADERS_DIR) + + # 1e. Derive service-projection metadata (category / encodings / network). + # Runs before the merge so manual annotations override the heuristics. + print(f" enriching {len(idl['functions'])} functions...", file=sys.stderr) + idl = enrich_idl(idl) + # 2. Merge with manual metadata if META_PATH.exists(): print(f"[2/3] Merging with {META_PATH}...", file=sys.stderr) @@ -40,13 +66,53 @@ def main(): file=sys.stderr) idl = attach_portable_aliases(idl, PORTABLE_PATH) + # 4. Attach the SQL-name map (@sqlfn/@sqlop) from the vendored source. + # The source root is overridable (MDB_SRC_ROOT) so a binding can point the + # @sqlfn/@ingroup extraction at the SAME pinned checkout as the headers, + # keeping the catalog reproducibly equivalent to that pin. + SRC_ROOT = Path(os.environ.get("MDB_SRC_ROOT", "./_mobilitydb")) + MEOS_SRC = SRC_ROOT / "meos" / "src" + MDB_SRC = SRC_ROOT / "mobilitydb" / "src" + if MEOS_SRC.exists() and MDB_SRC.exists(): + idl, nsql = attach_sqlfn_map(idl, MEOS_SRC, MDB_SRC) + print(f"[4/4] Attached {nsql} @sqlfn SQL names", file=sys.stderr) + # Guard: a copy-paste @csqlfn in meos/src can point an ever/always function at + # the opposite-prefix wrapper (eintersects_* tagged #Aintersects_*), flipping its + # SQL name and breaking the binding overload dispatch. The parser is faithful, so + # surface the SOURCE mistag here rather than ship a wrong catalog silently. + ea_bad = lint_ea_sqlfn(idl) + if ea_bad: + print(f" ⚠ {len(ea_bad)} @csqlfn e/a-prefix mistag(es) in meos/src " + f"(fix at source — wrong @sqlfn resolved):", file=sys.stderr) + for cname, sf in ea_bad: + print(f" {cname} -> @sqlfn {sf}", file=sys.stderr) + # Guard: @sqlfn names that differ only by case (e.g. tDistance vs tdistance) + # are the SAME SQL function (PostgreSQL folds the identifier) but DISTINCT + # binding names — a case-insensitive engine (Spark SQL) registers both under + # one UDF, so one silently shadows the other. Invisible in SQL; surface the + # casing straggler here, to be fixed at the MEOS-C @sqlfn source. + case_bad = lint_sqlfn_case_collisions(idl) + if case_bad: + print(f" ⚠ {len(case_bad)} @sqlfn case-collision(s) (pick ONE canonical " + f"spelling at the MEOS-C source — binding-breaking otherwise):", file=sys.stderr) + for _lo, spellings in case_bad: + print(f" {' vs '.join(spellings)}", file=sys.stderr) + + # 5. Attach the doxygen module group (@ingroup) from the vendored source, so + # bindings organize their generated surface like the reference manual. + if MEOS_SRC.exists(): + idl, ngrp = attach_groups(idl, MEOS_SRC) + print(f"[5/5] Attached {ngrp} doxygen @ingroup groups", file=sys.stderr) + idl_path = OUTPUT_DIR / "meos-idl.json" with open(idl_path, "w") as f: json.dump(idl, f, indent=2) print(f" → {idl_path} written", file=sys.stderr) pa = idl.get("portableAliases", {}).get("count", 0) - print(f"\nDone: {len(idl['functions'])} functions, " + exposable = idl.get("enrichment", {}).get("exposableFunctions", 0) + print(f"\nDone: {len(idl['functions'])} functions " + f"({exposable} stateless-exposable), " f"{len(idl['structs'])} structs, " f"{len(idl['enums'])} enums, " f"{pa} portable bare-name aliases", file=sys.stderr) diff --git a/tests/test_coverage_gate.py b/tests/test_coverage_gate.py new file mode 100644 index 0000000..5b0e8e9 --- /dev/null +++ b/tests/test_coverage_gate.py @@ -0,0 +1,53 @@ +"""Coverage non-regression gate. python3 tests/test_coverage_gate.py + +Skipped unless an enriched ``output/meos-idl.json`` is present (so CI +without libclang still passes). When present, it asserts public coverage +does not regress below the established floor and that the worklist stays +consistent — a heuristic change that silently drops coverage fails here. +""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from report import build_report + +_CATALOG = Path(__file__).resolve().parents[1] / "output" / "meos-idl.json" +_FLOOR = 90.0 # public %, ratchet up as upstream uniformization lands + + +@unittest.skipUnless(_CATALOG.exists(), "run `python run.py` first") +class CoverageGateTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.r = build_report(json.loads(_CATALOG.read_text())) + + def test_public_coverage_not_regressed(self): + self.assertGreaterEqual( + self.r["coveragePct"], _FLOOR, + f"public coverage {self.r['coveragePct']}% < floor {_FLOOR}% " + "— a heuristic change dropped coverage") + + def test_worklist_consistent(self): + # one actionable entry per gap; classes partition the gap + self.assertEqual(len(self.r["worklist"]), self.r["gap"]) + self.assertEqual(sum(self.r["byClass"].values()), self.r["gap"]) + self.assertTrue(all(w["suggest"] and w["class"] + for w in self.r["worklist"])) + + def test_no_internal_or_silent_gap(self): + # internal never counted as public; every gap has a reason + cat = json.loads(_CATALOG.read_text()) + for f in cat["functions"]: + if f.get("api") == "internal": + continue + if not f["network"]["exposable"]: + self.assertTrue(f["network"]["reason"], + f"{f['name']}: gap without a reason") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_enrich.py b/tests/test_enrich.py new file mode 100644 index 0000000..df2b823 --- /dev/null +++ b/tests/test_enrich.py @@ -0,0 +1,287 @@ +"""Unit tests for parser/enrich.py. + +Runs without libclang or pytest: python3 tests/test_enrich.py + +The fixture uses the *canonical* C spellings libclang actually emits +(``struct Temporal *``, ``unsigned char``, ``int`` for booleans, enum +parameters), so the assertions double as a specification. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.enrich import enrich_idl, classify_category, build_type_encodings + + +def fn(name, ret, *params): + return { + "name": name, + "file": "meos.h", + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t} for t, n in params], + } + + +T = "const struct Temporal *" +FUNCTIONS = [ + fn("temporal_in", "struct Temporal *", ("const char *", "str")), + fn("temporal_out", "char *", (T, "temp")), + fn("temporal_from_mfjson", "struct Temporal *", ("const char *", "str")), + fn("temporal_as_hexwkb", "char *", + (T, "temp"), ("unsigned char", "variant"), ("int *", "size_out")), + fn("bigintset_in", "struct Set *", ("const char *", "str")), + fn("bigintset_out", "char *", ("const struct Set *", "set")), + fn("temporal_eq", "int", (T, "temp1"), (T, "temp2")), + fn("tpoint_speed", "struct Temporal *", (T, "temp")), + fn("tjsonb_to_ttext", "struct Temporal *", (T, "temp")), + fn("union_set_set", "struct Set *", + ("const struct Set *", "s1"), ("const struct Set *", "s2")), + fn("temporal_num_instants", "int", (T, "temp")), + fn("tsequence_make", "struct TSequence *", + ("struct TInstant **", "instants"), ("int", "count"), + ("interpType", "interp")), + fn("tjsonb_value_at_timestamptz", "int", + (T, "temp"), ("long", "t"), ("int", "strict"), + ("struct Jsonb **", "value")), + fn("temporal_set_interp", "struct Temporal *", + (T, "temp"), ("interpType", "interp")), + fn("meos_initialize", "void"), + fn("rtree_insert", "void", + ("struct RTree *", "rtree"), ("void *", "box"), ("long", "id")), + fn("temporal_timestamps", "int *", (T, "temp"), ("int *", "count")), + # aux args: a defaultable formatting scalar (maxdd) is allowed and + # defaulted; a semantic *type tag disqualifies the helper entirely. + fn("box_in", "struct Box *", ("const char *", "str")), + fn("box_out", "char *", + ("const struct Box *", "box"), ("int", "maxdd")), + fn("weird_in", "struct Weird *", + ("const char *", "str"), ("int", "basetype")), + # An otherwise-exposable function living in the internal header: it must + # be policy-excluded (api=internal), like the programmer Datum API. + dict(fn("internal_op", "struct Temporal *", (T, "temp")), + file="meos_internal.h"), + # Scalar out-parameter accessor: bool f(.., int *result) — the value is + # returned through the trailing out-param, the bool is a presence flag. + fn("setspan_value_n", "int", + ("const struct Set *", "s"), ("int", "n"), ("int *", "result")), + # Opaque out-parameter accessor: bool f(.., Box **result) — the value + # comes back as an opaque pointer, serialised via the type's encoder. + fn("boxset_value_n", "int", + ("const struct Set *", "s"), ("int", "n"), + ("struct Box **", "result")), + # Input-array builder: f(Elem **arr, int count) — the (array,count) pair + # becomes one JSON-array wire param; the count is implicit. + fn("temporal_merge_array", "struct Temporal *", + ("struct Temporal **", "temparr"), ("int", "count")), + # Array return: Elem **f(.., int *count) — a freshly-allocated element + # array; the count out-param is implicit, result is a JSON array. + fn("temporal_components", "struct Temporal **", + (T, "temp"), ("int *", "count")), +] + +STRUCTS = [{"name": n, "fields": []} for n in + ("Temporal", "TSequence", "Set", "RTree", "Jsonb", "TInstant", + "Box", "Weird")] +ENUMS = [{"name": "interpType", "values": []}] + + +def make_idl(): + return enrich_idl({ + "functions": [dict(f, returnType=dict(f["returnType"]), + params=[dict(p) for p in f["params"]]) + for f in FUNCTIONS], + "structs": [dict(s) for s in STRUCTS], + "enums": [dict(e) for e in ENUMS], + }) + + +def by_name(idl): + return {f["name"]: f for f in idl["functions"]} + + +class CategoryTests(unittest.TestCase): + def test_categories(self): + c = {f["name"]: classify_category(f) for f in FUNCTIONS} + self.assertEqual(c["temporal_in"], "io") + self.assertEqual(c["temporal_from_mfjson"], "io") + self.assertEqual(c["temporal_as_hexwkb"], "io") + self.assertEqual(c["temporal_eq"], "predicate") # returns int + self.assertEqual(c["tpoint_speed"], "transformation") + self.assertEqual(c["tjsonb_to_ttext"], "conversion") + self.assertEqual(c["union_set_set"], "setop") + self.assertEqual(c["temporal_num_instants"], "accessor") + self.assertEqual(c["tsequence_make"], "constructor") + self.assertEqual(c["meos_initialize"], "lifecycle") + self.assertEqual(c["rtree_insert"], "index") + + +class TypeEncodingTests(unittest.TestCase): + def setUp(self): + self.te = build_type_encodings( + FUNCTIONS, {s["name"] for s in STRUCTS}) + + def test_struct_prefix_stripped_and_round_trip(self): + self.assertIn("Temporal", self.te) # not "struct Temporal" + # temporal_as_hexwkb is still excluded — its `size_out` is a pointer + # (out-param), not a defaultable scalar — so no wkb encoder here. + self.assertEqual(self.te["Temporal"]["encodings"], + ["mfjson", "text"]) + self.assertEqual(self.te["Temporal"]["in"], "temporal_in") + self.assertEqual(self.te["Temporal"]["out"], "temporal_out") + self.assertEqual(self.te["Set"]["in"], "bigintset_in") + self.assertEqual(self.te["Set"]["out"], "bigintset_out") + + def test_defaultable_aux_accepted_type_tag_rejected(self): + # box_out(box, int maxdd) qualifies; maxdd defaults to 15. + self.assertEqual(self.te["Box"]["out"], "box_out") + self.assertEqual(self.te["Box"]["out_aux"], + [{"name": "maxdd", "kind": "integer", + "default": 15}]) + self.assertEqual(self.te["Box"]["in"], "box_in") + self.assertEqual(self.te["Box"]["in_aux"], []) + # weird_in(str, int basetype): the *type tag disqualifies it, so + # Weird gets no decoder at all. + self.assertNotIn("Weird", self.te) + + def test_no_primitive_or_intermediate_false_positives(self): + self.assertNotIn("int", self.te) # was a real false positive + self.assertNotIn("char", self.te) + self.assertNotIn("TSequence", self.te) # builder-only type + for k in self.te: + self.assertNotIn("struct ", k) + + def test_struct_serialization_folded(self): + s = {x["name"]: x for x in make_idl()["structs"]} + self.assertIn("serialization", s["Temporal"]) + self.assertNotIn("serialization", s["TSequence"]) + + +class ExposabilityTests(unittest.TestCase): + def setUp(self): + self.fns = by_name(make_idl()) + + def n(self, name): + return self.fns[name]["network"] + + def test_int_returning_predicate_exposable(self): + self.assertTrue(self.n("temporal_eq")["exposable"]) + self.assertEqual(self.fns["temporal_eq"]["wire"]["result"], + {"kind": "json", "json": "integer"}) + + def test_serialized_round_trip(self): + w = self.fns["tpoint_speed"]["wire"] + self.assertTrue(self.n("tpoint_speed")["exposable"]) + self.assertEqual(w["params"][0]["kind"], "serialized") + self.assertEqual(w["params"][0]["decode"], "temporal_in") + self.assertEqual(w["result"]["encode"], "temporal_out") + + def test_enum_param_is_scalar_and_exposable(self): + f = self.fns["temporal_set_interp"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual(f["wire"]["params"][1], + {"name": "interp", "kind": "json", + "json": "string", "enum": "interpType"}) + + def test_io_parse_serialize_exposable(self): + for name in ("temporal_in", "temporal_out", "temporal_from_mfjson", + "bigintset_in", "bigintset_out"): + self.assertTrue(self.n(name)["exposable"], name) + + def test_out_param_not_exposable(self): + r = self.n("temporal_as_hexwkb")["reason"] + self.assertFalse(self.n("temporal_as_hexwkb")["exposable"]) + self.assertIn("array-or-out-param:size_out", r) + r2 = self.n("tjsonb_value_at_timestamptz")["reason"] + self.assertIn("array-or-out-param:value", r2) + + def test_array_param_and_missing_encoder(self): + r = self.n("tsequence_make")["reason"] + self.assertFalse(self.n("tsequence_make")["exposable"]) + self.assertIn("array-or-out-param:instants", r) + self.assertIn("no-encoder:TSequence", r) + + def test_array_return_not_exposable(self): + r = self.n("temporal_timestamps")["reason"] + self.assertFalse(self.n("temporal_timestamps")["exposable"]) + self.assertIn("unsupported-return:int *", r) + + def test_lifecycle_and_index_not_exposable(self): + self.assertIn("lifecycle", self.n("meos_initialize")["reason"]) + self.assertIn("index", self.n("rtree_insert")["reason"]) + + +class ApiClassificationTests(unittest.TestCase): + def setUp(self): + self.fns = by_name(make_idl()) + + def test_internal_policy_excluded(self): + f = self.fns["internal_op"] + self.assertEqual(f["api"], "internal") + self.assertFalse(f["network"]["exposable"]) + self.assertIn("internal", f["network"]["reason"]) + + def test_public_default(self): + self.assertEqual(self.fns["temporal_eq"]["api"], "public") + self.assertTrue(self.fns["temporal_eq"]["network"]["exposable"]) + + def test_scalar_outparam_projected_as_result(self): + f = self.fns["setspan_value_n"] + self.assertTrue(f["network"]["exposable"]) + pnames = [p["name"] for p in f["wire"]["params"]] + self.assertEqual(pnames, ["s", "n"]) # 'result' not a param + r = f["wire"]["result"] + self.assertEqual(r["kind"], "json") + self.assertEqual(r["json"], "integer") + self.assertEqual(r["from_outparam"], "result") + self.assertTrue(r["presence_return"]) # int return = presence + + def test_opaque_outparam_projected_as_serialized(self): + f = self.fns["boxset_value_n"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual([p["name"] for p in f["wire"]["params"]], ["s", "n"]) + r = f["wire"]["result"] + self.assertEqual(r["kind"], "serialized") # opaque -> encoded + self.assertEqual(r["encode"], "box_out") + self.assertEqual(r["from_outparam"], "result") + self.assertTrue(r["presence_return"]) + + def test_array_return(self): + f = self.fns["temporal_components"] + self.assertTrue(f["network"]["exposable"]) + self.assertEqual([p["name"] for p in f["wire"]["params"]], ["temp"]) + r = f["wire"]["result"] + self.assertEqual(r["kind"], "array") + self.assertEqual(r["count_outparam"], "count") + self.assertEqual(r["element"]["kind"], "serialized") + self.assertEqual(r["element"]["encode"], "temporal_out") + + def test_input_array_builder(self): + f = self.fns["temporal_merge_array"] + self.assertTrue(f["network"]["exposable"]) + params = f["wire"]["params"] + self.assertEqual(len(params), 1) # count is implicit + a = params[0] + self.assertEqual(a["name"], "temparr") + self.assertEqual(a["kind"], "array") + self.assertEqual(a["count_param"], "count") + self.assertEqual(a["element"]["kind"], "serialized") + self.assertEqual(a["element"]["decode"], "temporal_in") + self.assertEqual(f["wire"]["result"]["kind"], "serialized") + + +class SummaryTests(unittest.TestCase): + def test_enrichment_summary(self): + e = make_idl()["enrichment"] + self.assertEqual(sum(e["categoryCounts"].values()), len(FUNCTIONS)) + self.assertEqual(e["internalFunctions"], 1) # internal_op + self.assertEqual(e["publicFunctions"], len(FUNCTIONS) - 1) + # 13 + setspan_value_n + boxset_value_n + temporal_merge_array + # + temporal_components (array return); internal_op excluded. + self.assertEqual(e["exposableFunctions"], 17) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_header_types.py b/tests/test_header_types.py new file mode 100644 index 0000000..a91cb74 --- /dev/null +++ b/tests/test_header_types.py @@ -0,0 +1,98 @@ +"""Unit tests for parser/header_types.py. + +Runs without libclang or pytest: python3 tests/test_header_types.py +""" + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.header_types import scan_headers, reconcile + +_HEADER = """ +/* a comment + spanning lines */ +extern TimestampTz add_timestamptz_interval(TimestampTz t, + const Interval *interv); +extern bool contains_set_text(const Set *s, text *t); // trailing comment +extern bool bigintset_value_n(const Set *s, int n, int64 *result); +extern Temporal *temporal_copy(const Temporal *temp); +extern void meos_initialize(void); +""" + + +class ScanTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.TemporaryDirectory() + (Path(self.d.name) / "meos.h").write_text(_HEADER) + self.h = scan_headers(Path(self.d.name)) + + def tearDown(self): + self.d.cleanup() + + def test_signatures_recovered(self): + self.assertEqual(self.h["add_timestamptz_interval"]["params"], + ["TimestampTz", "const Interval *"]) + self.assertEqual(self.h["contains_set_text"]["params"], + ["const Set *", "text *"]) + self.assertEqual(self.h["meos_initialize"]["params"], []) + self.assertEqual(self.h["temporal_copy"]["ret"], "Temporal *") + + +class ReconcileTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.TemporaryDirectory() + (Path(self.d.name) / "meos.h").write_text(_HEADER) + + def tearDown(self): + self.d.cleanup() + + def idl(self): + # Mimic the libclang output *after* the stub erased the names. + return { + "enums": [], + "functions": [ + {"name": "add_timestamptz_interval", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "t", "cType": "int", "canonical": "int"}, + {"name": "interv", "cType": "const int *", + "canonical": "const int *"}]}, + {"name": "contains_set_text", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "s", "cType": "const struct Set *", + "canonical": "const struct Set *"}, + {"name": "t", "cType": "int *", + "canonical": "int *"}]}, + {"name": "bigintset_value_n", + "returnType": {"c": "int", "canonical": "int"}, + "params": [{"name": "s", "cType": "const struct Set *", + "canonical": "const struct Set *"}, + {"name": "n", "cType": "int", "canonical": "int"}, + {"name": "result", "cType": "int *", + "canonical": "int *"}]}, + ], + } + + def test_opaque_pointers_restored_scalars_left_alone(self): + idl = reconcile(self.idl(), Path(self.d.name)) + f = {x["name"]: x for x in idl["functions"]} + # const Interval * restored from the header source + self.assertEqual(f["add_timestamptz_interval"]["params"][1]["canonical"], + "const Interval *") + # TimestampTz return stays the resolved scalar (int) — not restored + self.assertEqual(f["add_timestamptz_interval"]["returnType"]["canonical"], + "int") + # text * restored + self.assertEqual(f["contains_set_text"]["params"][1]["canonical"], + "text *") + # genuine int* out-param (header also says int64*) is a scalar + # pointer -> left exactly as libclang produced it + self.assertEqual(f["bigintset_value_n"]["params"][2]["canonical"], + "int *") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_movfeat.py b/tests/test_movfeat.py new file mode 100644 index 0000000..d6c7bc7 --- /dev/null +++ b/tests/test_movfeat.py @@ -0,0 +1,230 @@ +"""Unit tests for generator/movfeat.py. + +Runs without libclang or a MEOS runtime: pure dict→dict. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from generator.movfeat import build_movfeat_openapi, _OGC_MOVFEAT_ROUTES + +TEMP = "const struct Temporal *" + + +def serialized(name, ctype, decode): + return {"name": name, "kind": "serialized", "cType": ctype, + "decode": decode, "encodings": ["mfjson", "text", "wkb"]} + + +# A synthetic enriched catalog covering all the MEOS functions the MovFeat +# projection references. Functions not listed here remain unreferenced and +# unexposed; the test for the missing-function reporting uses an +# intentionally trimmed catalog. +FULL_CATALOG = { + "functions": [ + { + "name": "temporal_as_mfjson", "category": "io", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "as_mfjson", + "encodings": ["mfjson"]}, + }, + }, + { + "name": "temporal_from_mfjson", "category": "io", + "network": {"exposable": True}, + "wire": { + "params": [{"name": "json", "kind": "json", "json": "string"}], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_speed", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_cumulative_length", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "tpoint_azimuth", "category": "tpoint", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + { + "name": "temporal_derivative", "category": "tnumber", + "network": {"exposable": True}, + "wire": { + "params": [serialized("temp", TEMP, "temporal_in")], + "result": {"kind": "serialized", "cType": TEMP, + "encode": "temporal_out", + "encodings": ["mfjson", "wkb"]}, + }, + }, + ], + "typeEncodings": { + "Temporal": { + "encodings": ["mfjson", "text", "wkb"], + "in": "temporal_in", + "out": "temporal_out", + } + }, + "enums": [], +} + + +class TestMovFeat(unittest.TestCase): + + def test_paths_under_movfeat_hierarchy(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path in doc["paths"]: + self.assertTrue( + path.startswith("/collections"), + f"non-OGC path emitted: {path!r}", + ) + + def test_movfeat_tag_present(self): + doc = build_movfeat_openapi(FULL_CATALOG) + tag_names = {t["name"] for t in doc.get("tags", [])} + self.assertIn("MovingFeatures", tag_names) + + def test_meos_backed_routes_carry_x_meos_function(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path, ops in doc["paths"].items(): + for method, op in ops.items(): + if op.get("x-meos-function"): + # Sanity: should be one of our known MEOS function names. + self.assertIn( + op["x-meos-function"], + {f["name"] for f in FULL_CATALOG["functions"]}, + f"{method} {path} → x-meos-function " + f"{op['x-meos-function']!r} not in catalog", + ) + + def test_persistence_layer_routes_have_no_x_meos_function(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # /collections (no item-id, no MEOS analogue) is persistence-layer. + self.assertIsNone( + doc["paths"]["/collections"]["get"].get("x-meos-function"), + ) + self.assertIsNone( + doc["paths"]["/collections"]["post"].get("x-meos-function"), + ) + + def test_mfjson_routes_advertise_geo_json_content_type(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # getTrajectory uses temporal_as_mfjson (encode == "as_mfjson"), + # so its 200 response must advertise application/geo+json. + op = doc["paths"][ + "/collections/{collectionId}/items/{featureId}/tgsequence" + ]["get"] + content_types = set(op["responses"]["200"]["content"].keys()) + self.assertIn("application/geo+json", content_types) + self.assertIn("application/json", content_types) + + def test_meos_default_error_response_referenced_on_every_op(self): + doc = build_movfeat_openapi(FULL_CATALOG) + for path, ops in doc["paths"].items(): + for method, op in ops.items(): + self.assertIn( + "default", op["responses"], + f"{method} {path} missing default error response", + ) + self.assertEqual( + op["responses"]["default"]["$ref"], + "#/components/responses/MeosError", + ) + + def test_path_parameter_objects_have_descriptions(self): + doc = build_movfeat_openapi(FULL_CATALOG) + op = doc["paths"][ + "/collections/{collectionId}/items/{featureId}" + ]["get"] + params = {p["name"]: p for p in op.get("parameters", [])} + for name in ("collectionId", "featureId"): + self.assertIn(name, params) + self.assertTrue(params[name]["description"], + f"{name} parameter has empty description") + self.assertEqual(params[name]["in"], "path") + self.assertTrue(params[name]["required"]) + + def test_route_count_matches_manifest(self): + doc = build_movfeat_openapi(FULL_CATALOG) + # Manifest has multiple methods per path; count flattened routes. + routes = sum(len(ops) for ops in doc["paths"].values()) + self.assertEqual(routes, len(_OGC_MOVFEAT_ROUTES)) + + def test_missing_meos_function_is_reported(self): + # Strip one MEOS function the manifest references. + catalog = { + "functions": [ + f for f in FULL_CATALOG["functions"] + if f["name"] != "tpoint_speed" + ], + "typeEncodings": FULL_CATALOG["typeEncodings"], + "enums": FULL_CATALOG["enums"], + } + doc = build_movfeat_openapi(catalog) + coverage = doc["info"]["x-meos-coverage"] + self.assertIn("tpoint_speed", coverage["missing_in_catalog"]) + + def test_meos_coverage_summary_arithmetic(self): + doc = build_movfeat_openapi(FULL_CATALOG) + cov = doc["info"]["x-meos-coverage"] + # routes == meos_backed + persistence_only by construction + self.assertEqual( + cov["routes"], cov["meos_backed"] + cov["persistence_only"], + f"coverage arithmetic mismatch: {cov}", + ) + + def test_openapi_3_1_top_level_shape(self): + doc = build_movfeat_openapi(FULL_CATALOG) + self.assertEqual(doc["openapi"], "3.1.0") + self.assertIn("info", doc) + self.assertIn("paths", doc) + self.assertIn("components", doc) + self.assertIn("schemas", doc["components"]) + self.assertIn("responses", doc["components"]) + self.assertIn("MeosError", doc["components"]["responses"]) + + def test_component_schemas_share_with_generic_openapi(self): + # Sanity: a referenced opaque type produces a component schema + # carrying the typeEncodings metadata. Same helper as the generic + # OpenAPI generator, so the two projections expose byte-identical + # schemas for shared types. + doc = build_movfeat_openapi(FULL_CATALOG) + schemas = doc["components"]["schemas"] + if "Temporal" in schemas: + ts = schemas["Temporal"] + self.assertEqual(ts["title"], "Temporal") + self.assertIn("x-meos-encodings", ts) + self.assertIn("x-meos-in", ts) + self.assertIn("x-meos-out", ts) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_nullable.py b/tests/test_nullable.py new file mode 100644 index 0000000..0d37bc4 --- /dev/null +++ b/tests/test_nullable.py @@ -0,0 +1,75 @@ +"""Regression tests for parser/nullable.py. + +Nullability is read from the C Doxygen `@param ... may be NULL` notes (the +source of truth) and folded into each function's ``shape.nullable`` for the +params that actually exist on the IDL function. + +Plain unittest, no pytest dependency; writes a tiny synthetic source tree. +""" +import tempfile +import unittest +from pathlib import Path + +from parser.nullable import extract_nullable, merge_nullable + +SAMPLE = ''' +/** + * @ingroup meos_temporal_inout + * @brief Return the MF-JSON representation + * @param[in] temp Temporal value + * @param[in] srs Spatial reference system, may be `NULL` + */ +char * +temporal_as_mfjson(const Temporal *temp, char *srs) +{ + return NULL; +} + +/** + * @brief Append an instant + * @param[in] inst Instant + * @param[in] maxt Maximum time interval, may be `NULL` + * @param[in] interp Interpolation + */ +Temporal * +temporal_append_tinstant(const TInstant *inst, const Interval *maxt, int interp) +{ + return NULL; +} +''' + + +class NullableTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + src = Path(self.tmp.name) / "src" + src.mkdir() + (src / "sample.c").write_text(SAMPLE) + (Path(self.tmp.name) / "include").mkdir() + + def tearDown(self): + self.tmp.cleanup() + + def test_extracts_only_may_be_null_params(self): + nul = extract_nullable(self.tmp.name) + self.assertEqual(nul["temporal_as_mfjson"], ["srs"]) + self.assertEqual(nul["temporal_append_tinstant"], ["maxt"]) + # `temp`, `inst`, `interp` carry no NULL note -> not nullable + self.assertNotIn("temp", nul["temporal_as_mfjson"]) + + def test_merge_only_existing_params(self): + idl = {"functions": [ + {"name": "temporal_as_mfjson", + "params": [{"name": "temp"}, {"name": "srs"}]}, + # function whose nullable param is NOT in its IDL signature + {"name": "temporal_append_tinstant", "params": [{"name": "inst"}]}, + ]} + idl, n = merge_nullable(idl, self.tmp.name) + self.assertEqual(idl["functions"][0]["shape"]["nullable"], ["srs"]) + # maxt absent from the IDL params -> not added + self.assertNotIn("shape", idl["functions"][1]) + self.assertEqual(n, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_openapi.py b/tests/test_openapi.py new file mode 100644 index 0000000..2e33ff8 --- /dev/null +++ b/tests/test_openapi.py @@ -0,0 +1,183 @@ +"""Unit tests for generator/openapi.py. + +Runs without libclang or pytest: python3 tests/test_openapi.py +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from generator.openapi import build_openapi + +TEMP = "const struct Temporal *" + + +def serialized(name, ctype, decode): + return {"name": name, "kind": "serialized", "cType": ctype, + "decode": decode, "encodings": ["mfjson", "text", "wkb"]} + + +CATALOG = { + "functions": [ + { # serialized params, scalar result + "name": "temporal_eq", "category": "predicate", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": { + "params": [serialized("temp1", TEMP, "temporal_in"), + serialized("temp2", TEMP, "temporal_in")], + "result": {"kind": "json", "json": "integer"}, + }, + }, + { # enum param, serialized result + "name": "temporal_set_interp", "category": "transformation", + "doc": "Set the interpolation of a temporal value.", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": { + "params": [ + serialized("temp", TEMP, "temporal_in"), + {"name": "interp", "kind": "json", "json": "string", + "enum": "interpType"}, + ], + "result": {"kind": "serialized", + "cType": "struct Temporal *", + "encode": "temporal_out", + "encodings": ["mfjson", "text", "wkb"]}, + }, + }, + { # no params, void result + "name": "noop_op", "category": "transformation", + "network": {"exposable": True, "method": "POST", "reason": None}, + "wire": {"params": [], "result": {"kind": "void"}}, + }, + { # input-array builder + "name": "temporal_merge_array", "category": "transformation", + "network": {"exposable": True, "method": "POST", + "reason": None}, + "wire": { + "params": [{"name": "temparr", "kind": "array", + "count_param": "count", + "element": {"kind": "serialized", + "cType": "struct Temporal *", + "decode": "temporal_in", + "encodings": ["text"]}}], + "result": {"kind": "serialized", + "cType": "struct Temporal *", + "encode": "temporal_out", + "encodings": ["text"]}, + }, + }, + { # not exposable -> excluded + "name": "tsequence_make", "category": "constructor", + "network": {"exposable": False, "method": None, + "reason": "array-or-out-param:instants"}, + "wire": {"params": [], "result": {"kind": "unsupported"}}, + }, + ], + "typeEncodings": { + "Temporal": {"encodings": ["mfjson", "text", "wkb"], + "in": "temporal_in", "out": "temporal_out"}, + }, + "enums": [{"name": "interpType", + "values": [{"name": "STEP", "value": 0}, + {"name": "LINEAR", "value": 1}]}], + "structs": [], +} + + +class OpenApiTests(unittest.TestCase): + def setUp(self): + self.spec = build_openapi(CATALOG, version="9.9.9") + + def test_envelope(self): + self.assertEqual(self.spec["openapi"], "3.1.0") + self.assertEqual(self.spec["info"]["version"], "9.9.9") + self.assertEqual(self.spec["info"]["x-meos-coverage"], + {"functions": 5, "exposed": 4}) + + def test_array_param(self): + op = self.spec["paths"]["/temporal_merge_array"]["post"] + sch = op["requestBody"]["content"]["application/json"]["schema"] + a = sch["properties"]["temparr"] + self.assertEqual(a["type"], "array") + self.assertEqual(a["items"], + {"$ref": "#/components/schemas/Temporal"}) + self.assertEqual(a["x-meos-decode"], "temporal_in") + + def test_non_exposable_excluded(self): + self.assertNotIn("/tsequence_make", self.spec["paths"]) + self.assertEqual(len(self.spec["paths"]), 4) + + def test_paths_sorted(self): + keys = list(self.spec["paths"]) + self.assertEqual(keys, sorted(keys)) + + def test_predicate_operation(self): + op = self.spec["paths"]["/temporal_eq"]["post"] + self.assertEqual(op["operationId"], "temporal_eq") + self.assertEqual(op["tags"], ["predicate"]) + self.assertEqual(op["x-meos-category"], "predicate") + body = op["requestBody"]["content"]["application/json"]["schema"] + self.assertEqual(body["required"], ["temp1", "temp2"]) + self.assertFalse(body["additionalProperties"]) + temp1 = body["properties"]["temp1"] + self.assertEqual(temp1["allOf"], + [{"$ref": "#/components/schemas/Temporal"}]) + self.assertEqual(temp1["x-meos-decode"], "temporal_in") + r200 = op["responses"]["200"]["content"]["application/json"]["schema"] + self.assertEqual(r200, {"type": "integer"}) + self.assertEqual(op["responses"]["default"], + {"$ref": "#/components/responses/MeosError"}) + + def test_enum_param_and_serialized_result(self): + op = self.spec["paths"]["/temporal_set_interp"]["post"] + self.assertEqual(op["summary"], + "Set the interpolation of a temporal value.") + props = op["requestBody"]["content"]["application/json"]["schema"][ + "properties"] + self.assertEqual(props["interp"], + {"$ref": "#/components/schemas/interpType"}) + self.assertEqual(op["x-meos-encode"], "temporal_out") + r = op["responses"]["200"]["content"]["application/json"]["schema"] + self.assertEqual(r, {"$ref": "#/components/schemas/Temporal"}) + + def test_void_operation(self): + op = self.spec["paths"]["/noop_op"]["post"] + self.assertNotIn("requestBody", op) + self.assertIn("204", op["responses"]) + + def test_components(self): + schemas = self.spec["components"]["schemas"] + self.assertEqual(schemas["Temporal"]["type"], "string") + self.assertEqual(schemas["Temporal"]["x-meos-in"], "temporal_in") + self.assertEqual(schemas["Temporal"]["x-meos-encodings"], + ["mfjson", "text", "wkb"]) + self.assertEqual(schemas["interpType"]["enum"], ["STEP", "LINEAR"]) + self.assertTrue(schemas["interpType"]["x-meos-c-enum"]) + self.assertIn("MeosError", self.spec["components"]["responses"]) + + def test_all_refs_resolve(self): + import json + schemas = self.spec["components"]["schemas"] + responses = self.spec["components"]["responses"] + for ref in self._refs(self.spec): + parts = ref.split("/") # #/components// + kind, name = parts[2], parts[3] + target = schemas if kind == "schemas" else responses + self.assertIn(name, target, f"dangling $ref {ref}") + + def _refs(self, node): + if isinstance(node, dict): + for k, v in node.items(): + if k == "$ref": + yield v + else: + yield from self._refs(v) + elif isinstance(node, list): + for v in node: + yield from self._refs(v) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..213a921 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,72 @@ +"""Unit tests for report.py. python3 tests/test_report.py""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from report import build_report + + +def f(name, api, exposable, reason=None, ret="int", params=None): + return {"name": name, "api": api, + "returnType": {"canonical": ret}, "params": params or [], + "network": {"exposable": exposable, "reason": reason}} + + +CATALOG = {"functions": [ + f("temporal_eq", "public", True), + f("tpoint_speed", "public", True), + f("tsequence_make", "public", False, "array-or-out-param:instants"), + f("geo_collect", "public", False, + "array-or-out-param:a; no-decoder:GBOX"), + f("temporal_tagg", "public", False, "no-decoder:SkipList"), + f("meos_initialize", "public", False, "lifecycle"), + f("some_internal", "internal", False, "internal; no-decoder:Datum"), +]} + + +class ReportTests(unittest.TestCase): + def setUp(self): + self.r = build_report(CATALOG) + + def test_counts(self): + self.assertEqual(self.r["publicTotal"], 6) # excludes internal + self.assertEqual(self.r["exposable"], 2) + self.assertEqual(self.r["gap"], 4) + self.assertEqual(self.r["internalExcluded"], 1) + self.assertEqual(self.r["coveragePct"], round(2 * 100 / 6, 1)) + + def test_grouping_by_reason_tagset(self): + br = self.r["byReason"] + # detail is stripped; multi-tag reasons collapse to the tag set + self.assertIn("array-or-out-param", br) + self.assertEqual(br["array-or-out-param"], ["tsequence_make"]) + self.assertEqual(br["array-or-out-param; no-decoder"], ["geo_collect"]) + self.assertEqual(br["no-decoder"], ["temporal_tagg"]) + self.assertEqual(br["lifecycle"], ["meos_initialize"]) + # internal never appears as a public gap + self.assertNotIn("some_internal", + [n for v in br.values() for n in v]) + + def test_byreason_sorted_largest_first(self): + sizes = [len(v) for v in self.r["byReason"].values()] + self.assertEqual(sizes, sorted(sizes, reverse=True)) + + def test_worklist_is_actionable(self): + wl = {w["name"]: w for w in self.r["worklist"]} + self.assertEqual(len(wl), self.r["gap"]) # one entry per gap + self.assertNotIn("some_internal", wl) # internal excluded + # each gap gets a class + a concrete upstream suggestion + self.assertEqual(wl["meos_initialize"]["class"], "plumbing") + self.assertEqual(wl["temporal_tagg"]["class"], "stateful") + gc = wl["geo_collect"] # no-decoder:GBOX + self.assertEqual(gc["class"], "no-codec") + self.assertIn("gbox_in", gc["suggest"]) + self.assertTrue(all(w["suggest"] for w in self.r["worklist"])) + self.assertEqual(sum(self.r["byClass"].values()), self.r["gap"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_shapeinfer.py b/tests/test_shapeinfer.py new file mode 100644 index 0000000..592cd41 --- /dev/null +++ b/tests/test_shapeinfer.py @@ -0,0 +1,70 @@ +"""Regression tests for parser/shapeinfer.py. + +The inferer derives array-output shape from the C signatures, replacing the +hand-maintained meta stub. The discriminator is the *count* parameter's form: + +* a written-back out-array pairs with a by-pointer ``int *count`` (the callee + fills the length) -> ``outputArrays`` + ``arrayReturn.lengthFrom`` +* a read-only in-array pairs with a by-value ``int count`` -> left untouched + +Plain unittest, no pytest dependency; fully synthetic IDL (no generated file). +""" +import unittest + +from parser.shapeinfer import infer_shapes + + +def _fn(name, ret, params): + return {"name": name, + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t} for n, t in params]} + + +class ShapeInferTests(unittest.TestCase): + def test_output_array_with_pointer_count(self): + # temporal_time_split-style: non-const ** out-array + by-pointer count + idl = {"functions": [_fn( + "temporal_time_split", "Temporal **", + [("temp", "const Temporal *"), ("duration", "const Interval *"), + ("torigin", "TimestampTz"), ("time_bins", "TimestampTz **"), + ("count", "int *")])]} + idl, stats = infer_shapes(idl) + sh = idl["functions"][0]["shape"] + self.assertEqual(sh["outputArrays"], [{"param": "time_bins"}]) + self.assertEqual(sh["arrayReturn"]["lengthFrom"], + {"kind": "param", "name": "count"}) + self.assertEqual(stats["outputArrays"], 1) + + def test_two_parallel_output_arrays(self): + idl = {"functions": [_fn( + "tfloat_value_time_split", "Temporal **", + [("temp", "const Temporal *"), ("vsize", "double"), + ("value_bins", "double **"), ("time_bins", "TimestampTz **"), + ("count", "int *")])]} + idl, _ = infer_shapes(idl) + self.assertEqual(idl["functions"][0]["shape"]["outputArrays"], + [{"param": "value_bins"}, {"param": "time_bins"}]) + + def test_input_array_with_value_count_untouched(self): + # tsequence_make-style: ** input array carries its length BY VALUE + idl = {"functions": [_fn( + "tsequence_make", "TSequence *", + [("instants", "const TInstant **"), ("count", "int"), + ("lower_inc", "bool")])]} + idl, stats = infer_shapes(idl) + self.assertNotIn("shape", idl["functions"][0]) + self.assertEqual(stats["outputArrays"], 0) + + def test_nonconst_input_array_with_value_count_untouched(self): + # tsequenceset_make_gaps-style: non-const ** but BY-VALUE count => input + idl = {"functions": [_fn( + "tsequenceset_make_gaps", "TSequenceSet *", + [("instants", "TInstant **"), ("count", "int"), + ("maxt", "const Interval *")])]} + idl, stats = infer_shapes(idl) + self.assertEqual(stats["outputArrays"], 0) + self.assertNotIn("shape", idl["functions"][0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_typerecover.py b/tests/test_typerecover.py index ed10993..de68f07 100644 --- a/tests/test_typerecover.py +++ b/tests/test_typerecover.py @@ -81,6 +81,22 @@ def test_no_gserialized_left_collapsed_to_int(self): self.assertGreater(len(geo_fns), 50, "GSERIALIZED* collapsed toward int — typerecover regression?") + # ---- jsonb Jsonb / jsonpath JsonPath (opaque PG types, collapse to int) - + + def test_jsonb_recovered_when_tjsonb_present(self): + # The temporal-JSONB surface is built only when MEOS is compiled with + # JSON=ON, so this assertion is conditional on the parsed source + # carrying it (skipped otherwise to stay source-agnostic). + jsonb_fns = [n for n in self.by_name + if "jsonb" in n.lower() or "tjsonb" in n.lower()] + if not jsonb_fns: + self.skipTest("source parsed without the JSON=ON tjsonb surface") + # An out-parameter that pre-fix came back as ``int **``. + self.assertIn("Jsonb **", self._param_ctypes("jsonbset_value_n")) + carriers = [f for f in self.by_name.values() if "Jsonb *" in json.dumps(f)] + self.assertGreater(len(carriers), 20, + "Jsonb* collapsed toward int — typerecover regression?") + # ---- other PG-vendored opaque types (Interval / DateADT / Datum / ...) - def test_interval_params_recovered(self): diff --git a/verify-train.sh b/verify-train.sh new file mode 100755 index 0000000..ea72ba7 --- /dev/null +++ b/verify-train.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# verify-train.sh - compose the dependency-ordered integration train +# (meta/integration-train.json) and run each wave's gate, so ecosystem-wide +# 100% parity is PROVABLE at one point instead of asserted per-PR. +# +# Operationalizes MobilityDB discussion #895 (wave-based merge plan). +# +# Honesty contract: a wave reports PASS only when its gate is just-run and +# green here; otherwise BLOCKED: with the exact gate it needs. +# Nothing is faked, nothing is skipped silently (green-ci-same-commit). +# +# Scope/safety: operates ONLY on this repo and its own ./_mobilitydb +# sparse checkout. NEVER touches a shared PyMEOS / MobilityDB working copy +# (work-independently-parallel-sessions). +# +# Usage: +# ./verify-train.sh # Wave 0 (fully automatable here) + honest +# # BLOCKED status for Waves 1-3 +# PYMEOS_ENV=/path ./verify-train.sh # also run Wave 2/3 gates if a +# # bump-ready PyMEOS env is provided +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +cd "$HERE" +MDB="_mobilitydb" +RC=0 +say() { printf '\n=== %s ===\n' "$*"; } +pass() { printf ' [PASS] %s\n' "$*"; } +fail() { printf ' [FAIL] %s\n' "$*"; RC=1; } +block() { printf ' [BLOCKED] %s\n' "$*"; } + +# --------------------------------------------------------------------------- +say "WAVE 0 - MobilityDB core extended-type C surface (compose + gate)" +# Compose: linear stack tip (#1085 contains #1081..#1085) + the clean +# #1051->#951 pair cherry-picked on top (proven clean). +if [ ! -d "$MDB/.git" ]; then + fail "no $MDB checkout - run: python3 setup.py (then re-run)" +else + git -C "$MDB" fetch --no-tags --depth=20 origin \ + pull/1085/head pull/1051/head pull/951/head >/dev/null 2>&1 + STACK_TIP=$(git -C "$MDB" rev-parse FETCH_HEAD 2>/dev/null) # last fetched + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1085/head >/dev/null 2>&1 + STACK=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/1051/head >/dev/null 2>&1 + P1051=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" fetch --no-tags --depth=20 origin pull/951/head >/dev/null 2>&1 + P951=$(git -C "$MDB" rev-parse FETCH_HEAD) + git -C "$MDB" checkout -q -B _train_w0 "$STACK" 2>/dev/null + CK=0 + for c in $(git -C "$MDB" rev-list --reverse "$(git -C "$MDB" rev-parse "$P951"^^)..$P951"); do + git -C "$MDB" cherry-pick -x "$c" >/dev/null 2>&1 || { git -C "$MDB" cherry-pick --abort >/dev/null 2>&1; CK=1; } + done + [ "$CK" = 0 ] && pass "composed: stack #1081..#1085 + #1051->#951 (clean)" \ + || fail "Wave-0 compose conflict (#1051/#951 pair vs stack)" + # Sync headers (replicate setup.py step_sync, no git reset). + python3 - <<'PY' +import shutil, pathlib +src=pathlib.Path("_mobilitydb/meos/include"); dst=pathlib.Path("meos/include") +stub={"pg_config.h","postgres_int_defs.h","postgres_ext_defs.in.h"} +[shutil.copy2(h,dst/h.name) for h in src.glob("*.h") if h.name not in stub] +PY + if ! python3 -c 'import json,sys; json.load(open("meta/object-model.json"))' 2>/dev/null; then + fail "MEOS-API catalog anchor missing: this branch must be stacked on feat/object-model (PR #10). See meta/integration-train.json#/catalog_anchor." + fi + mkdir -p output + python3 run.py >/dev/null 2>"output/.train_run.log" \ + && pass "MEOS-API catalog regenerated" \ + || fail "run.py failed (see output/.train_run.log)" + python3 - <<'PY' +import json,sys +om=json.load(open("output/meos-idl.json"))["objectModel"] +f2c=om["functionToClass"]; n=om["summary"]["functionsTotal"] +exp={"tcbuffer_from_mfjson":"TCbuffer","tnpoint_from_mfjson":"TNpoint", + "tpose_from_mfjson":"TPose","tcbufferinst_make":"TCbufferInst", + "tposeinst_make":"TPoseInst"} +bad=[k for k,v in exp.items() if (f2c.get(k) or {}).get("class")!=v] +sys.exit(0 if (n==2699 and not bad) else 1) +PY + [ $? = 0 ] && pass "Wave-0 gate: 2699 fns, from_mfjson + ctors uniform" \ + || fail "Wave-0 gate: catalog count/classification mismatch" + python3 -m pytest tests/ -q >"output/.train_pytest.log" 2>&1 \ + && grep -q "21 passed" "output/.train_pytest.log" \ + && pass "Wave-0 gate: PR #10 object-model 21/21" \ + || fail "Wave-0 gate: PR #10 tests not 21/21 (see output/.train_pytest.log)" +fi + +# --------------------------------------------------------------------------- +say "WAVE 1 - PyMEOS-CFFI MEOS-1.4 substrate" +block "needs Wave-1 env: regenerate PyMEOS-CFFI builder/meos-idl.json vs Wave-0 MEOS, then pymeos_cffi import (owner: live PyMEOS session)" + +say "WAVE 2 - CRITICAL PATH: PyMEOS master health (MEOS-1.4 bump #81/#82)" +block "PyMEOS master broken vs MEOS master (geoset_*->spatialset_*, tpoint_*->tspatial_*, pgis_geometry_in->geom_in, spanset_make arity); pristine master 426 failed/2721 passed. Gate: test.yml 426->0 vs Wave-1/Wave-0. Owner: live PyMEOS session. Build the bump against the composed Wave-0, not bare master." + +say "WAVE 3 - PyMEOS features (gated on Wave 2)" +if [ -n "${PYMEOS_ENV:-}" ] && [ -d "$PYMEOS_ENV" ]; then + block "PYMEOS_ENV given - run per-PR gates there: #85 black; #87 portable_parity 0-unbacked; #88 collect; #89 codegen --check; #90/#91 mixin suites; #84 tests/io 4/4 + footer==MobilityDuck#146 (manifest has exact gates)" +else + block "set PYMEOS_ENV= to run #85/#87/#88/#89/#90/#91/#84 gates; all are green-in-isolation and gated only on Wave 2" +fi + +# --------------------------------------------------------------------------- +say "SUMMARY" +if [ "$RC" = 0 ]; then + echo " Wave 0 PROVEN here. Waves 1-3 gated solely on Wave 2 (the MEOS-1.4 bump)." + echo " 100% parity is demonstrable the moment Wave 2 lands and this train" + echo " is re-run with PYMEOS_ENV set. Merge order: see meta/integration-train.json." +else + echo " Wave 0 gate FAILED above - parity train cannot proceed; fix before merge." +fi +exit "$RC"