From c8f66b136657fa64e6d524ad06d1311f90829a50 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 6 Aug 2026 11:00:28 +0200 Subject: [PATCH 1/7] Accept raw event IDs and make workflow tests self-contained --- agent-docs/current-work.md | 12 ++ src/ams_smartabase/workflow.py | 7 +- tests/fixtures/workflow/README.md | 11 ++ tests/fixtures/workflow/config.json | 12 ++ .../training load template synthetic.csv | 4 + tests/test_workflow.py | 149 ++++++++++++------ 6 files changed, 149 insertions(+), 46 deletions(-) create mode 100644 tests/fixtures/workflow/README.md create mode 100644 tests/fixtures/workflow/config.json create mode 100644 tests/fixtures/workflow/training load template synthetic.csv diff --git a/agent-docs/current-work.md b/agent-docs/current-work.md index 7b88d8c..0f5724e 100644 --- a/agent-docs/current-work.md +++ b/agent-docs/current-work.md @@ -169,6 +169,18 @@ or on an independent machine. ## Recent Session Summary +2026-08-06: + +- Replaced the workflow tests' dependency on ignored local + `use_case_examples/synthetic_data` files with a committed, deterministic + fixture pair under `tests/fixtures/workflow`. +- The fixture contains fictional test-only values and is passed explicitly to + `load_example_event_workflow_input()`; the ignored operational example + directory remains excluded from version control. +- The 13 previously known missing-fixture failures are resolved. All 21 + workflow tests pass, and the full connector suite passes 144 tests with the + opt-in live test skipped. No live AMS request or mutation was performed. + 2026-08-05: - Corrected roster parsing for the official nested user-result batch shape. diff --git a/src/ams_smartabase/workflow.py b/src/ams_smartabase/workflow.py index 838f039..fa76e95 100644 --- a/src/ams_smartabase/workflow.py +++ b/src/ams_smartabase/workflow.py @@ -25,7 +25,12 @@ DEFAULT_EXAMPLE_CONFIG = Path("use_case_examples/synthetic_data/config.json") DEFAULT_EXAMPLE_CSV = Path("use_case_examples/synthetic_data/csv/training load template 1777445863459.csv") -EVENT_ID_KEYS = EXACT_EVENT_ID_KEYS +# Smartabase event-export responses use the generic field name `id` for the +# event's stable ID. Accept it only while parsing known event records here. +# Keep the shared diagnostics keys stricter: diagnostics inspects several +# response types, where an unqualified `id` may identify something other than +# an event and must never accidentally become a deletion target. +EVENT_ID_KEYS = EXACT_EVENT_ID_KEYS + ("id",) _TIME_COLUMN = "Time" _DATE_COLUMN = "Date" _NAME_COLUMNS = ("First Name", "Last Name") diff --git a/tests/fixtures/workflow/README.md b/tests/fixtures/workflow/README.md new file mode 100644 index 0000000..f94cc0d --- /dev/null +++ b/tests/fixtures/workflow/README.md @@ -0,0 +1,11 @@ +# Workflow test fixtures + +These files contain deterministic, fictional data created only for automated +connector tests. They are not exports from Teamworks AMS and do not contain +credentials, real athlete information, tenant identifiers, or operational +Smartabase data. + +Workflow tests pass these paths explicitly to +`load_example_event_workflow_input()`. The ignored `use_case_examples/` +directory remains reserved for local operational examples and must not be used +as a test dependency. diff --git a/tests/fixtures/workflow/config.json b/tests/fixtures/workflow/config.json new file mode 100644 index 0000000..c65fff6 --- /dev/null +++ b/tests/fixtures/workflow/config.json @@ -0,0 +1,12 @@ +{ + "athletes": [ + { + "first_name": "Taylor", + "last_name": "Example", + "about": "Taylor Example", + "smartabase_user_id": 100001, + "username": "taylor.example", + "email": "taylor@example.invalid" + } + ] +} diff --git a/tests/fixtures/workflow/training load template synthetic.csv b/tests/fixtures/workflow/training load template synthetic.csv new file mode 100644 index 0000000..e9e3fbc --- /dev/null +++ b/tests/fixtures/workflow/training load template synthetic.csv @@ -0,0 +1,4 @@ +First Name,Last Name,Date,Time,Form ID,Session RPE,Duration +Taylor,Example,01/05/2026,09:00,SYNTH-LOAD-001,5,60 +Taylor,Example,02/05/2026,10:30,SYNTH-LOAD-002,6,75 +Taylor,Example,03/05/2026,08:15,SYNTH-LOAD-003,4,45 diff --git a/tests/test_workflow.py b/tests/test_workflow.py index c00e945..7f7be2b 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -24,6 +24,18 @@ ) +_WORKFLOW_FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "workflow" +_WORKFLOW_FIXTURE_CONFIG = _WORKFLOW_FIXTURE_DIR / "config.json" +_WORKFLOW_FIXTURE_CSV = _WORKFLOW_FIXTURE_DIR / "training load template synthetic.csv" + + +def _load_synthetic_workflow_target(): + return load_example_event_workflow_input( + csv_path=_WORKFLOW_FIXTURE_CSV, + config_path=_WORKFLOW_FIXTURE_CONFIG, + ) + + class ReplayClient: def __init__(self, target, *, sandbox=True): package = build_event_import_payloads(target.records, form=target.form, mode="insert") @@ -119,6 +131,7 @@ def insert_event(self, records, *, form=None, entered_by_user_id=None, dry_run=T class StaticEventClient: def __init__(self, payload): self.payload = payload + self.delete_calls = 0 def get_event( self, @@ -132,6 +145,10 @@ def get_event( ): return self.payload + def delete_event(self, event_ids, *, dry_run=True, confirm=False): + self.delete_calls += 1 + raise AssertionError("delete_event must not be called after exact-ID validation fails") + class WorkflowTests(unittest.TestCase): def test_build_event_write_targets_groups_records_by_form_and_user(self): @@ -153,7 +170,7 @@ def test_build_event_write_targets_groups_records_by_form_and_user(self): self.assertEqual(targets[1].about, "Ada Lovelace") def test_count_event_entries_extracts_ids_from_realistic_eventsearch_payload(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) result = count_event_entries(client, form=target.form, user_id=target.user_id, date_range=target.date_range) @@ -195,43 +212,85 @@ def test_count_event_entries_extracts_ids_from_nested_result_batches(self): self.assertEqual([row["Score"] for row in result.rows], ["5", "6"]) def test_count_event_entries_rejects_lossy_or_nonpositive_event_ids(self): - for value in (True, 1.9, 0, -1, "1.9", "+1"): - with self.subTest(value=value): - result = count_event_entries( - StaticEventClient({"events": [{"eventId": value}]}), - form="Synthetic Wellness", - user_id=1, - date_range=("01/05/2026", "01/05/2026"), - ) - self.assertEqual(result.entry_count, 1) - self.assertEqual(result.event_ids, []) - - generic = count_event_entries( - StaticEventClient({"events": [{"id": 7, "formName": "Synthetic Wellness"}]}), - form="Synthetic Wellness", + for id_key in ("eventId", "id"): + for value in (True, 1.9, 0, -1, "1.9", "+1"): + with self.subTest(id_key=id_key, value=value): + event = { + id_key: value, + "userId": 1, + "formName": "Synthetic Wellness", + "startDate": "01/05/2026", + } + result = count_event_entries( + StaticEventClient({"events": [event]}), + form="Synthetic Wellness", + user_id=1, + date_range=("01/05/2026", "01/05/2026"), + ) + self.assertEqual(result.entry_count, 1) + self.assertEqual(result.event_ids, []) + + def test_exact_event_id_requirement_accepts_raw_eventsearch_id(self): + result = count_event_entries( + StaticEventClient( + { + "events": [ + { + "id": 7, + "userId": 1, + "formName": "WSV Wellness", + "startDate": "01/05/2026", + } + ] + } + ), + form="WSV Wellness", user_id=1, date_range=("01/05/2026", "01/05/2026"), ) - self.assertEqual(generic.entry_count, 1) - self.assertEqual(generic.event_ids, []) - def test_exact_event_id_requirement_rejects_generic_id_without_deleting(self): - result = count_event_entries( - StaticEventClient( - {"events": [{"id": 7, "athleteId": 1, "formName": "Synthetic Wellness"}]} - ), + self.assertEqual(result.event_ids, [7]) + self.assertEqual(require_exact_event_ids(result), [7]) + + def test_count_event_entries_rejects_an_id_only_object(self): + with self.assertRaisesRegex(ValueError, "recognizable event objects"): + count_event_entries( + StaticEventClient({"events": [{"id": 7}]}), + form="Synthetic Wellness", + user_id=1, + date_range=("01/05/2026", "01/05/2026"), + ) + + def test_invalid_raw_event_id_stops_delete_workflow_before_deletion(self): + target = SimpleNamespace( + csv_path=Path("unused.csv"), form="Synthetic Wellness", user_id=1, + about="Synthetic Athlete", + username="synthetic.athlete", + email="synthetic@example.invalid", date_range=("01/05/2026", "01/05/2026"), + expected_upload_count=1, + ) + client = StaticEventClient( + { + "events": [ + { + "id": True, + "userId": target.user_id, + "formName": target.form, + "startDate": target.date_range[0], + } + ] + } ) - with self.assertRaises(AMSEventIdUnavailableError) as raised: - require_exact_event_ids(result) + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(AMSEventIdUnavailableError) as raised: + run_event_delete_workflow(client, target, base_dir=temp_dir) - self.assertEqual(raised.exception.code, "exact_event_id_unavailable") self.assertFalse(raised.exception.request_sent) - self.assertIn("athleteId", raised.exception.details["returned_id_like_fields"]) - self.assertIn("No deletion request was sent", str(raised.exception)) + self.assertEqual(client.delete_calls, 0) def test_exact_event_id_requirement_rejects_unrecognized_response_shape(self): result = EventCountResult( @@ -271,19 +330,19 @@ def test_deletion_planning_rejects_mixed_nested_event_shape(self): with self.assertRaisesRegex(ValueError, "direct records and nested result batches were mixed"): plan_event_deletions(client, records) - def test_load_example_event_workflow_input_uses_example_training_load_data(self): - target = load_example_event_workflow_input() + def test_load_example_event_workflow_input_uses_committed_synthetic_data(self): + target = _load_synthetic_workflow_target() self.assertEqual(target.form, "Training Load") - self.assertEqual(target.user_id, 60521) - self.assertEqual(target.about, "NOC01 MaleAthlete") - self.assertEqual(target.date_range, ("14/04/2026", "08/06/2026")) - self.assertEqual(target.expected_upload_count, 69) + self.assertEqual(target.user_id, 100001) + self.assertEqual(target.about, "Taylor Example") + self.assertEqual(target.date_range, ("01/05/2026", "03/05/2026")) + self.assertEqual(target.expected_upload_count, 3) self.assertEqual(target.records[0]["start_time"], "09:00") - self.assertEqual(target.records[0]["Form ID"], "5000001") + self.assertEqual(target.records[0]["Form ID"], "SYNTH-LOAD-001") def test_run_event_replay_workflow_counts_delete_and_reinsert_for_example_target(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) with tempfile.TemporaryDirectory() as tmp: @@ -311,7 +370,7 @@ def test_run_event_replay_workflow_counts_delete_and_reinsert_for_example_target self.assertIn("count_after_upload", manifest_path.read_text(encoding="utf-8")) def test_run_event_replay_workflow_warns_when_more_existing_events_are_deleted_than_uploaded(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) duplicate = dict(client.events[0]) duplicate["eventId"] = client.next_event_id @@ -336,7 +395,7 @@ def test_run_event_replay_workflow_warns_when_more_existing_events_are_deleted_t self.assertIn("same number of datapoints is uploaded", str(caught[0].message)) def test_run_event_replay_workflow_preserves_unmatched_events_in_same_date_range(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) unrelated = dict(client.events[0]) unrelated["eventId"] = client.next_event_id @@ -359,7 +418,7 @@ def test_run_event_replay_workflow_preserves_unmatched_events_in_same_date_range self.assertEqual(summary["counts"]["after_upload"]["entry_count"], target.expected_upload_count + 1) def test_run_event_replay_workflow_can_delete_all_events_in_range_in_sandbox(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) unrelated = dict(client.events[0]) unrelated["eventId"] = client.next_event_id @@ -386,7 +445,7 @@ def test_run_event_replay_workflow_can_delete_all_events_in_range_in_sandbox(sel self.assertEqual(summary["counts"]["after_upload"]["entry_count"], target.expected_upload_count) def test_run_event_replay_workflow_rejects_delete_all_in_range_outside_sandbox(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target, sandbox=False) with self.assertRaisesRegex(PermissionError, "delete_all_in_range"): @@ -399,7 +458,7 @@ def test_run_event_replay_workflow_rejects_delete_all_in_range_outside_sandbox(s ) def test_plan_event_deletions_summarizes_targeted_deletes(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) unrelated = dict(client.events[0]) unrelated["eventId"] = client.next_event_id @@ -416,7 +475,7 @@ def test_plan_event_deletions_summarizes_targeted_deletes(self): self.assertEqual(plans[0]["unmatched_existing_count"], 1) def test_plan_event_deletions_supports_full_range_mode(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) duplicate = dict(client.events[0]) duplicate["eventId"] = client.next_event_id @@ -430,7 +489,7 @@ def test_plan_event_deletions_supports_full_range_mode(self): self.assertTrue(plans[0]["delete_all_for_targets"]) def test_run_event_replace_workflow_replaces_records_for_generic_inputs(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) unrelated = dict(client.events[0]) unrelated["eventId"] = client.next_event_id @@ -457,7 +516,7 @@ def test_run_event_replace_workflow_replaces_records_for_generic_inputs(self): self.assertEqual(summary["counts"]["after_upload"][0]["entry_count"], target.expected_upload_count + 1) def test_run_event_delete_workflow_deletes_without_uploading(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) with tempfile.TemporaryDirectory() as tmp: @@ -478,7 +537,7 @@ def test_run_event_delete_workflow_deletes_without_uploading(self): self.assertEqual(client.insert_calls, 0) def test_run_event_delete_workflow_can_delete_all_events_in_range_in_sandbox(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target) unrelated = dict(client.events[0]) unrelated["eventId"] = client.next_event_id @@ -505,7 +564,7 @@ def test_run_event_delete_workflow_can_delete_all_events_in_range_in_sandbox(sel self.assertEqual(len(caught), 1) def test_run_event_delete_workflow_rejects_delete_all_in_range_outside_sandbox(self): - target = load_example_event_workflow_input() + target = _load_synthetic_workflow_target() client = ReplayClient(target, sandbox=False) with self.assertRaisesRegex(PermissionError, "delete_all_in_range"): From c45d1732c481e9d62cd7224675ead1cc7936a39c Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 6 Aug 2026 11:29:39 +0200 Subject: [PATCH 2/7] Document ownership of empty-scope upload safeguards --- docs/connector-caller-diagnostics-boundary.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/connector-caller-diagnostics-boundary.md b/docs/connector-caller-diagnostics-boundary.md index 967ae3a..28f03b5 100644 --- a/docs/connector-caller-diagnostics-boundary.md +++ b/docs/connector-caller-diagnostics-boundary.md @@ -175,6 +175,30 @@ must not reduce those safeguards to an unattended or agent-supplied approval. - **Rationale:** These are workflow policy and have no generic AMS meaning. - **Relevant aims:** E6, F3. +### DIAG-008: Empty-scope upload policy + +- **Date:** 2026-08-06 +- **Status:** Accepted +- **Decision:** The connector exposes read-only event counts, verified exact + event IDs, immutable mutation inputs, and generic mutation safeguards. It + does not prohibit inserting into a non-empty event scope because appending + events is valid generic AMS behavior. +- **Caller responsibility:** Synthetic Data requires its generated n-day range + to be empty before upload. Existing events block the complete refresh and are + offered only through separately executed exact-ID deletion plans. Records + before the generated range remain untouched. +- **Rationale:** Whether existing data constitutes a collision follows from the + caller's synthetic-data purpose and pipeline sequencing, not the AMS API + contract. A read-before-write check is not an atomic server guarantee, so the + caller repeats it immediately before its first upload and reports that + limitation honestly. +- **Safeguards:** No partial conflict-free upload, no automatic deletion, fresh + refresh after remediation, separate preview and human-confirmed execution + commands for every exact-ID delete plan, final read-only empty-range recheck, + and existing connector dry-run, sandbox, exact-ID, immutable-plan, and + confirmation gates. +- **Relevant aims:** C3, C5, C7, D2, D6-D10, E6, F3. + ## Agent Decision Checklist Before adding or moving a diagnostic, an agent must answer: From d1e35e4ade7a6b6a98e97d341c6ee9f086a71af2 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 6 Aug 2026 13:38:47 +0200 Subject: [PATCH 3/7] Recognize exact-ID deletion success responses Accept the verified AMS 'Deleted ' response only when it matches the requested ID. Preserve fail-closed handling for ambiguous responses and document the connector/caller responsibility boundary. --- docs/connector-caller-diagnostics-boundary.md | 22 +++++++ src/ams_smartabase/diagnostics.py | 47 +++++++++++++++ tests/test_diagnostics.py | 58 +++++++++++++++++++ 3 files changed, 127 insertions(+) diff --git a/docs/connector-caller-diagnostics-boundary.md b/docs/connector-caller-diagnostics-boundary.md index 28f03b5..1512350 100644 --- a/docs/connector-caller-diagnostics-boundary.md +++ b/docs/connector-caller-diagnostics-boundary.md @@ -199,6 +199,28 @@ must not reduce those safeguards to an unattended or agent-supplied approval. confirmation gates. - **Relevant aims:** C3, C5, C7, D2, D6-D10, E6, F3. +### DIAG-009: Exact-ID delete success messages + +- **Date:** 2026-08-06 +- **Status:** Accepted +- **Decision:** AMS Connector recognizes the verified delete-endpoint response + `{"message": "Deleted "}` as successful only when the message + exactly matches that shape and the returned positive integer equals the + requested event ID. Similar free-form text, mismatched IDs, and responses + containing error fields remain partial or unknown. +- **Caller responsibility:** Synthetic Data explains the structured reason and + safe next step when a deletion remains partial or unknown. It must not + automatically retry an uncertain mutation. +- **Rationale:** The delete endpoint's response grammar and exact-ID meaning are + generic AMS API behavior. Human-facing recovery instructions depend on the + caller's immutable plan and pipeline sequence. +- **Safeguards:** Strict full-message matching, exact requested-ID comparison, + fail-closed handling for ambiguous/error responses, redacted audit output, + and no automatic retry. +- **Relevant aims:** D2, D5, D6, D12, F6. +- **Revisit when:** Authorized API evidence establishes an additional delete + success response shape. + ## Agent Decision Checklist Before adding or moving a diagnostic, an agent must answer: diff --git a/src/ams_smartabase/diagnostics.py b/src/ams_smartabase/diagnostics.py index 84706f4..5234adf 100644 --- a/src/ams_smartabase/diagnostics.py +++ b/src/ams_smartabase/diagnostics.py @@ -31,6 +31,7 @@ "deleted_count", "deletedCount", ) +DELETE_SUCCESS_MESSAGE_RE = re.compile(r"Deleted ([1-9]\d*)", re.IGNORECASE) class MutationState(str, Enum): @@ -234,6 +235,14 @@ def assess_operation_execution( responses = list(getattr(execution, "responses", [])) response_count = len(responses) event_ids = tuple(_extract_event_ids(responses)) + if operation == "delete_event": + # AMS' delete endpoint can confirm the exact deleted ID only in a + # message such as ``{"message": "Deleted 123"}``. Keep this strict and + # delete-specific: interpreting arbitrary response text as an event ID + # would weaken exact-ID deletion validation. + event_ids = tuple( + sorted(set(event_ids).union(_extract_delete_success_event_ids(responses))) + ) acceptance_states = tuple( state for response in responses @@ -340,6 +349,10 @@ def response_confirms_success(response: object, expected_count: int, operation: return ( response.strip().casefold() in SUCCESS_STATUSES or smartabase_acceptance_state(response) is not None + or ( + operation == "delete_event" + and _delete_success_event_id(response) is not None + ) ) if not isinstance(response, Mapping) or response.get("__is_rpc_exception__"): return False @@ -352,6 +365,10 @@ def response_confirms_success(response: object, expected_count: int, operation: status not in SUCCESS_STATUSES and response.get("success") is not True and smartabase_acceptance_state(response) is None + and not ( + operation == "delete_event" + and _delete_success_event_id(response) is not None + ) ): return False expected = 1 if operation in {"delete_event", "upsert_profile"} else expected_count @@ -440,6 +457,36 @@ def visit(item: object) -> None: return sorted(found) +def _extract_delete_success_event_ids(value: object) -> list[int]: + """Extract IDs only from the verified delete-endpoint success message.""" + + found: set[int] = set() + + def visit(item: object) -> None: + event_id = _delete_success_event_id(item) + if event_id is not None: + found.add(event_id) + if isinstance(item, list): + for nested in item: + visit(nested) + + visit(value) + return sorted(found) + + +def _delete_success_event_id(response: object) -> int | None: + if isinstance(response, Mapping): + if response.get("__is_rpc_exception__") or _response_has_error(response): + return None + message = response.get("message") + else: + message = response + if not isinstance(message, str): + return None + match = DELETE_SUCCESS_MESSAGE_RE.fullmatch(message.strip()) + return int(match.group(1)) if match is not None else None + + def _emit_endpoint_warning(message: str, enabled: bool, stacklevel: int) -> None: if enabled: warnings.warn( diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index f76f8d1..5367294 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -142,6 +142,64 @@ def test_lossy_or_boolean_counts_are_not_success_evidence(self): ) ) + def test_delete_message_confirms_only_the_exact_requested_event_id(self): + execution = OperationExecution( + endpoint="deleteevent", + attempted_count=1, + dry_run=False, + executed=True, + body=[{"eventId": 123}], + row_operations=[], + responses=[{"message": "Deleted 123"}], + ) + + assessment = assess_operation_execution( + execution, + expected_count=1, + operation="delete_event", + expected_event_id=123, + ) + + self.assertEqual(assessment.state, MutationState.CONFIRMED_COMPLETE) + self.assertTrue(assessment.confirmed) + self.assertEqual(assessment.event_ids, (123,)) + + mismatch = assess_operation_execution( + execution, + expected_count=1, + operation="delete_event", + expected_event_id=999, + ) + self.assertEqual(mismatch.state, MutationState.PARTIAL_OR_UNKNOWN) + self.assertFalse(mismatch.confirmed) + self.assertEqual(mismatch.reason_code, "delete_event_id_mismatch") + + def test_delete_message_parser_rejects_ambiguous_or_error_responses(self): + for response in ( + {"message": "Deletion queued 123"}, + {"message": "Deleted 123 extra"}, + {"message": "Deleted 0"}, + {"message": "Deleted 123", "error": "permission denied"}, + ): + with self.subTest(response=response): + execution = OperationExecution( + endpoint="deleteevent", + attempted_count=1, + dry_run=False, + executed=True, + body=[{"eventId": 123}], + row_operations=[], + responses=[response], + ) + assessment = assess_operation_execution( + execution, + expected_count=1, + operation="delete_event", + expected_event_id=123, + ) + self.assertEqual(assessment.state, MutationState.PARTIAL_OR_UNKNOWN) + self.assertFalse(assessment.confirmed) + def test_acceptance_parser_rejects_error_envelopes(self): self.assertEqual( smartabase_acceptance_state({"state": "SUCCESSFULLY_IMPORTED"}), From 4aa154f1d8c4d63d16bef91a0e88136b3755c37e Mon Sep 17 00:00:00 2001 From: Rens Date: Tue, 11 Aug 2026 10:38:51 +0200 Subject: [PATCH 4/7] Document continuation read-back normalization boundary --- docs/connector-caller-diagnostics-boundary.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/connector-caller-diagnostics-boundary.md b/docs/connector-caller-diagnostics-boundary.md index 1512350..a406d6c 100644 --- a/docs/connector-caller-diagnostics-boundary.md +++ b/docs/connector-caller-diagnostics-boundary.md @@ -221,6 +221,66 @@ must not reduce those safeguards to an unattended or agent-supplied approval. - **Revisit when:** Authorized API evidence establishes an additional delete success response shape. +### DIAG-010: Verified continuation after partial sandbox refresh + +- **Date:** 2026-08-10 +- **Status:** Accepted +- **Decision:** The connector remains generic and does not decide whether a + non-empty scope is safe to continue. It exposes flattened fetched rows, + verified exact event IDs, immutable mutation inputs, and mutation-state + evidence. Synthetic Data may treat existing rows as an interrupted refresh + only when an explicitly selected immutable parent plan and a fresh read-only + comparison prove exact normalized row-multiset agreement. +- **Caller responsibility:** Synthetic Data owns parent-plan lineage, row + canonicalization, clean-form-boundary policy, observed-snapshot fingerprints, + missing-payload selection, continuation preview and confirmation, and final + full-parent reconciliation. It must create a new immutable continuation plan + and authorization; it cannot reuse or resume the stopped authorization or an + attempted child plan. +- **Rationale:** A manually reviewed recovery plan containing only rows proven + absent is not an automatic retry and does not change the immutable parent + intent. It provides a non-destructive recovery path while retaining duplicate + prevention and human control. +- **Safeguards:** DIAG-008 remains the default for ordinary refreshes. Count-only + agreement, an inferred latest run, unexpected or ambiguous rows, a partial + form, changed local artifacts, changed live snapshot, unknown environment, or + non-interactive execution all fail closed. Continuation remains sandbox-only, + preview-first, bounded, newly confirmed, and stopped on every partial or + unknown result. +- **Relevant aims:** C3, C5, C7, C8, E6, F3, F6. +- **Revisit when:** AMS provides server-side idempotency keys, an authoritative + operation-status endpoint, or response semantics that can prove row-level + import completion without read-back comparison. + +### DIAG-011: Typed read-back normalization remains caller policy + +- **Date:** 2026-08-11 +- **Status:** Accepted; clarifies DIAG-010 without changing connector behavior +- **Decision:** The connector preserves generic event metadata and returned form + pair values. It does not infer that an imported string and an AMS read-back + value are equivalent. Synthetic Data may apply a versioned, immutable-plan- + bound comparison policy for its own generated forms when read-only evidence + proves a transformation is meaning-preserving. +- **Caller responsibility:** Synthetic Data explicitly allowlists fields and + accepts only exact finite-decimal equivalence, strict duration-to-integral- + millisecond equivalence, reviewed singleton-list wrapping, omitted blank + pairs, and redundant `Date`/`Time` omission after proving equality with event + start metadata. It reports only redacted field/category/count diagnostics. +- **Rejected equivalences:** Numeric rounding or tolerance, AM/PM changes, + boolean or case synonyms, missing nonblank values, and generic list decoding + remain conflicts. In particular, a cross-midnight finish time read back 12 + hours later is a semantic difference rather than formatting. +- **Rationale:** Form type, precision, and source semantics belong to the + calling workflow. Keeping the connector lossless prevents one caller's + organization-specific schema assumptions from changing generic responses. +- **Safeguards:** Policy version and hash are frozen into continuation plans; + changed policies require a fresh preview; unknown encodings fail closed; raw + fetched values are neither logged nor persisted by default; connector + flattening and payload construction remain unchanged. +- **Relevant aims:** C3, C5, C7, C8, E6, F3, F6. +- **Revisit when:** AMS exposes authoritative form field types and precision or + a typed response contract that is generic across tenants. + ## Agent Decision Checklist Before adding or moving a diagnostic, an agent must answer: From f0ca6ec9b65d05a674c1b2787e5ed417ca65a45b Mon Sep 17 00:00:00 2001 From: Rens Date: Tue, 11 Aug 2026 16:47:07 +0200 Subject: [PATCH 5/7] Add initial specifications and quickstart guide for Incremental Synthetic Data Pipeline - Created quickstart guide detailing the operational pattern, prerequisites, and local workflow for the planned incremental pipeline. - Developed a comprehensive research document outlining key decisions and rationale for the incremental synthetic data pipeline. - Established a feature specification document defining user scenarios, functional requirements, and success criteria for the pipeline. --- .specify/feature.json | 3 + ams-python-connector.code-workspace | 7 - .../checklists/requirements.md | 35 ++ .../contracts/artifact-contracts.md | 150 +++++++ .../contracts/cli-contract.md | 155 +++++++ specs/001-daily-data-pipeline/data-model.md | 249 +++++++++++ specs/001-daily-data-pipeline/plan.md | 395 ++++++++++++++++++ specs/001-daily-data-pipeline/quickstart.md | 201 +++++++++ specs/001-daily-data-pipeline/research.md | 295 +++++++++++++ specs/001-daily-data-pipeline/spec.md | 165 ++++++++ 10 files changed, 1648 insertions(+), 7 deletions(-) create mode 100644 .specify/feature.json delete mode 100644 ams-python-connector.code-workspace create mode 100644 specs/001-daily-data-pipeline/checklists/requirements.md create mode 100644 specs/001-daily-data-pipeline/contracts/artifact-contracts.md create mode 100644 specs/001-daily-data-pipeline/contracts/cli-contract.md create mode 100644 specs/001-daily-data-pipeline/data-model.md create mode 100644 specs/001-daily-data-pipeline/plan.md create mode 100644 specs/001-daily-data-pipeline/quickstart.md create mode 100644 specs/001-daily-data-pipeline/research.md create mode 100644 specs/001-daily-data-pipeline/spec.md diff --git a/.specify/feature.json b/.specify/feature.json new file mode 100644 index 0000000..920561c --- /dev/null +++ b/.specify/feature.json @@ -0,0 +1,3 @@ +{ + "feature_directory": "specs/001-daily-data-pipeline" +} diff --git a/ams-python-connector.code-workspace b/ams-python-connector.code-workspace deleted file mode 100644 index 362d7c2..0000000 --- a/ams-python-connector.code-workspace +++ /dev/null @@ -1,7 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ] -} \ No newline at end of file diff --git a/specs/001-daily-data-pipeline/checklists/requirements.md b/specs/001-daily-data-pipeline/checklists/requirements.md new file mode 100644 index 0000000..8463933 --- /dev/null +++ b/specs/001-daily-data-pipeline/checklists/requirements.md @@ -0,0 +1,35 @@ +# Specification Quality Checklist: Incremental Synthetic Data Pipeline + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-08-11 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No non-governance implementation details +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No `[NEEDS CLARIFICATION]` markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No non-governance implementation details leak into specification + +## Notes + +- Validation passed on the first review iteration. +- Scheduled execution is intentionally preparation-only under the current constitution and risk model. diff --git a/specs/001-daily-data-pipeline/contracts/artifact-contracts.md b/specs/001-daily-data-pipeline/contracts/artifact-contracts.md new file mode 100644 index 0000000..8e9f034 --- /dev/null +++ b/specs/001-daily-data-pipeline/contracts/artifact-contracts.md @@ -0,0 +1,150 @@ +# Artifact Contracts + +## Store Layout + +```text +STATE_ROOT/ +└── datasets/DATASET_ID/VERSION/ + ├── dataset.json + ├── profiles/athletes.json + ├── inputs/ + │ ├── templates/... + │ ├── roster/athletes.csv + │ └── form-map.json + ├── staging/ATTEMPT_ID/... + ├── days/date=YYYY-MM-DD/ + │ ├── csv/... + │ ├── validation.json + │ ├── state.json + │ └── manifest.json + ├── publication/date=YYYY-MM-DD/ + │ ├── plans/CHILD_ID/operation_plan.json + │ ├── snapshots/... + │ └── records/... + ├── bundles/BUNDLE_ID/master_upload_plan.json + ├── reports/... + └── audit/... +``` + +`days/.../manifest.json` is written last and is the only generation commit +marker. `execution_results.json` or its versioned successor is exclusively +created before the first live request and may not be reused. + +## Store Protocol + +The platform-neutral engine depends on this logical protocol: + +```python +class StateStore(Protocol): + def load_dataset(self, dataset_id: str) -> DatasetVersion: ... + def scan_commits(self, dataset_id: str) -> Sequence[DailyCommitManifest]: ... + def read_checkpoint(self, manifest: DailyCommitManifest) -> StateCheckpoint: ... + def stage_day(self, attempt: StagedDay) -> StagedDayRef: ... + def compare_and_commit(self, expected_head_hash: str | None, staged: StagedDayRef) -> DailyCommitManifest: ... + def save_immutable_plan(self, plan: DailyUploadPlan) -> PlanRef: ... + def reserve_publication(self, bundle: PublicationBundle) -> PublicationReservation: ... + def append_publication_record(self, record: PublicationRecord) -> None: ... +``` + +Implementations: + +- `FilesystemStateStore`: local path, exclusive lock, same-filesystem staging, + exclusive commit marker, and compare-before-commit. +- `DatabricksStateStore`: immutable artifacts on a Unity Catalog Volume and + conditional claim/commit in a managed Delta control table. The Delta row is + an index/lease; the manifest hash remains authoritative. + +If cross-process exclusivity cannot be demonstrated for a backend, that backend +is prepare-read-only and cannot commit or publish. + +## Canonical Hashing + +- SHA-256 lowercase hexadecimal. +- UTF-8 JSON, sorted object keys, compact separators, Unicode preserved. +- Dates are `YYYY-MM-DD`; timestamps are UTC RFC 3339 with a `Z` suffix. +- Paths stored in hashed plans are normalized relative paths beneath the + immutable bundle/dataset root. +- Hash fields are omitted while their containing object is hashed. +- Operational timestamps, host/run IDs, and staging attempt IDs live in separate + audit/index records and are excluded from deterministic content and plan + fingerprints. +- Arrays retain semantic order. Set-like collections are sorted before encoding. +- Floating-point values use one documented finite decimal encoding; NaN and + infinity are invalid. +- Every schema and row canonicalization policy has an explicit version. A policy + change cannot reinterpret an already signed plan. + +## Daily Commit Sequence + +1. Acquire/claim `(dataset_version, event_date)` against the expected prior head. +2. Write all output, validation, and after-state files to a unique staging path. +3. Re-open and validate schemas, counts, relationships, routing, IDs, and hashes. +4. Build a manifest binding prior manifest, before/after states, files, inputs, + versions, and validation. +5. Copy/move immutable content to the final date partition. +6. Create the final manifest exclusively. +7. Commit the final path/hash to the control ledger when that adapter is used. +8. Release the claim. Orphan staging is diagnostic and never authoritative. + +A crash before step 6 leaves no committed date. A crash after step 6 is +recoverable by rebuilding the ledger/index from the manifest chain. + +## Remote Scope Snapshot + +A read-only snapshot binds: + +- normalized exact sandbox URL and environment verification evidence; +- requested date interval, stable user IDs, event form names, and fields; +- connector/request schema version and pagination/completeness evidence; +- per-date/form/athlete result counts and canonical row-multiset hashes; +- zero-result scopes explicitly, not by omission; +- timestamp and redacted response/request metadata. + +Raw credentials and unnecessary raw event values are excluded. If stable athlete +attribution, pagination completeness, or any requested zero-result scope cannot +be proven, classification is `READ_UNCERTAIN`. + +## Upload Plan Compatibility + +The pipeline extends rather than replaces the existing immutable upload-plan +contract. Existing plan hash, prepared payload, exact target, batch, confirmation, +execution-reservation, and mutation-state rules remain authoritative. New daily +lineage fields are versioned and included in the hash. + +Allowed operation: event insert. + +Disallowed operations: profile import, event update/upsert/replace, overwrite, +archive, and delete. + +## Publication Bundle Validation + +Before confirmation and again before each child: + +- resolve every relative child path beneath the bundle directory; +- validate full child hash and affected count; +- validate strictly increasing child sequence and nondecreasing event date; +- validate one exact sandbox target and event-only operations; +- validate all child execution results are absent/unconsumed; +- validate date count `<=90`, total records `<=25,000`, child records `<=500`, + and request batches `<=100`; +- verify the fresh remote snapshot covers the exact bundle scope and remains + empty for new dates; and +- reserve the master and child execution records before the first mutation. + +The earliest blocked unpublished date terminates the ready prefix. Later dates +are not added to a bundle around it. + +## Secret And Data Classification + +| Artifact | Contains athlete data | Contains credentials | Source-control eligible | +|---|---:|---:|---:| +| Dataset/profile/generated CSV/checkpoint | Yes | Never | No | +| Prepared upload payload/remote snapshot | Yes or sensitive metadata | Never | No | +| Daily/master plan and execution result | Potentially | Never | No | +| Delta/control ledger | Aggregate identifiers/hashes only | Never | No | +| Synthetic/redacted schema fixtures | Reviewed synthetic only | Never | Yes | +| Documentation | Placeholders only | Never | Yes | + +Local stores require restrictive permissions and existing ignore rules. +Databricks stores require least-privilege Unity Catalog grants. Retention and +cleanup are explicit operator policies; cleanup must never delete AMS records. diff --git a/specs/001-daily-data-pipeline/contracts/cli-contract.md b/specs/001-daily-data-pipeline/contracts/cli-contract.md new file mode 100644 index 0000000..98c03cb --- /dev/null +++ b/specs/001-daily-data-pipeline/contracts/cli-contract.md @@ -0,0 +1,155 @@ +# CLI Contract: `sandbox-data` + +This contract describes the intended interface after implementation. It is not +yet available in the current scripts. + +## Global Rules + +- Every command identifies one `--dataset` and one authoritative `--state-root`. +- The same dataset version must not be advanced from unsynchronized local and + Databricks copies. Moving execution contexts requires a complete hash-valid + export/import or a shared store adapter. +- Paths are explicit; no command relies on the repository working directory. +- Default behavior is read-only with respect to AMS. +- Credentials are resolved by a provider. CLI flags, task parameters, logs, + manifests, and dataset files never accept or contain plaintext passwords. +- Machine-readable output uses `--output json`; secrets and row values remain + excluded. + +## `init` + +Create a new immutable dataset version and initial checkpoint. + +```text +sandbox-data init \ + --dataset DATASET_ID \ + --dataset-version VERSION \ + --state-root PATH \ + --roster PATH \ + --templates PATH \ + --form-map PATH \ + --simulation-epoch YYYY-MM-DD \ + --seed VALUE +``` + +Rules: + +- Validates registered-athlete identity and organization routing. +- Freezes athlete simulation profiles and all output-affecting hashes. +- Refuses to reuse an existing version with different content. +- Does not call AMS and does not generate a day. + +## `prepare` + +Validate state, generate missing dates, and optionally prepare read-only-verified +upload plans. + +```text +sandbox-data prepare \ + --dataset DATASET_ID \ + --state-root PATH \ + [--through-date YYYY-MM-DD] \ + [--remote-check auto|require|skip] \ + [--credential-provider local|databricks] +``` + +Rules: + +- `--through-date` defaults to yesterday in the dataset's IANA timezone. +- The command validates the complete hash chain before doing work. +- It advances at most 90 missing dates per invocation, oldest first. If more + remain, it reports the next date and requires another invocation; there is no + unlimited override. +- A committed target is validated and becomes a no-op. +- `auto` performs remote classification when credentials are available; + otherwise generation succeeds but publication remains `UNPLANNED`. +- `require` fails closed if AMS cannot be classified completely. +- `skip` performs no AMS calls and cannot produce a ready publication bundle. +- It performs no live AMS mutation under any option. +- It plans only the earliest contiguous unpublished interval. A blocked earlier + date prevents a bundle from silently skipping ahead. +- It refuses publication bundles above 90 dates or 25,000 total records. + +## `status` + +Validate and summarize generation and publication state without mutation. + +```text +sandbox-data status --dataset DATASET_ID --state-root PATH [--output text|json] +``` + +The summary includes dataset/generator versions, contiguous generation head, +next missing date, generated-but-unpublished dates, immutable plan hashes, +blocked/reconcile-required dates, and redacted artifact paths. It does not print +athlete row values or credentials. + +## `publish` + +Interactively execute one exact immutable publication bundle. + +```text +sandbox-data publish --publication-plan PATH --execute --credential-provider local +``` + +Rules: + +- Requires a real interactive terminal. Piped input, Databricks job parameters, + environment variables, agents, and programmatic callbacks cannot supply the + confirmation. +- Requires `--execute` so selecting a plan remains distinct from mutation intent. +- Revalidates bundle and child hashes, exact sandbox URL, event-only operations, + limits, unconsumed execution results, and current credentials. +- Performs one fresh whole-scope read immediately before preview/confirmation. +- Displays environment, operation, dates, athletes/groups, forms, fields, child + hashes, child/record counts, and exact typed phrase derived from the plan. +- Consumes one confirmation only for the frozen child order. +- Does not retry live mutations. Any mismatch, partial/unknown result, or + unclassified response stops later children. +- Never updates, upserts, overwrites, or deletes. + +There is deliberately no `--yes`, `--confirm`, `--force`, `--auto-upload`, +confirmation environment variable, or scheduled publish mode. + +## `reconcile` + +Read AMS and compare exact canonical row multisets against a known immutable +publication lineage. + +```text +sandbox-data reconcile --publication-plan PATH --credential-provider local +``` + +Rules: + +- Always read-only. +- `RECONCILED` requires matching plan/execution lineage and exact versioned row + canonicalization. +- An unexplained non-empty or subset scope remains blocked. +- A known partial parent may produce a new immutable remainder proposal only + through the existing reviewed continuation policy; it is never executed by + this command. + +## `report` + +Build inspection dashboards from committed partitions without advancing state or +calling AMS. + +```text +sandbox-data report --dataset DATASET_ID --state-root PATH [--from-date YYYY-MM-DD] [--through-date YYYY-MM-DD] +``` + +This command preserves the project's Plotly inspection requirements while +keeping self-contained HTML generation off the daily prepare path. + +## Exit Status Contract + +| Code | Meaning | +|---|---| +| `0` | Completed, including a validated no-op | +| `2` | Invalid input, incompatible version, or failed local validation | +| `3` | Corrupt chain, hash conflict, concurrent commit, or unexpected AMS data | +| `4` | AMS read incomplete/uncertain or reconciliation required | +| `5` | Live mutation partial/unknown/contradictory; stop and reconcile | +| `6` | Per-run or publication-bundle hard limit reached | + +Status `5` must never be configured as an automatic retry condition. diff --git a/specs/001-daily-data-pipeline/data-model.md b/specs/001-daily-data-pipeline/data-model.md new file mode 100644 index 0000000..2af6978 --- /dev/null +++ b/specs/001-daily-data-pipeline/data-model.md @@ -0,0 +1,249 @@ +# Data Model: Incremental Synthetic Data Pipeline + +## Model Principles + +- Generation is an append-only, causally contiguous chain per dataset version. +- Publication is a separate append-only chain; generation never depends on AMS + upload status. +- JSON used for hashing is UTF-8 canonical JSON with sorted keys, no insignificant + whitespace, ISO dates, and an explicit schema version. +- Names, credentials, payload values, and raw AMS responses do not belong in the + compact control ledger. Athlete data stays in the protected artifact store. +- `latest` pointers and indexes are caches. Hash-valid immutable manifests are + authoritative. + +## Entity Relationships + +```text +DatasetVersion 1 ─── * AthleteSimulationProfile + │ + ├── 1 ─── * DailyCommit ─── 1 StateCheckpoint + │ │ + │ ├── * GeneratedFile + │ └── * PendingEvent (inside checkpoint until due) + │ + └── 1 ─── * PublicationDay ─── * UploadChildPlan + └── 0..1 PublicationAttempt + +PublicationBundle 1 ─── * ordered UploadChildPlan references +``` + +## DatasetVersion + +Immutable identity for one synthetic timeline. + +| Field | Type | Rules | +|---|---|---| +| `schema_version` | string | Required artifact contract version | +| `dataset_id` | string | Operator-facing stable name; path-safe and non-secret | +| `dataset_version` | string | Unique immutable version identifier | +| `generator_version` | string | Pins algorithm, RNG streams, and renderer policy | +| `base_seed` | integer/string | Stored as non-secret generation input | +| `simulation_epoch` | ISO date | Absolute origin for seasonal/campaign phase | +| `timezone` | IANA name | `Europe/Amsterdam` by default | +| `template_manifest_sha256` | SHA-256 | Binds every source template and schema | +| `form_map_sha256` | SHA-256 | Binds exact mapped destination forms | +| `roster_snapshot_sha256` | SHA-256 | Binds effective-dated athlete profiles | +| `generator_config_sha256` | SHA-256 | Hash of all future-output-affecting settings | +| `created_at` | UTC timestamp | Audit metadata excluded from deterministic config/content fingerprints | + +Changing the seed, generator algorithm, simulation epoch, templates, form map, +or an output-affecting policy requires a new version or an explicit effective- +dated cutover model that has its own hash. It may not silently alter this entity. + +## AthleteSimulationProfile + +Frozen synthetic identity keyed by a stable AMS sandbox user ID. + +| Field | Type | Rules | +|---|---|---| +| `athlete_id` | stable ID | Required; never use row position or display name as identity | +| `organization` | enum | `KNLTB`, `WSV`, or reviewed shared-only classification | +| `pattern_id` | string | Frozen pattern family/version | +| `profile_slot` | integer/string | Stable source for synthetic attributes; never recomputed after reorder | +| `synthetic_attributes` | object | Age, squad, physiology, location, and other frozen properties | +| `effective_from` | ISO date | First date eligible for generation | +| `effective_to` | ISO date/null | Last eligible date; removal preserves history | +| `source_registry_hash` | SHA-256 | Registry evidence used when the profile was adopted | + +Names needed in generated CSVs remain in the protected profile artifact, not in +the Delta/control ledger or status logs. + +## AthleteState + +Minimal state required to generate one athlete's next day. + +| Field | Type | Rules | +|---|---|---| +| `athlete_id` | stable ID | Must match one frozen profile | +| `last_generated_date` | ISO date/null | Must equal the checkpoint date after commit | +| `fatigue` | number | Validated finite bounded value | +| `previous_complaint` | boolean | Explicit legacy behavior state | +| `active_complaint` | object/null | Stable complaint ID, start/end dates, area, cause, version | +| `recent_practiced_dates` | date array | Bounded to the generator's lookback window | +| `week_plan` | object/null | Complete Monday-Sunday plan and week-start key | +| `pending_events` | object array | Future-dated decisions not emitted yet | + +Python `random.Random.getstate()`, a global session counter, registry index, and +invocation-relative day index are deliberately absent. Random streams and IDs +are derived from dataset version, athlete ID, absolute date, component name, +and per-day ordinal. + +## StateCheckpoint + +Immutable aggregate after-state for a committed date. + +| Field | Type | Rules | +|---|---|---| +| `checkpoint_date` | ISO date | The committed simulation date | +| `dataset_version` | string | Required exact match | +| `athletes` | map | Stable athlete ID to `AthleteState`, sorted canonically | +| `pending_event_count` | integer | Must equal all queued entries | +| `checkpoint_sha256` | SHA-256 | Hash over canonical content excluding this field | + +The next generation transition takes exactly this checkpoint as its before- +state. Inactive athletes may be retained for audit but are not advanced after +their `effective_to` date. + +## DailyGeneration And GeneratedFile + +A date-level materialization containing all rows whose actual event date equals +the partition date. An event decided earlier is emitted from the pending queue +only when due. + +| Field | Type | Rules | +|---|---|---| +| `event_date` | ISO date | Partition key; CSV values still render `dd/mm/yyyy` | +| `forms` | map | Mapped form name to generated file metadata | +| `record_count` | integer | Sum of rendered records | +| `validation_summary` | object | Schema, routing, relation, ID collision, and distribution checks | + +Each `GeneratedFile` records relative path, form name, schema hash, row count, +byte count, and SHA-256. A lightweight daily statistics file is allowed. Plotly +HTML is a separate periodic/report artifact. + +## DailyCommitManifest + +The commit marker written only after every other artifact validates. + +| Field | Type | Rules | +|---|---|---| +| `event_date` | ISO date | Exactly previous committed date plus one | +| `previous_manifest_sha256` | SHA-256/null | Null only for the initial checkpoint | +| `before_state_sha256` | SHA-256 | Must equal prior checkpoint hash | +| `after_state_path` | relative path | Must remain under the dataset root | +| `after_state_sha256` | SHA-256 | Must match checkpoint content | +| `generated_files` | array | Complete `GeneratedFile` set in canonical order | +| `dataset_config_sha256` | SHA-256 | Must match `DatasetVersion` | +| `roster_snapshot_sha256` | SHA-256 | Exact effective roster used for this date | +| `validation_status` | enum | Must be `passed` to commit | +| `manifest_sha256` | SHA-256 | Canonical manifest hash | + +Attempt IDs, wall-clock timestamps, host names, and Databricks run IDs belong in +separate audit/index records and are excluded from the deterministic daily +manifest. This is required for one-shot, resumed, local, and Databricks runs to +produce the same manifest fingerprint. + +### Generation State Transitions + +```text +ABSENT -> STAGED -> VALIDATED -> COMMITTED + | | + +--------> FAILED +``` + +- `STAGED`, `VALIDATED`, and `FAILED` attempts are never authoritative. +- `COMMITTED` is immutable. +- A repeat with the same hash is a no-op; a different hash is a conflict. +- An interior missing/corrupt commit followed by later commits is `CORRUPT_CHAIN` + and cannot be filled as an ordinary gap. + +## DailyUploadPlan + +Existing immutable connector upload-plan schema plus the daily lineage needed by +the caller. + +| Field | Type | Rules | +|---|---|---| +| `operation` | enum | Event insert only | +| `environment` | enum | Must be verified `sandbox` | +| `target_url` | URL | Exact normalized URL containing reviewed `/sandbox` path | +| `event_date` | ISO date | One daily scope only | +| `daily_manifest_sha256` | SHA-256 | Binds generated source | +| `scopes` | array | Stable athlete IDs, forms, fields, and exact date | +| `prepared_records` | array | Existing canonical prepared payload rows | +| `affected_count` | integer | `1..500` per child | +| `batch_size` | integer | `1..100` | +| `canonicalization_version` | string | Exact read-back comparison policy | +| `plan_sha256` | SHA-256 | Existing immutable plan fingerprint | + +One date may require several child plans. Every child for that date is either +included in an authorized publication bundle or none is executed. + +## PublicationBundle + +One reviewed, bounded, ordered authorization scope for the earliest contiguous +ready prefix. + +| Field | Type | Rules | +|---|---|---| +| `bundle_id` | string | Immutable identifier | +| `start_date` / `end_date` | ISO date | Contiguous daily interval | +| `target_url` | URL | Same verified sandbox for every child | +| `children` | array | Relative path, full SHA-256, count, date, and sequence | +| `date_count` | integer | `1..90` | +| `child_plan_count` | integer | Must equal child list length | +| `affected_count` | integer | Must equal child totals and be `<=25,000` | +| `bundle_sha256` | SHA-256 | Binds exact ordered scope | + +The expected confirmation is derived from this plan, for example +`UPLOAD 2500 RECORDS IN 5 PLANS f59c0e`; it is never accepted from a flag, +environment variable, notebook parameter, or agent. + +## PublicationRecord + +Append-only, redacted outcome lineage. It never changes generation commits. + +| Field | Type | Rules | +|---|---|---| +| `event_date` | ISO date | Daily status key | +| `daily_manifest_sha256` | SHA-256 | Source generation | +| `bundle_sha256` | SHA-256/null | Authorizing master | +| `child_plan_hashes` | array | Exact attempted children | +| `scope_snapshot_sha256` | SHA-256 | Fresh pre-execution read evidence, recorded without changing the immutable bundle | +| `status` | enum | State below | +| `accepted_count` | integer | Connector-classified count only | +| `unknown_count` | integer | Must be visible when nonzero | +| `response_summary` | object | Redacted status/attempt metadata, no payload values | +| `created_at` | UTC timestamp | Audit ordering | + +### Publication State Transitions + +```text +UNPLANNED + ├──> READ_UNCERTAIN --------------------------> BLOCKED + ├──> UNEXPECTED_NONEMPTY ---------------------> BLOCKED + ├──> MATCHING_WITH_LINEAGE -------------------> RECONCILED + └──> READY -> AWAITING_HUMAN -> IN_PROGRESS + ├──> RECONCILED + ├──> ACCEPTED_UNVERIFIED -> RECONCILE_REQUIRED + ├──> PARTIAL_OR_UNKNOWN -> RECONCILE_REQUIRED + └──> RECONCILIATION_MISMATCH -> BLOCKED +``` + +Only `RECONCILED` is terminal success. No mutation state automatically returns +to `READY`. A reviewed continuation after a known partial parent uses a new +remainder plan, fresh snapshot, preview, and confirmation; an unexplained subset +without parent lineage is a conflict. + +## Databricks Control Ledger + +The optional managed Delta table is an index/coordination layer with one row per +dataset version and event date. It stores hashes, relative paths, aggregate +counts/statuses, Databricks run ID, and audit timestamps—never names, generated +row values, credentials, or raw API responses. A conditional `MERGE` claims or +commits a date. Immutable Volume manifests remain the content authority. + +Recommended key: `(dataset_version, event_date)`. Required constraints are one +terminal generation commit per key and no transition out of a terminal +publication result except by appending a new reconciliation record. diff --git a/specs/001-daily-data-pipeline/plan.md b/specs/001-daily-data-pipeline/plan.md new file mode 100644 index 0000000..d2edecb --- /dev/null +++ b/specs/001-daily-data-pipeline/plan.md @@ -0,0 +1,395 @@ +# Implementation Plan: Incremental Synthetic Data Pipeline + +**Feature Branch**: `001-daily-data-pipeline` | **Current Git Branch**: +`agent/native-event-parsing` | **Date**: 2026-08-11 | **Spec**: [spec.md](spec.md) + +**Input**: Feature specification from `specs/001-daily-data-pipeline/spec.md` + +## Summary + +Replace the generator's invocation-scoped athlete/day loop with a deterministic, +checkpointed day-transition engine. A single `sandbox-data` command will scan a +durable dataset, validate its contiguous commit chain, generate every missing +date through an explicit target, and prepare immutable date-scoped upload plans. +The same library and command contract will run against a local directory or a +Unity Catalog Volume. A scheduled Databricks run remains prepare-only; live AMS +event insertion remains a separately invoked, interactive, plan-derived, +human-confirmed operation. + +This is the recommended alternative to full 90-day regeneration or prebuilding +a year. It computes only missing days, retains the longitudinal state needed by +future days, never rewrites committed history, and can catch up after irregular +runs. Daily scheduling is optional: a weekly or irregular prepare run produces +the same data and is usually cheaper when Databricks startup time dominates. + +## Option Decision + +| Option | Advantages | Disadvantages | Decision | +|---|---|---|---| +| Regenerate, delete, and upload 90 days | Reuses the existing batch flow and produces one coherent range | Highest generation/read/write cost; duplicates work; deletion makes routine refresh Critical; largest failure scope | Reject for routine operation; retain temporarily only as rollback tooling | +| Generate one independent day | Minimum daily generation and storage work | Current generator resets causal history, RNG/counters, seasonal phase, weekly context, and IDs | Use only after the stateful transition refactor; then it becomes the normal one-day case | +| Pre-generate one year and reveal one day at a time | Almost no generation compute on publish days | Freezes future roster/config choices; stores unused athlete data; corrections and pending-event changes require awkward invalidation | Reject | +| Stateful missing-date catch-up | Work proportional to actual gaps; identical daily/weekly/irregular behavior; no historical deletion; easy no-op/recovery | Requires versioned state, immutable commits, and concurrency handling | **Selected** | + +The selected workflow can still prepare exactly one day when run daily; it is +not a separate generator mode. A weekly invocation simply applies that same +daily transition seven times under one process and one Databricks startup. + +## Governance And Risk + +**Primary connector aims**: B2, B4, B6, B8, C1-C3, C5-C8, E1, E4, +E6-E9, F3, and F6. Existing roster discovery additionally supports A1; package +deployment carries F5, F7, and F8. + +**Highest risk level**: **High**, because the reviewed interactive phase can +create event records in AMS. Generation, local persistence, read-only remote +classification, and scheduled planning are lower-risk stages, but the complete +feature inherits the highest level. + +**Reviewed trigger scenarios**: wrong sandbox/athlete/form/date scope; +ambiguous or incomplete reads; duplicate creation after retry or lost local +state; configuration drift; partial or uncertain mutation results; unattended +confirmation; excessive batch/catch-up scope; credentials or athlete data in +artifacts; concurrent writers; and misleading executable documentation. + +**Required safeguards carried into design**: + +- default to `prepare`/read-only behavior and verify the exact `/sandbox` target; +- scheduled and non-interactive runs cannot enter the live mutation path; +- publish only event inserts; never update, upsert, overwrite, or delete; +- classify the whole frozen daily scope before planning and again just before + execution; unexpected or uncertain existing data blocks the entire date; +- bind exact child paths, full hashes, counts, ordering, environment, athletes, + forms, fields, and dates into one immutable master plan; +- retain existing 500-record child-plan and 100-record request-batch limits; +- add reviewed hard limits of 90 missing dates and 25,000 total records per + catch-up publication bundle, with no CLI bypass; split larger backlogs into + separately reviewed bundles; +- request the existing scope-derived typed phrase only in an interactive + `publish` invocation, consume it once, and require a fresh phrase after any + change; +- never automatically retry a mutation or continue after partial/unknown state; +- store credentials only in memory from local environment/credential storage or + a Databricks secret provider, never in dataset or Volume artifacts; +- use a single logical writer, immutable daily commits, hash validation, and + crash-safe staging; and +- use synthetic/redacted offline fixtures by default, with guarded live sandbox + tests only when explicitly requested. + +The 25,000-record bundle cap is a new conservative publication limit, not a +claim about an AMS API maximum. It matches the repository's existing bounded +master-operation scale and must be revisited only through safety review. + +## Technical Context + +**Language/Version**: Python 3.10+ (matching `ams-python-connector`) + +**Primary Dependencies**: Python standard library, installed +`ams-python-connector`, `requests` through that connector, and Plotly only for +separately requested/periodic reports; Databricks SDK is isolated to the +Databricks credential/control adapter if required + +**Storage**: Portable append-only filesystem artifacts for generated CSVs, +checkpoints, and immutable plans; local directory for workstation runs; Unity +Catalog Volume plus a small managed Delta control table for Databricks claims +and status indexing, with immutable manifests—not the table—as content authority + +**Testing**: `pytest` for the new package and offline workflow doubles; retain +existing generator tests during migration; opt-in guarded live sandbox tests + +**Target Platform**: Local Windows/Linux Python and Azure Databricks serverless +Jobs on Linux, using a Unity Catalog-enabled workspace + +**Project Type**: Installable Python library plus CLI, implemented in the +sibling `ams-sandbox-data-synthesis` repository and consuming the generic +connector without organization-specific changes to that connector + +**Performance Goals**: Generate and commit only missing dates; perform at most +one AMS event-range read per mapped form per classification pass where the API +proves complete multi-athlete results; create no Plotly report on the daily hot +path; complete a no-op without AMS calls unless explicit reconciliation is due + +**Constraints**: Preserve `dd/mm/yyyy` generated CSV dates and template schemas; +use only registered sandbox athletes and organization routing; causal dates must +be contiguous; committed dates are immutable; maximum 90 missing dates and +25,000 planned records per publication bundle; one writer per dataset version; +scheduled runs perform zero AMS mutations + +**Scale/Scope**: Existing authorized sandbox roster, currently mapped event +forms, a rolling operational horizon usually under 90 days, and one logical +dataset version at a time + +## Constitution Check + +*GATE: Passed before Phase 0 research and rechecked against the Phase 1 design.* + +- **Python-only runtime — PASS**: The engine, stores, CLI, connector calls, and + Databricks deployment are Python-only. +- **Direct AMS/Smartabase API use — PASS**: The synthesis package uses the + installed generic Python connector; no R bridge or copied HTTP client is added. +- **Mutation safety — PASS**: `prepare` is the default and scheduled ceiling; + `publish` is a separate interactive command over immutable plans. +- **Destructive production restriction — PASS**: Update, upsert, overwrite, + profile mutation, and deletion are excluded. Unknown environments fail closed. +- **Auditability — PASS**: Hash-linked daily commits, upload plans, redacted + publication results, and reconciliation evidence preserve intent and outcome. +- **Library-first design — PASS**: The state engine, storage, planning, and + publishing policies live in an installable package; root scripts become thin + compatibility wrappers. +- **Stable interfaces — PASS**: The existing batch generator and synchronization + interfaces remain during migration; new artifact schemas carry explicit + versions and canonical hashing rules. +- **Testing — PASS**: Offline determinism, restart, conflict, concurrency, + confirmation, and partial-result tests are required before rollout. +- **Data protection — PASS**: Runtime artifacts remain ignored/access-controlled; + secrets never enter manifests, parameters, logs, or test fixtures. +- **Reviewability and documentation — PASS**: Work is split into small phases, + with generation correctness completed before upload orchestration and + Databricks deployment. + +**Post-design gate**: PASS. No constitutional exception is required. Fully +unattended AMS publication would fail Principles III and IV and is explicitly +outside this plan. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-daily-data-pipeline/ +├── plan.md +├── research.md +├── data-model.md +├── quickstart.md +├── contracts/ +│ ├── artifact-contracts.md +│ └── cli-contract.md +└── tasks.md # created later by speckit-tasks +``` + +### Source Code (implementation target) + +```text +../ams-sandbox-data-synthesis/ +├── pyproject.toml +├── src/ams_sandbox_synthesis/ +│ ├── __init__.py +│ ├── cli.py # init, prepare, status, publish, reconcile, report +│ ├── config.py # versioned dataset/config loading and hashing +│ ├── models.py # explicit state, commit, and publication models +│ ├── engine.py # initialize and advance one athlete/day +│ ├── orchestration.py # scan, catch up, validate, and commit dates +│ ├── storage.py # portable store protocol and local/Volume store +│ ├── planning.py # remote scope classification and immutable plans +│ ├── publishing.py # interactive master-plan execution/reconciliation +│ └── reporting.py # optional aggregate reports, off the daily path +├── generate_sandbox_data.py # compatibility wrapper during migration +├── refresh_sandbox_data.py # compatibility wrapper during migration +├── sync_smartabase_generated_data.py +└── tests/ + ├── unit/ + ├── integration/ + ├── contract/ + └── fixtures/ # synthetic/redacted only + +ams-python-connector/ +├── src/ams_smartabase/ # existing generic API client, unchanged for MVP +└── tests/ # changed only if a generic batched-read gap is proven +``` + +**Structure Decision**: Put organization-specific synthesis and destination +policy in the synthesis package. Reuse connector public APIs for discovery, +range reads, payload creation, and event insertion. Do not move daily workflow +policy into `ams-python-connector`. Keep root scripts as compatibility shims so +the package can be extracted without one large breaking rewrite. + +## Architecture And Data Flow + +```text +target date + | + v +validate commit chain -> generate missing dates -> commit day + after-state + | + v + classify exact AMS date scopes + / | \ + empty known match conflict/read error + | | | + v v v + freeze plans reconcile/skip block date + | + scheduled run stops here (zero writes) + | + v + interactive publish master plan + | + final scope recheck + typed confirmation + | + v + ordered event inserts; stop on uncertainty +``` + +Generation state and AMS publication state are deliberately separate. A date +can be generated but unpublished without blocking generation of later dates. +This allows irregular preparation while preserving causal simulation history. + +## Implementation Phases + +### Phase 1 — Characterize And Freeze Existing Behavior + +1. Add golden tests for template schemas, routing, complaint duration, recent + practice effects, weekly totals, and representative pattern distributions. +2. Capture the legacy generator version and document intended compatibility + changes; do not promise byte-for-byte equivalence with the current global RNG. +3. Add package metadata and a thin CLI without changing current script entry + points. + +**Exit evidence**: Existing tests pass, golden fixtures are synthetic, and every +intentional output change needed by the stateful engine is reviewed. + +### Phase 2 — Build The Deterministic Daily Engine + +1. Freeze an effective-dated athlete simulation profile keyed by stable AMS + user ID; stop deriving identity or pattern from registry position. +2. Implement `initialize_simulation`, `build_week_plan`, + `generate_athlete_day`, and `generate_date`; implement range generation only + as repeated date transitions. +3. Replace the shared RNG and global counters with SHA-256-derived RNG streams + and stable date/athlete/component identifiers. Version stream names and draw + order. +4. Replace range-relative seasonal progression with an absolute simulation + epoch/campaign phase. +5. Carry fatigue, complaint state, bounded recent-practice history, complete + Monday-Sunday plan context, and future-dated pending events across days. +6. Move weekly calculations and IDs into the engine so renderers only render + already-decided entities. + +**Exit evidence**: One 90-day invocation equals 90 serialized one-day +invocations at every daily manifest and final state hash. + +### Phase 3 — Add Durable Catch-Up Storage And Commands + +1. Implement the store protocol, versioned artifact schemas, canonical JSON + hashing, staging directories, commit-marker-last behavior, and chain + validation. +2. Implement `init`, `prepare`, and `status`. Default `prepare` target is the + previous completed calendar day in `Europe/Amsterdam`; `--through-date` + remains explicit for reproducibility and irregular runs. +3. On normal runs generate `head + 1 ... through_date`. Treat an interior hole + followed by later commits as corruption, not as an independently fillable + gap; provide read-only diagnostics and rebuild guidance. +4. Add a local exclusive lock and a store-level compare-before-commit check. + Configure Databricks job maximum concurrent runs to one; use a conditional + Delta claim/commit for the Databricks adapter before allowing scheduled and + manually triggered runs to share a dataset root. +5. Make report generation a separate `report` command/task. The daily hot path + writes compact validation summaries, not self-contained Plotly dashboards. + +**Exit evidence**: No-op, crash recovery, tamper detection, incompatible version, +and two-writer tests all fail safe without rewriting a commit. + +### Phase 4 — Add Date-Scoped Planning And Interactive Publication + +1. Reuse the existing connector and upload-plan builders. Split payloads into + child plans of at most 500 records and request batches of at most 100. +2. Query all authorized athletes for one mapped form and the complete candidate + date range in one paginated read when contract tests prove result + completeness and attribution. Otherwise retain the existing smaller queries; + request minimization must not weaken correctness. +3. Partition read results by date, athlete, and form, then classify each daily + target as empty, matching a prior immutable publication, unexpected, or + read-uncertain. Any unexpected row blocks that date as a unit. +4. Freeze one ordered catch-up master plan across the earliest contiguous ready + prefix only, binding dates and complete hashes. A blocked earlier date cannot + be skipped. Reject bundles above 90 days or 25,000 records. +5. Implement `publish ` as TTY-only, with a fresh whole-range read, + full preview, existing hash-derived typed confirmation, immutable child + revalidation, and one-time ordered authorization. +6. Preserve the connector's reviewed mutation-state policy: a confirmed result + may continue, and an explicitly classified accepted-but-unverified result may + finish only the already-authorized bundle before mandatory reconciliation. + Stop later writes after mismatch, partial success, unknown, contradictory, or + unclassified results. Recovery starts with `reconcile`, never an automatic + retry; a changed remainder requires a new immutable plan and fresh human + review. + +**Exit evidence**: Offline doubles prove zero writes for scheduled, non-TTY, +conflicting, read-uncertain, changed-plan, mismatched-confirmation, repeated, +partial, and unknown-result cases. + +### Phase 5 — Deploy The Prepare Job To Databricks + +1. Build and install a pinned wheel for the synthesis package plus a pinned + internal connector wheel; run the same CLI entry point as local execution. +2. Provision an access-controlled Unity Catalog Volume for datasets and plans. + Store code in Git/workspace deployment assets, not in the Volume. +3. Retrieve the AMS secret through a Databricks credential adapter into memory. + Do not write a `.env` file or pass plaintext credentials as task parameters. +4. Use a serverless Python wheel task with Standard performance mode, maximum + concurrent runs `1`, queueing, no mutation task, and alerts on failure, + conflict, partial/unknown state, duration, or backlog-limit breach. +5. Schedule for the required freshness, using `Europe/Amsterdam` and a + through-date of yesterday. Start weekly to amortize startup overhead; move to + daily only if daily sandbox freshness is worth the additional cold starts. +6. Add a separate optional weekly report task after successful preparation. + +**Exit evidence**: A deployed prepare job catches up a synthetic test dataset in +a non-production workspace, produces the same hashes as local execution, emits +zero AMS writes, and passes secret/artifact inspection. + +### Phase 6 — Documentation, Rollout, And Cleanup + +1. First update `agents.md`, `project_requirements.md`, and user documentation to + define lightweight daily validation plus periodic/on-demand dashboards. This + deliberately resolves the current instruction that every local generation + produces full Plotly output; implementation must not silently bypass it. +2. Document initialization, local and Databricks preparation, interactive + publication, status interpretation, roster changes, dataset-version cutover, + retention, and recovery. +3. State beside every execution command that this independent project is not + affiliated with, endorsed by, or maintained by Teamworks and that operators + remain responsible for authorization and policy compliance. +4. Run both workflows in shadow prepare-only mode, compare daily outputs and + distribution checks, then enable interactive incremental publication. +5. Keep the legacy 90-day refresh available during a bounded rollback window; + remove it only in a separate reviewed change after the new pipeline proves + stable. + +## Validation Matrix + +| Area | Required evidence | +|---|---| +| Determinism | one-shot vs daily/resumed equality; registry reorder stability; independent RNG streams | +| Longitudinal behavior | fatigue, complaints, recent practice, week boundary, pending next-day event, campaign phase | +| Storage | stage/commit crash points, hash chain, orphan cleanup, immutable rerun, incompatible version, lock contention | +| Dates | leap day, month/year boundary, DST transition, target before head, no-op, long outage | +| AMS reads | pagination, all requested users/forms, missing users, range attribution, incomplete/ambiguous response | +| Planning | empty/matching/unexpected/read-uncertain scopes, 500-child and bundle limits, canonical plan hashes | +| Publication | sandbox verification, final recheck, TTY gate, confirmation mismatch, changed child/order, partial/unknown stop, no retry | +| Protection | no secrets in logs/manifests; generated and fetched athlete artifacts ignored; redacted fixtures only | +| Portability | identical local and Volume manifest hashes from identical state/config; wheel smoke test on Databricks | + +## Rollout And Operational Recommendation + +Start with local `prepare` and `status`, then deploy the identical prepare command +weekly in Databricks. Keep `publish` local and interactive. Once operational +evidence shows the generator is fast and stable, choose frequency solely from +freshness needs: every run catches up causally, so running seven missing days at +once is valid and usually costs less than seven Databricks cold starts. + +Do not pre-generate a year: that freezes future roster/config choices, stores +unused athlete data, and makes longitudinal corrections awkward. Do not rebuild +and delete 90 days: it is the most expensive option and introduces a Critical +destructive workflow. A rolling 90-day consumer view should be a query/filter, +not scheduled deletion. + +## Complexity Tracking + +No constitution violations are planned. The two storage adapters are justified +by the explicit local/Databricks portability requirement; both implement one +small artifact-store protocol and share canonical schemas. The Databricks +adapter adds a compact Delta control ledger because job concurrency settings do +not coordinate every possible manual writer. The planned reporting change is a +reviewed update to lower-level synthesis guidance, not a constitutional +exception; it must land before the daily hot path stops producing dashboards. diff --git a/specs/001-daily-data-pipeline/quickstart.md b/specs/001-daily-data-pipeline/quickstart.md new file mode 100644 index 0000000..6c789f5 --- /dev/null +++ b/specs/001-daily-data-pipeline/quickstart.md @@ -0,0 +1,201 @@ +# Quickstart: Planned Incremental Pipeline + +> This is a design-validation quickstart for the interface to be implemented. +> The `sandbox-data` command does not exist yet. + +This independent project is not affiliated with, endorsed by, or maintained by +Teamworks. The operator remains responsible for authorization, privacy, +security, organizational policy, and every live AMS mutation. + +## Recommended Operating Pattern + +```text +weekly or daily: prepare through yesterday -> inspect status/plan -> notify +when desired: interactive publish -> reconcile +``` + +Preparation may run locally or as a Databricks job. Publication is event-insert +only and remains interactive under current governance. A scheduled job must not +upload, update, upsert, overwrite, or delete AMS data. + +## Prerequisites After Implementation + +- Python 3.10 or newer. +- The synthesis wheel and exact approved connector wheel installed. +- An authorized sandbox athlete registry, templates, and exact form map. +- A protected, ignored local state directory, or a restricted Unity Catalog + Volume and managed Delta control table. +- Credentials supplied by the approved local credential mechanism or a + Databricks secret provider. Never put credentials in commands, notebooks, + `.env` files on a Volume, or operation artifacts. + +## Local Workflow + +### 1. Initialize Once + +```powershell +sandbox-data init --dataset "rowing-sandbox" --dataset-version "v1" --state-root "C:\secure\ams-sandbox-state" --roster "sandbox_state\athletes.csv" --templates "data_formats" --form-map "smartabase_form_map.json" --simulation-epoch "2026-01-01" --seed "20260429" +``` + +Review the frozen dataset/config, athlete-profile assignments, routing summary, +and hashes. If any output-affecting input changes later, create a new dataset +version or a reviewed effective-dated cutover. + +### 2. Generate Missing Dates Without AMS Access + +```powershell +sandbox-data prepare --dataset "rowing-sandbox" --state-root "C:\secure\ams-sandbox-state" --through-date "2026-08-10" --remote-check skip +``` + +Expected first-run behavior: + +- validates the current commit chain; +- generates dates oldest first, at most 90 per invocation; +- commits each date independently; +- records `UNPLANNED` publication state; and +- performs zero AMS requests and mutations. + +Run the same command again to validate the no-op. If more than 90 days remain, +rerun; the next invocation resumes at the next causal date. + +### 3. Prepare A Publication Bundle + +Load credentials through the existing protected local mechanism, then run: + +```powershell +sandbox-data prepare --dataset "rowing-sandbox" --state-root "C:\secure\ams-sandbox-state" --through-date "2026-08-10" --remote-check require --credential-provider local +``` + +This reuses committed data, reads only the earliest contiguous unpublished AMS +interval, classifies every exact date/athlete/form scope, and freezes plans for +empty scopes. It performs zero writes. A remote read error or unexpected row +blocks the affected date and all later dates from that publication bundle. + +### 4. Inspect Status And Artifacts + +```powershell +sandbox-data status --dataset "rowing-sandbox" --state-root "C:\secure\ams-sandbox-state" +``` + +Verify at minimum: + +- exact normalized URL is the intended `/sandbox` environment; +- dataset and generator versions are expected; +- generation dates form one hash-valid contiguous chain; +- planned dates are the earliest unpublished dates; +- athlete IDs, forms, fields, counts, and child hashes are expected; +- no profile/update/upsert/delete operation is present; and +- total scope is within 90 dates, 25,000 records, 500 records per child, and 100 + records per request batch. + +### 5. Publish Separately And Interactively + +Use the exact plan path printed by `status`: + +```powershell +sandbox-data publish --publication-plan "C:\secure\ams-sandbox-state\datasets\rowing-sandbox\v1\bundles\BUNDLE_ID\master_upload_plan.json" --execute --credential-provider local +``` + +The command performs a fresh read, prints the complete scope, and asks for the +plan-derived typed phrase. Type it yourself only after reviewing the preview. +Do not pipe it, script it, store it, or ask an agent to provide it. + +If the command reports partial, unknown, contradictory, or read-uncertain state, +do not rerun it. Start with the read-only command: + +```powershell +sandbox-data reconcile --publication-plan "C:\secure\ams-sandbox-state\datasets\rowing-sandbox\v1\bundles\BUNDLE_ID\master_upload_plan.json" --credential-provider local +``` + +## Databricks Prepare Job + +### 1. Provision Protected State + +Create: + +- a restricted managed Unity Catalog Volume for inputs and immutable artifacts; +- a managed Delta control table keyed by dataset version and event date; +- a dedicated secret scope readable only by the job principal; +- preferably a read-only AMS credential for scheduled scope checks; and +- a restricted serverless network policy allowing only the reviewed AMS sandbox + FQDN and approved package source. + +Do not copy the local `.env` file into the Volume. Serverless jobs use a runtime +credential adapter and in-memory credentials. + +### 2. Deploy The Pinned Wheels And Saved Job + +```bash +databricks bundle validate -t prod +databricks bundle deploy -t prod +``` + +The bundle defines one saved serverless Python wheel task using Standard +performance mode, `max_concurrent_runs: 1`, queueing, timeout/duration warnings, +failure notifications, and non-sensitive cost tags. Its command is equivalent +to: + +```text +sandbox-data prepare --dataset DATASET_ID --state-root /Volumes/CATALOG/SCHEMA/VOLUME --remote-check require --credential-provider databricks +``` + +The job has no `publish` task and no mutation-capable parameter. + +### 3. Validate Manually Before Scheduling + +Run the saved job for an explicit historical test date in a non-production +workspace. Confirm: + +- local and Databricks runs from identical initial state produce the same daily + manifest hashes; +- the Delta ledger points only at hash-valid Volume manifests; +- no credentials appear in task parameters, logs, Volume files, or Delta rows; +- no AMS mutation method is called; and +- a second run is a validated no-op. + +### 4. Schedule For Freshness, Not Correctness + +Start weekly in timezone `Europe/Amsterdam`; each run targets yesterday and +catches up all missing dates in one task. Move to daily only if one-day freshness +is required. Missing-date calculation is authoritative, so a delayed or skipped +trigger does not lose a date. + +### 5. Human Publication Of Databricks-Prepared Plans + +For the initial Databricks rollout, use a secure interactive terminal with +access to the authoritative Volume and run the same `sandbox-data publish` +command against its exact plan path. This may be a tightly controlled +Databricks web terminal on interactive compute. Do not convert a notebook/job +parameter into confirmation. + +If workstation publication of a Databricks-owned dataset is required, implement +and review a remote artifact-store/export-result adapter before rollout. Do not +advance one dataset from a copied, unsynchronized local directory. A wholly +local dataset remains fully supported by the local workflow above. + +## Periodic Reports + +Build the heavier self-contained Plotly inspection output separately: + +```powershell +sandbox-data report --dataset "rowing-sandbox" --state-root "C:\secure\ams-sandbox-state" --from-date "2026-08-01" --through-date "2026-08-10" +``` + +Daily preparation retains lightweight validation summaries. Before implementing +this split, update the synthesis repository's existing reporting requirements so +the daily exception and periodic reporting obligation are explicit. + +## Acceptance Smoke Test + +From one synthetic initial dataset version: + +1. Prepare 90 dates in one invocation and save manifest hashes. +2. Reset only the synthetic test store and prepare the same period with 90 + one-day invocations, serializing/reloading after each. +3. Assert every daily manifest and the ending checkpoint match. +4. Repeat the final `prepare`; assert a no-op and zero AMS requests. +5. Reorder registry rows; assert existing athlete profiles and output stay the + same. +6. Inject an unexpected AMS row in an offline double; assert zero mutation calls + and a blocked date. +7. Invoke `publish` without a TTY; assert refusal before any mutation method. diff --git a/specs/001-daily-data-pipeline/research.md b/specs/001-daily-data-pipeline/research.md new file mode 100644 index 0000000..779c9c7 --- /dev/null +++ b/specs/001-daily-data-pipeline/research.md @@ -0,0 +1,295 @@ +# Research: Incremental Synthetic Data Pipeline + +## Decision 1 — Use Stateful Incremental Catch-Up + +**Decision**: Generate only the causally missing interval from the last +hash-valid committed checkpoint through the requested date, one date at a time. + +**Rationale**: The current generator already carries longitudinal values inside +one invocation—fatigue, complaint state, practice history, and active complaints— +but loses them between invocations. Making that state explicit produces the same +pattern behavior without regenerating or deleting prior dates. It also makes a +daily, weekly, or irregular run the same operation. + +**Alternatives considered**: + +- **Regenerate/delete/upload 90 days**: rejected. It repeats nearly all compute + and network work and turns ordinary maintenance into a Critical destructive + workflow. +- **Generate one independent day with the current function**: rejected. The + current global RNG, counters, registry indexes, range-relative day index, and + in-memory history reset, so dates are not longitudinally equivalent. +- **Pre-generate a year and release one partition per day**: rejected. It has + cheap daily reads but freezes future roster/config choices, retains unused + athlete data, complicates correction, and still needs durable publication + state. +- **Stateful incremental generation**: selected. Its work is proportional to + missed dates and it naturally supports irregular schedules. + +## Decision 2 — Make One Day A Pure Versioned Transition + +**Decision**: Refactor into `generate_athlete_day(profile, state, day, +week_plan, rng_streams)` and `generate_date(simulation_state, day)`. A range is +only repeated daily transitions. + +**Rationale**: The current athlete-outer/day-inner loop uses a shared +`random.Random(seed)` and global counters. Reversing loops alone changes random +draw order and does not create restart equivalence. Explicit transitions make +state serializable and independently testable. + +The checkpoint carries fatigue, previous-complaint flag, complete active +complaint details, bounded recent practiced dates, the full Monday-Sunday plan, +and future-dated pending events. A Sunday complaint can currently decide a +Monday consultation, so an output partition must be based on actual event date, +not only decision date. + +**Alternatives considered**: + +- Persist `Random.getstate()` and counters: smaller initial edit, but brittle to + changes in iteration order and unrelated random draws. +- Replay all history before each day: deterministic but reintroduces linear + historical compute and makes corruption recovery opaque. + +## Decision 3 — Derive Random Streams And IDs From Stable Inputs + +**Decision**: Seed independent component streams with SHA-256 over canonical +`(generator_version, base_seed, athlete_id, ISO date, stream_name)` and derive +stable IDs from the same identity plus per-day ordinal. + +**Rationale**: This removes registry-order, invocation-length, and global-counter +dependencies. Separate plan, outcome, complaint, wellness, and Garmin streams +prevent a new draw in one component from perturbing unrelated output. + +Python `hash()` is excluded because it is process-randomized. Numeric IDs use a +documented bounded digest mapping plus collision validation. Any change to +stream names, draw order, or ID policy increments `generator_version`. + +## Decision 4 — Freeze Absolute Pattern And Roster Context + +**Decision**: Use a stable simulation epoch/campaign phase and an effective- +dated `AthleteSimulationProfile` keyed by AMS user ID. + +**Rationale**: The current seasonal factor depends on `day_index / total_days`, +so the same date changes when invocation length changes. Current pattern, +synthetic age, squad, physiology, location, and some IDs depend on registry row +position. Freezing those inputs retains pattern character across resumed runs +and registry reorder. + +New athletes receive explicit `effective_from` profiles and fresh baseline +state. Removed athletes receive `effective_to`; their history is retained. +Pattern or profile changes are new versions or explicit future cutovers, never +silent historical reinterpretation. + +## Decision 5 — Use Hash-Linked Daily Commits + +**Decision**: Stage all artifacts, validate and hash them, then write an +immutable daily commit manifest last. Each manifest binds its predecessor and +before/after checkpoints. + +**Rationale**: A filename alone cannot distinguish complete from interrupted +generation. Commit-marker-last behavior allows safe recovery from orphan +staging, prevents rewriting a committed date, and detects hidden interior gaps. + +The normal missing range is `contiguous_head + 1 ... through_date`. A missing +date behind a later commit is corruption because later state may depend on it; +it is not independently synthesized. A deterministic repair is allowed only in +a separately designed repair mode that proves the regenerated after-state hash +equals the already recorded successor. + +## Decision 6 — Keep Generation And Publication State Separate + +**Decision**: Later dates may be generated while earlier dates await human +publication. Publication itself advances only through the earliest contiguous +ready prefix. + +**Rationale**: Generation is a causal local simulation concern; publication is +a high-risk remote mutation concern. Coupling them would make a transient AMS +read or unavailable operator block future state. Allowing publication to skip a +blocked earlier date would create a silent remote gap, so only the contiguous +ready prefix enters a master plan. + +## Decision 7 — Prepare On Schedule, Publish Interactively + +**Decision**: The unified command exposes `prepare`, `status`, `publish`, and +`reconcile`, but scheduled Databricks execution stops after preparation. + +**Rationale**: Connector governance requires dry-run planning and separate +human confirmation for live writes. Scheduled workflows cannot supply that +confirmation. Existing synthesis code also rejects live upload from a non-TTY. +The interactive publisher reuses the existing immutable master-plan, +hash-derived typed-confirmation, child revalidation, execution reservation, and +partial/unknown-stop behavior. + +**Alternative considered**: A job parameter or secret containing confirmation +was rejected because it converts confirmation into credentials/configuration +and bypasses the required human decision. A future unattended insert workflow +requires a separate governance decision, dedicated derived destination forms, +least-privileged credentials, and a new design; it is not an implementation +phase of this feature. + +## Decision 8 — Classify Exact Daily AMS Scopes Before Planning + +**Decision**: Classify candidate scopes as `EMPTY`, +`MATCHING_WITH_LINEAGE`, `UNEXPECTED_NONEMPTY`, or `READ_UNCERTAIN`. + +**Rationale**: Empty scopes can safely produce insert plans. A matching scope is +safe only when immutable local plan/execution lineage exists and exact +versioned row-multiset reconciliation succeeds. Existing AMS records without +that lineage may be unrelated and must block rather than be adopted or deleted. +Incomplete reads, missing stable athlete attribution, or pagination uncertainty +also block. + +The pipeline retains the initial read, fresh pre-execution read, and final +reconciliation. These requests are safety controls. Partial or unknown mutation +results never retry automatically. A known exact subset can enter only the +existing reviewed continuation model with a new immutable remainder plan and +fresh authorization. + +## Decision 9 — Batch Read-Only Checks By Form And Interval When Proven + +**Decision**: Aim for one authenticated paginated read per mapped form over the +earliest candidate interval, with all stable athlete IDs, then partition locally. + +**Rationale**: The connector accepts multiple `user_ids`, while the current +fetch caller loops athlete by form. For 11 forms, a complete batched pass can be +roughly form-count requests rather than athlete-count multiplied by form-count, +subject to pagination. One client/session is reused per stage. + +**Constraint**: This optimization is gated by offline and live-sandbox contract +evidence proving server limits, pagination, zero-result coverage, and stable +athlete attribution. If completeness cannot be proven, the caller retains +smaller requests. Request reduction never weakens fail-closed classification. + +## Decision 10 — Use Filesystem Artifacts Locally, Volume Plus Delta On Databricks + +**Decision**: Keep one logical `StateStore`. Use a local directory with tested +exclusive locking for workstation datasets. Use a restricted managed Unity +Catalog Volume for non-tabular artifacts and a small managed Delta control +table for claims, commit/status indexing, and optimistic conflict detection in +Databricks. + +**Rationale**: Unity Catalog Volumes are designed for governed path-based files +and expose `/Volumes////...`, which fits the existing +CSV/JSON/`pathlib` model. Delta supplies ACID commits for coordination when a +scheduled and manually triggered run might overlap. Immutable manifest content, +not the Delta index, remains authoritative. See [Unity Catalog +Volumes](https://learn.microsoft.com/en-us/azure/databricks/volumes/) and +[Delta Lake ACID guarantees](https://learn.microsoft.com/en-us/azure/databricks/lakehouse/acid). + +**Alternatives considered**: + +- Volume JSON pointer only: acceptable for a prototype only after testing + exclusive-create/replace semantics across processes; not assumed for the + durable job. +- Store every generated row in Delta: stronger tabular querying but adds Spark + coupling to the local core and disrupts the current CSV/template workflow. +- SQLite on the Volume: rejected because shared/network filesystem locking is + the wrong concurrency primitive. + +One dataset version has one authoritative state root. Unsynchronized local and +Databricks copies cannot both advance it. Operational local control of a +Databricks-owned dataset should trigger the saved job or use a reviewed complete +store transfer, not create a competing history. + +## Decision 11 — Package A Wheel And Run One Serverless Job Task + +**Decision**: Convert reusable synthesis behavior to an installable wheel, pin +the connector wheel, and deploy one saved serverless Python wheel job through a +Databricks Asset Bundle/Declarative Automation Bundle. + +**Rationale**: One task processes all missing dates and pays startup overhead +once. Databricks documents wheel tasks as a reliable packaged job form and +supports building/deploying them through Bundles. See [Python wheel tasks](https://learn.microsoft.com/en-us/azure/databricks/jobs/how-to/use-python-wheels-in-workflows) +and [Bundle wheel workflow](https://learn.microsoft.com/en-us/azure/databricks/dev-tools/bundles/python-wheel). + +Use serverless Standard performance mode for cost efficiency. Databricks notes +that it uses fewer DBUs but can add roughly four to six minutes of startup, a +good trade for a non-urgent prepare job. Use a saved Job, because standard mode +is not available to one-off `runs/submit` executions. See [Run serverless +Jobs](https://learn.microsoft.com/en-us/azure/databricks/jobs/run-serverless-jobs). + +Set maximum concurrent runs to one, enable queueing, add timeout/duration +warnings and failure notifications, and tag runs for cost attribution. See +[job concurrency and queueing](https://learn.microsoft.com/en-gb/azure/databricks/jobs/configure-job), +[job notifications](https://learn.microsoft.com/en-us/azure/databricks/jobs/notifications), +and [Jobs system tables](https://learn.microsoft.com/en-us/azure/databricks/admin/system-tables/jobs). + +Generation retries remain disabled initially and may be enabled only after +deterministic crash/retry tests. Any future mutation task must have no retries, +no timeout retry, and serverless auto-optimization retries disabled; Databricks +specifically calls this out for at-most-once tasks in the [serverless Jobs +documentation](https://learn.microsoft.com/en-us/azure/databricks/jobs/run-serverless-jobs). + +## Decision 12 — Prefer Weekly Scheduling Unless Daily Freshness Is Required + +**Decision**: Start with a weekly prepare schedule; use daily only when the +sandbox must be no more than one day behind. + +**Rationale**: Missing-date catch-up makes frequency a freshness decision, not a +correctness decision. A lightweight Python generation is likely shorter than +serverless startup, so processing seven dates in one weekly run usually costs +less than seven cold starts. The target remains yesterday in +`Europe/Amsterdam`; explicit missing-date calculation is authoritative even if +daylight-saving behavior changes a trigger time. Databricks supports named +timezones and documents DST caveats in [scheduled +Jobs](https://learn.microsoft.com/en-us/azure/databricks/jobs/scheduled). + +The implementation records wall time, DBUs/cost attribution metadata, generated +dates, rows, and AMS request counts. Revisit cadence after four weeks of actual +runs rather than assuming generation is the cost driver. + +## Decision 13 — Inject Credentials; Never Create `.env` On A Volume + +**Decision**: The core accepts a credential provider or constructed client. +Local execution may use the existing protected environment/credential flow. +Databricks retrieves a narrowly scoped secret at runtime and passes credentials +only in memory. + +**Rationale**: Serverless compute does not support configured environment +variables, so copying the local `.env` convention is neither supported nor +safe. See [serverless compute +limitations](https://learn.microsoft.com/en-us/azure/databricks/compute/serverless/limitations) +and [Databricks secret management](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/). + +Use a dedicated read-only AMS principal for scheduled classification if the AMS +authorization model permits. Interactive publication uses separately controlled +write credentials. Grant the job principal only the minimum Volume, Delta, and +secret-scope privileges. A Key Vault-backed secret scope is appropriate where +central rotation already exists; otherwise a dedicated Databricks-backed scope +is simpler. + +With serverless egress controls, allow only the reviewed AMS sandbox FQDN and +approved package source, while retaining the application's exact `/sandbox` +verification. See [serverless egress +policies](https://learn.microsoft.com/en-us/azure/databricks/security/network/serverless-network-security/network-policies). + +## Decision 14 — Move Heavy Reports Off The Daily Hot Path + +**Decision**: Daily commits store lightweight validation/statistics. The +self-contained Plotly dashboard is generated weekly, per publication bundle, or +on demand by `report`. + +**Rationale**: The current generator renders all dashboards for every run. That +is disproportionate for a one-day task and creates unnecessary CPU and files. +Separating reporting preserves inspection without coupling it to causal state. + +**Repository-policy impact**: Current synthesis guidance says local generation +also produces timeline/PDF dashboards. Before implementation, update +`agents.md`, `project_requirements.md`, and user documentation in a reviewed +change to define the daily lightweight exception and periodic report obligation. +This is an explicit prerequisite, not a silent bypass. + +## Decision 15 — Bound Work Without A Force Flag + +**Decision**: Advance at most 90 generation dates per invocation and freeze no +publication bundle above 90 dates or 25,000 records. Existing child plans remain +at most 500 records and request batches at most 100. + +**Rationale**: Bounds keep previews reviewable, limit one authorization's blast +radius, and avoid runaway catch-up after a configuration mistake. A backlog over +90 dates is processed in repeated validated invocations; a publication backlog +is split into separately reviewed contiguous bundles. There is no `--force` or +unlimited override. + +The 25,000-record figure is a project safeguard, not an asserted AMS limit. Any +change requires documented safety review and tests. diff --git a/specs/001-daily-data-pipeline/spec.md b/specs/001-daily-data-pipeline/spec.md new file mode 100644 index 0000000..8d73541 --- /dev/null +++ b/specs/001-daily-data-pipeline/spec.md @@ -0,0 +1,165 @@ +# Feature Specification: Incremental Synthetic Data Pipeline + +**Feature Branch**: `001-daily-data-pipeline` (Spec Kit feature identifier; +the existing worktree remains on `agent/native-event-parsing`) + +**Created**: 2026-08-11 + +**Status**: Draft + +**Input**: User description: "Create a daily or periodically run synthetic-data pipeline that detects missing dates, generates those dates sequentially with longitudinal state, and provides one safe workflow for local or scheduled Databricks use." + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Fill Missing Dates With Longitudinal Data (Priority: P1) + +An operator runs the pipeline for a requested through-date. The pipeline identifies every uncommitted date after the last complete date, generates the missing dates in chronological order, and carries each athlete's relevant history into the next date. + +**Why this priority**: This creates the core value: the same workflow works daily, after an interruption, or after an irregular gap without regenerating completed history. + +**Independent Test**: Generate a fixed interval once, then generate the same interval through consecutive daily invocations. The committed daily outputs and ending state must be equivalent. + +**Acceptance Scenarios**: + +1. **Given** a complete committed dataset through 1 August, **When** the operator requests generation through 5 August, **Then** the pipeline generates 2–5 August in order and commits each date once. +2. **Given** the requested through-date is already complete, **When** the pipeline runs, **Then** it reports a successful no-op and does not rewrite committed dates. +3. **Given** a partially written but uncommitted date, **When** the pipeline reruns, **Then** it rejects or replaces only the incomplete attempt from the last valid checkpoint and does not advance from uncertain state. + +--- + +### User Story 2 - Run The Same Workflow Locally Or On A Schedule (Priority: P2) + +An operator can invoke one pipeline contract from a local machine or a persistent scheduled workspace. Both contexts use the same date-selection, generation, validation, state, and operation-plan rules. + +**Why this priority**: A single operational path prevents local runs and scheduled runs from producing incompatible datasets or bypassing safeguards. + +**Independent Test**: Run the workflow in two isolated environments from identical configuration and prior state, then compare the committed manifests and plan hashes. + +**Acceptance Scenarios**: + +1. **Given** identical prior state and configuration, **When** the same missing date is prepared locally and in a scheduled environment, **Then** both produce the same daily content and manifest fingerprint. +2. **Given** a non-interactive scheduled run, **When** missing dates are found, **Then** generation and upload planning may complete but no live AMS mutation is executed. +3. **Given** a local interactive run, **When** reviewed daily upload plans are ready, **Then** the operator can continue through the existing scope-derived confirmation workflow. + +--- + +### User Story 3 - Publish Only Safe Missing Daily Scopes (Priority: P3) + +An operator prepares new synthetic event records for dates that are absent from the target sandbox scope. Previously published dates are reconciled and skipped, while unexpected existing records block the affected daily scope without deletion or partial conflict-free upload. + +**Why this priority**: Append-only daily publication minimizes requests and avoids making destructive refresh behavior routine. + +**Independent Test**: Exercise empty, previously published, partially populated, and unexpectedly populated target dates using offline connector doubles and immutable plans. + +**Acceptance Scenarios**: + +1. **Given** an empty target date, **When** the pipeline prepares publication, **Then** it freezes an event-only immutable upload plan for that date. +2. **Given** a target date matching a previously published immutable plan, **When** the pipeline reconciles it, **Then** it marks the date complete without uploading it again. +3. **Given** unexpected existing records in any selected athlete/form scope for a date, **When** the pipeline checks that date, **Then** it blocks all upload for that date and neither deletes nor modifies existing records. +4. **Given** an upload with a partial or uncertain result, **When** the workflow stops, **Then** no automatic mutation retry occurs and recovery begins with read-only reconciliation. + +--- + +### User Story 4 - Evolve The Athlete Population Safely (Priority: P4) + +An operator can refresh the authorized sandbox athlete registry without changing the previously frozen pattern identity or historical data for existing athletes. + +**Why this priority**: Long-running scheduled synthesis must tolerate roster changes without silently rewriting established athlete histories. + +**Independent Test**: Add, remove, and reorder registry rows between daily runs and verify stable assignments and bounded effective dates. + +**Acceptance Scenarios**: + +1. **Given** existing athletes are reordered in the registry, **When** generation continues, **Then** their pattern assignments and future deterministic outputs remain stable. +2. **Given** a new authorized athlete appears, **When** the next run starts, **Then** the athlete receives an explicit effective-from date and initialized state without changing existing athletes. +3. **Given** an athlete is no longer selected, **When** later dates are generated, **Then** no new records are created for that athlete and existing committed history remains unchanged. + +### Edge Cases + +- The requested through-date precedes the last committed date. +- One or more dates are absent between otherwise committed dates. +- A daily partition exists but its manifest, state checkpoint, or hashes are missing or invalid. +- Configuration, generator version, seed, templates, form mappings, or athlete assignments change mid-series. +- A complaint or another generated entity begins on one date and produces a record on a later date. +- A week crosses a month, year, daylight-saving boundary, or scheduled-job outage. +- Multiple runs target the same dataset and date concurrently. +- A new athlete is first observed while the pipeline is catching up several historical dates. +- AMS contains records for a date that has no matching local publication evidence. +- AMS accepted a write but returned an incomplete or ambiguous acknowledgement. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The pipeline MUST accept an explicit target dataset and through-date and determine missing dates from durable committed state rather than from wall-clock assumptions alone. +- **FR-002**: The pipeline MUST process missing dates strictly in chronological order from the last valid checkpoint. +- **FR-003**: Each generated athlete-day MUST depend on an explicit prior athlete state containing only the history needed for future generation. +- **FR-004**: Given the same dataset version, configuration, athlete, date, and prior state, generation MUST produce identical content and identifiers. +- **FR-005**: Existing committed daily data MUST be immutable; reruns MUST be no-ops when hashes match and fail closed when they differ. +- **FR-006**: A date MUST become committed only after its generated records, validation result, next-state checkpoint, configuration references, and hashes are complete. +- **FR-007**: The system MUST preserve active multi-day conditions, recent-practice effects, fatigue, weekly planning context, and pending future-dated records across run boundaries. +- **FR-008**: Identifiers MUST remain unique and stable across independently generated dates and retries. +- **FR-009**: Pattern phase and progression MUST be based on a stable simulation timeline rather than the length of the current invocation. +- **FR-010**: Athlete-to-pattern assignments MUST remain stable when the registry is reordered or refreshed. +- **FR-011**: New and removed athletes MUST have explicit effective dates; roster changes MUST NOT rewrite committed history for existing athletes. +- **FR-012**: The same user-facing pipeline contract MUST support local interactive use and non-interactive scheduled preparation. +- **FR-013**: Scheduled execution MUST stop after read-only checks, generation, validation, and immutable upload-plan preparation; it MUST NOT supply confirmation or execute live AMS mutation under current governance. +- **FR-014**: Live upload MUST remain a separate interactive operation using the existing plan-derived confirmation, immutable-plan validation, sandbox verification, bounded batches, and audit results. +- **FR-015**: The incremental workflow MUST insert event records only; profile upsert, update, replace, overwrite, and deletion are out of scope. +- **FR-016**: Before preparing a new daily upload, the pipeline MUST classify the target daily scope as empty, previously published and matching, unexpectedly populated, or read-uncertain. +- **FR-017**: Unexpectedly populated or read-uncertain scopes MUST block the complete daily upload without deletion or partial conflict-free upload. +- **FR-018**: Previously published dates MUST be reconciled against their immutable local publication plan and MUST NOT be uploaded again. +- **FR-019**: Partial, unknown, or contradictory mutation results MUST stop processing without automatic mutation retry and MUST retain enough redacted audit metadata for read-only recovery. +- **FR-020**: The pipeline MUST detect concurrent attempts for the same dataset/date and permit at most one successful commit and one authorized publication path. +- **FR-021**: Generated athlete data, checkpoints, payload material, and operation artifacts MUST remain in ignored or access-controlled persistent storage and MUST never contain credentials. +- **FR-022**: Human-readable status MUST distinguish generated, planned, ready for review, published, reconciled, blocked, partial/unknown, and failed dates. +- **FR-023**: The pipeline MUST support catch-up after missed or irregular runs without requiring regeneration or deletion of earlier committed dates. +- **FR-024**: A configuration or schema change that could alter future generation MUST create an explicit new dataset version or reviewed cutover; it MUST NOT silently continue an incompatible state series. + +### Constitution Impact + +- **Runtime boundary**: Python-only runtime in both repositories; no R, Rscript, or `smartabaseR` execution. +- **AMS/Smartabase API use**: Read-only checks and event insert planning/execution use the installed Python connector directly. +- **Mutation safety**: High-risk event creation remains dry-run-first and human-confirmed. Scheduled live mutation, profile upsert, update, overwrite, and deletion are excluded. +- **Auditability**: Adds versioned daily generation checkpoints, immutable commit manifests, daily upload plans, publication status, and redacted reconciliation evidence. +- **Public interfaces**: Introduces a reusable incremental-generation API, persistent state/manifest schemas, and a unified local/scheduled command contract. +- **Testing evidence**: Requires offline equivalence, determinism, gap recovery, concurrency, conflict, confirmation, partial-result, and artifact-safety tests; live sandbox tests remain opt-in. +- **Credentials and athlete data**: Credentials remain runtime-only; registry, generated data, state, and operation artifacts remain uncommitted and access-controlled. +- **Documentation**: Requires local and scheduled quickstarts, state/storage lifecycle guidance, recovery instructions, and warnings beside upload execution. + +### Key Entities + +- **Dataset Version**: Immutable simulation identity binding the seed, generator policy, timeline, templates, form mappings, and athlete-pattern assignments. +- **Athlete State**: Minimal longitudinal state needed to generate the athlete's next day, including effective date and last generated date. +- **Daily Generation**: All generated entities attributable to one simulation date, including records scheduled for later emission. +- **Pending Event**: A generated entity decided on one date but emitted on a later event date. +- **State Checkpoint**: Immutable ending state for every selected athlete after a committed date. +- **Daily Commit Manifest**: Hash-bound declaration that a date's data, validation, and next checkpoint are complete. +- **Daily Upload Plan**: Immutable, event-only plan for one target date and exact sandbox scope. +- **Publication Record**: Redacted outcome and reconciliation status for a daily upload plan. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: For a fixed dataset version, a 90-day one-shot run and 90 consecutive daily runs produce identical daily manifests and equivalent ending state. +- **SC-002**: After any interruption between daily commits, a rerun generates every missing date exactly once and rewrites zero previously committed dates. +- **SC-003**: Repeating a preparation run for the same date produces the same daily and upload-plan fingerprints in 100% of offline determinism tests. +- **SC-004**: A run with no missing dates completes as a successful no-op without creating a new generation or mutation attempt. +- **SC-005**: Registry row reordering changes zero existing athlete assignments or previously deterministic future outputs. +- **SC-006**: Every unexpected non-empty or read-uncertain daily AMS scope causes zero write, update, upsert, or delete requests. +- **SC-007**: Every non-interactive scheduled run causes zero live AMS mutation requests under current governance. +- **SC-008**: Every partial or uncertain upload result stops later mutation attempts and produces an actionable read-only recovery status. +- **SC-009**: An operator can use one documented workflow locally or on a schedule to prepare all dates missed since the last successful run without manually listing those dates. +- **SC-010**: No credential or generated/fetched athlete-data artifact is introduced into source control by the feature's automated tests or normal workflows. + +## Assumptions + +- Version 1 targets explicitly mapped event forms in a verified Smartabase sandbox and excludes Athlete Profile. +- The desired synthetic timeline is append-only. Consumers can select a rolling 90-day view without scheduled deletion of older events. +- A non-interactive schedule prepares plans but does not perform live AMS mutation under the current constitution and risk model. +- A human may run the same pipeline interactively to review and confirm a prepared daily upload. +- Persistent storage is available to both local and scheduled executions; its concrete implementation is selected during planning. +- One logical writer operates per dataset version. Concurrent attempts are rejected or serialized. +- New athletes begin on an explicit effective-from date and are not backfilled before that date unless a future reviewed requirement adds that behavior. +- Existing connector batch, exact-target, dry-run, confirmation, and partial-result safeguards remain authoritative. From 07fac5e20450d3d7161562730ca85950e101d273 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 13 Aug 2026 13:59:00 +0200 Subject: [PATCH 6/7] Update daily data pipeline documentation and add task list for implementation phases. --- agent-docs/current-work.md | 40 +++++++++++ specs/001-daily-data-pipeline/spec.md | 2 +- specs/001-daily-data-pipeline/tasks.md | 99 ++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 specs/001-daily-data-pipeline/tasks.md diff --git a/agent-docs/current-work.md b/agent-docs/current-work.md index 0f5724e..f134558 100644 --- a/agent-docs/current-work.md +++ b/agent-docs/current-work.md @@ -2,6 +2,46 @@ This file is the current milestone and handoff note for coding agents. It should stay short and should not duplicate the long-term roadmap in [../docs/roadmap.md](../docs/roadmap.md). +## 001 Daily Data Pipeline (Implemented 2026-08-12) + +The sibling `ams-sandbox-data-synthesis` repository now contains the stateful, +append-only daily pipeline described by `specs/001-daily-data-pipeline`. +Applicable connector aims are B2, B4, B6-B8, C1-C3, C5-C8, E1, E4, E6-E9, +F3, F5, F7, and F8; the highest risk remains **High** because a separate local +interactive command can insert sandbox events. + +Implemented boundaries: + +- deterministic date transitions with immutable hash-linked checkpoints, + missing-date catch-up, stable IDs, weekly context, pending events, and + effective-dated roster cutovers; +- one local CLI for WSV, KNLTB, or both, including read-only remote + classification, immutable daily/master upload plans, status, and reporting; +- a protected-Volume/Delta Databricks adapter and a paused serverless + prepare-only job; scheduled publication is deliberately absent; and +- interactive event-only publication with a fresh complete scope read, exact + target/plan/hash checks, a real-TTY typed phrase, single-use authorization, + no mutation retry, and exact read-only reconciliation. + +Verification evidence: + +- 47 daily-focused synthesis tests pass after final safety changes; +- the complete connector suite passes 147 tests with 1 opt-in live test skipped; +- the synthesis wheel and connector wheel build, the synthesis wheel contains + all required root modules/package files and no state or credential files, and + an external wheel-path import smoke test passes; +- CLI help, form-map JSON, Databricks YAML structural safeguards, Python + compilation, task format/completion, and scoped `git diff --check` pass; and +- the full sibling synthesis suite runs 188 tests; 186 pass and two pre-existing + refresh tests error because current ignored template/form-map fixtures do not + match their assumptions. Neither failing test touches the new daily modules. + +No live AMS request, mutation, Databricks bundle validation, deployment, or +schedule activation was performed. The first recommended next action is the +README's local WSV-yesterday `init`/`prepare --remote-check skip` smoke test, +followed by the read-only `--remote-check require` step under an authorized +operator account. + ## Current Milestone The next major workflow is a sandbox-safe synthetic-data pipeline: diff --git a/specs/001-daily-data-pipeline/spec.md b/specs/001-daily-data-pipeline/spec.md index 8d73541..d6854a5 100644 --- a/specs/001-daily-data-pipeline/spec.md +++ b/specs/001-daily-data-pipeline/spec.md @@ -5,7 +5,7 @@ the existing worktree remains on `agent/native-event-parsing`) **Created**: 2026-08-11 -**Status**: Draft +**Status**: Implemented (offline verified; live sandbox and Databricks deployment unverified) **Input**: User description: "Create a daily or periodically run synthetic-data pipeline that detects missing dates, generates those dates sequentially with longitudinal state, and provides one safe workflow for local or scheduled Databricks use." diff --git a/specs/001-daily-data-pipeline/tasks.md b/specs/001-daily-data-pipeline/tasks.md new file mode 100644 index 0000000..929ae78 --- /dev/null +++ b/specs/001-daily-data-pipeline/tasks.md @@ -0,0 +1,99 @@ +# Tasks: Incremental Synthetic Data Pipeline + +**Input**: Design documents under `specs/001-daily-data-pipeline/` +**Implementation repository**: `../ams-sandbox-data-synthesis` + +## Phase 1: Setup + +- [X] T001 Add installable synthesis package metadata and console entry point in `../ams-sandbox-data-synthesis/pyproject.toml` and `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/__init__.py` +- [X] T002 Add daily dataset, staging, plan, audit, build, and Databricks artifact exclusions in `../ams-sandbox-data-synthesis/.gitignore` +- [X] T003 Record the lightweight-daily versus periodic-Plotly reporting policy before changing runtime behavior in `../ams-sandbox-data-synthesis/project_requirements.md` and `../ams-sandbox-data-synthesis/agents.md` + +## Phase 2: Foundational Models And Storage + +- [X] T004 [P] Add offline canonical-hash, schema-validation, and state-transition tests in `../ams-sandbox-data-synthesis/tests/test_daily_models.py` +- [X] T005 Implement versioned dataset, athlete profile/state, checkpoint, daily manifest, and publication models in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/models.py` +- [X] T006 [P] Add crash, immutable-commit, corruption, no-op, and lock-contention tests in `../ams-sandbox-data-synthesis/tests/test_daily_storage.py` +- [X] T007 Implement the portable store protocol and locked filesystem store in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/storage.py` +- [X] T008 Add optional Unity Catalog Volume plus injected Delta-ledger coordination in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/databricks.py` + +## Phase 3: User Story 1 — Fill Missing Dates With Longitudinal Data (P1) + +**Goal**: Generate each causally missing date once and carry explicit athlete history. + +**Independent test**: One 90-day call and 90 serialized one-day calls have identical daily manifests and ending checkpoints. + +- [X] T009 [P] [US1] Add daily-vs-range, restart, deterministic-ID, week-boundary, pending-event, complaint, fatigue, and seasonal-phase tests in `../ams-sandbox-data-synthesis/tests/test_daily_engine.py` +- [X] T010 [US1] Implement stable athlete profiles, SHA-256 RNG streams, absolute phase, daily generation transitions, weekly context, and pending events in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/engine.py` +- [X] T011 [P] [US1] Add missing-date, no-op, target-before-head, backlog-bound, crash-recovery, and corrupt-chain tests in `../ams-sandbox-data-synthesis/tests/test_daily_orchestration.py` +- [X] T012 [US1] Implement chronological catch-up, validation, daily CSV rendering, and immutable commits in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/orchestration.py` +- [X] T013 [P] [US1] Add `init`, `prepare`, and `status` CLI contract tests including `wsv`, `knltb`, and `both` in `../ams-sandbox-data-synthesis/tests/test_daily_cli.py` +- [X] T014 [US1] Implement `init`, local-only `prepare`, and `status` commands in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/cli.py` +- [X] T015 [US1] Add a repository-root compatibility launcher in `../ams-sandbox-data-synthesis/daily_sandbox_pipeline.py` + +## Phase 4: User Story 2 — Run The Same Workflow Locally Or On A Schedule (P2) + +**Goal**: Use the same CLI/library contract against local storage and Databricks persistence, with scheduled runs unable to mutate AMS. + +**Independent test**: Identical initial inputs in local and Volume-like stores produce identical deterministic manifests; non-interactive mode makes zero mutation calls. + +- [X] T016 [P] [US2] Add local/Volume-store equivalence, secret-provider, and non-interactive zero-mutation tests in `../ams-sandbox-data-synthesis/tests/test_daily_databricks.py` +- [X] T017 [US2] Add a Databricks in-memory credential-provider adapter without `.env` persistence in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/credentials.py` +- [X] T018 [US2] Wire state-backend and credential-provider selection into `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/cli.py` +- [X] T019 [US2] Add a saved serverless wheel prepare job with one-run concurrency, queueing, Standard mode, and no publish task in `../ams-sandbox-data-synthesis/databricks.yml` + +## Phase 5: User Story 3 — Publish Only Safe Missing Daily Scopes (P3) + +**Goal**: Freeze and interactively execute only empty event-insert scopes; reconcile known matching scopes and block uncertainty/conflicts. + +**Independent test**: Offline connector doubles cover empty, matching, unexpected, read-error, non-TTY, changed-plan, partial, and unknown states with zero unsafe writes. + +- [X] T020 [P] [US3] Add remote-scope classification, exact-lineage reconciliation, request-count, and immutable-bundle tests in `../ams-sandbox-data-synthesis/tests/test_daily_planning.py` +- [X] T021 [US3] Implement date-scoped read-only classification, daily upload-child planning, contiguous ready-prefix selection, and bounded master plans in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/planning.py` +- [X] T022 [P] [US3] Add TTY gate, exact sandbox recheck, confirmation mismatch, single-use authorization, tampering, partial/unknown stop, and no-retry tests in `../ams-sandbox-data-synthesis/tests/test_daily_publishing.py` +- [X] T023 [US3] Implement interactive master publication and read-only reconciliation using the existing immutable upload execution safeguards in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/publishing.py` +- [X] T024 [US3] Add `prepare --remote-check`, `publish`, and `reconcile` command handling in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/cli.py` + +## Phase 6: User Story 4 — Evolve The Athlete Population Safely (P4) + +**Goal**: Preserve established identities while handling explicit additions/removals. + +**Independent test**: Registry reorder changes no profile or future output; additions/removals use effective dates and never rewrite committed history. + +- [X] T025 [P] [US4] Add reorder, add, remove, ambiguous-organization, and cross-organization regression tests in `../ams-sandbox-data-synthesis/tests/test_daily_roster.py` +- [X] T026 [US4] Implement explicit effective-dated roster reconciliation in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/roster.py` +- [X] T027 [US4] Add reviewed roster reconciliation to `prepare` without implicit historical backfill in `../ams-sandbox-data-synthesis/src/ams_sandbox_synthesis/cli.py` + +## Phase 7: Polish And Cross-Cutting Validation + +- [X] T028 Add local setup/test commands and separate WSV, KNLTB, and both examples—including the one-day-behind WSV test—to `../ams-sandbox-data-synthesis/README.md` +- [X] T029 Add local/Databricks storage, secrets, scheduling, retention, recovery, independent-project, and operator-responsibility guidance to `../ams-sandbox-data-synthesis/README.md` +- [X] T030 Run the synthesis offline suite, connector offline suite, CLI smoke tests, wheel build/import smoke test, JSON validation, and scoped `git diff --check`; record results in `agent-docs/current-work.md` + +## Dependencies + +- Setup (T001-T003) precedes all behavior changes. +- Foundational models/storage (T004-T008) precede every user story. +- User Story 1 (T009-T015) is the MVP and precedes remote planning. +- User Story 2 (T016-T019) depends on the User Story 1 CLI/store contract but can be completed before remote publication. +- User Story 3 (T020-T024) depends on committed daily runs from User Story 1. +- User Story 4 (T025-T027) depends on frozen profiles from User Story 1 but is independent of publication execution. +- Documentation and complete validation (T028-T030) follow all stories. + +## Parallel Opportunities + +- T004 and T006 touch separate test files and may run in parallel after setup. +- T009, T011, and T013 define different P1 contracts but implementation remains ordered T010, T012, T014. +- T016 may be written while the Databricks adapter is isolated from publication. +- T020 and T022 cover separate planning/execution safety surfaces. +- T025 is independent of User Story 3 after stable profiles exist. + +## Implementation Strategy + +1. Deliver the P1 local MVP first: explicit initialization, deterministic daily + transitions, durable catch-up, organization selection, and status. +2. Prove local and Databricks-compatible state equivalence while keeping every + scheduled path mutation-free. +3. Integrate existing read-only fetch and immutable upload safeguards; do not + create a parallel connector or confirmation mechanism. +4. Add controlled roster evolution and complete documentation/validation. From 9df5b194e4e7b46a472042de1b6e96cef9eac877 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 13 Aug 2026 13:59:50 +0200 Subject: [PATCH 7/7] added a first proposed skill file to be provided with the repo so vibe-coders can use it effectively too --- agent-docs/proposed-skill-api-use.md | 280 +++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 agent-docs/proposed-skill-api-use.md diff --git a/agent-docs/proposed-skill-api-use.md b/agent-docs/proposed-skill-api-use.md new file mode 100644 index 0000000..2359a28 --- /dev/null +++ b/agent-docs/proposed-skill-api-use.md @@ -0,0 +1,280 @@ +skill idea ams-api +--- +name: ams-connector-review +description: Review and improve changes to the AMS Python Connector. Use this skill whenever working on the ams-python-connector repository, reviewing pull requests, implementing new API endpoints, fixing bugs, improving robustness, adding tests, refactoring, or planning changes. Prioritize API correctness, backwards compatibility, safe handling of Smartabase data, thorough testing, small reviewable commits, and collaborative Git workflows over generating large amounts of code. +--- + +# AMS Connector Review + +The objective of this skill is to help maintain a robust, maintainable Python client for Teamworks AMS / Smartabase while encouraging good collaborative development practices. + +Assume this repository is a shared project where code quality is more important than writing code quickly. + +## Overall priorities + +Always optimize for the following order: + +1. Correctness +2. Safety +3. Testability +4. Simplicity +5. Readability +6. Performance + +Never sacrifice correctness for cleverness. + +--- + +# Development workflow + +Before making changes: + +- Read the existing implementation. +- Understand how the feature currently works. +- Avoid unnecessary rewrites. +- Preserve the public API unless a breaking change is explicitly requested. + +If requirements are unclear: + +Ask questions instead of making assumptions. + +--- + +# Code review checklist + +Review every proposed change for: + +## Correctness + +- Does the implementation satisfy the requested behavior? +- Are edge cases handled? +- Are failure cases considered? + +## Robustness + +Look specifically for: + +- missing timeout handling +- authentication failures +- malformed responses +- missing fields +- null values +- pagination +- rate limiting +- retries where appropriate +- duplicate requests +- idempotency + +Prefer graceful failure over unexpected exceptions. + +--- + +## Security + +Never expose: + +- passwords +- API keys +- tokens +- session cookies + +Ensure: + +- secrets come from environment variables +- logs do not contain credentials +- destructive operations require explicit confirmation +- sandbox protections remain intact unless intentionally changed + +Never recommend disabling security safeguards merely for convenience. + +--- + +## Backwards compatibility + +Public interfaces should remain stable whenever possible. + +Avoid changing: + +- function names +- parameter names +- return structures + +If a breaking change is unavoidable: + +- explain why +- document migration steps +- identify downstream impact + +--- + +# Testing + +Every bug fix should include a regression test whenever practical. + +Prefer automated tests over manual verification. + +When reviewing tests, check that they include: + +- success cases +- expected failures +- invalid input +- empty responses +- unexpected API responses +- authentication failures + +If a change cannot easily be tested automatically, explain why. + +--- + +# API design + +Encourage: + +- descriptive exceptions +- meaningful error messages +- small focused methods +- minimal side effects + +Avoid: + +- deeply nested logic +- duplicated code +- hidden state +- silent failures + +--- + +# Pull requests + +Prefer: + +- one feature per pull request +- one logical change per commit +- descriptive commit messages + +If a proposed change is too large: + +Suggest splitting it into multiple pull requests. + +--- + +# Review comments + +When reviewing code: + +Explain: + +- what is wrong +- why it matters +- how it could be improved + +Do not simply rewrite code unless necessary. + +Teach as well as review. + +--- + +# Collaboration + +Assume multiple developers are working simultaneously. + +Encourage: + +- small commits +- frequent pushes +- draft pull requests +- discussion before large refactors + +Avoid recommending force pushes unless absolutely necessary. + +--- + +# Refactoring + +Only recommend refactoring when it clearly improves: + +- readability +- maintainability +- robustness +- testability + +Avoid cosmetic rewrites that increase review effort. + +--- + +# Documentation + +Whenever behavior changes: + +Recommend updating: + +- README +- examples +- docstrings +- changelog if applicable + +Documentation should describe *why* something exists, not merely repeat the code. + +--- + +# Response format + +When reviewing code, structure the response as follows: + +## Summary + +One paragraph describing the overall quality. + +## Strengths + +Bullet list. + +## Issues + +Order by severity: + +- Critical +- High +- Medium +- Low + +For each issue include: + +- problem +- reason +- suggested improvement + +## Testing + +Describe: + +- existing coverage +- missing tests +- recommended new tests + +## Merge readiness + +End with one of: + +- Ready to merge +- Ready after minor fixes +- Needs additional work +- Do not merge yet + +Always explain the recommendation. + +--- + +# Philosophy + +The goal is not simply to make the code work. + +The goal is to leave the repository in a better state after every change. + +Prefer maintainable code over clever code. + +Prefer explicit behavior over hidden magic. + +Prefer automated tests over manual confidence. + +Prefer small improvements made consistently over large disruptive rewrites. \ No newline at end of file