diff --git a/apps/api/src/cora/api/capture_watch_preflight.py b/apps/api/src/cora/api/capture_watch_preflight.py index 2de77ab0190..29b5c6e31ff 100644 --- a/apps/api/src/cora/api/capture_watch_preflight.py +++ b/apps/api/src/cora/api/capture_watch_preflight.py @@ -112,6 +112,32 @@ `"Unknown"` literal, `empty` for a blank string, `text(len=N)` for a real value, BAD only as `non-text`. +## Camera-prefix cross-check + +2-BM has two cameras behind two different PV prefixes (`2bmSP1:`, +`2bmSP2:`), and `full_file_name`'s configured PV is a hardcoded string: +nothing makes it follow which camera the operator actually has +selected. On 2026-08-20 an operator's camera switch left that role +reading the idle camera's stale filename readback, which is exactly the +value `RunWitnessRecorder` vaults into `run_capture_path` (personal +data) as the Run's capture path. When a code declares BOTH +`full_file_name` and the optional `camera_selected` role (the live +camera-selection readback PV, e.g. 2-BM's +`2bm:MCTOptics:CameraSelected`), this adds ONE further report line per +code, `camera_prefix_check`, comparing the two: reused readings only, +never a second `control_port.read()` of either PV. See +`_camera_prefix_check`. + +CORA does not know, and must not guess, whether the substrate's +`CameraSelected` resolves to a bare index or an ENUM label, so the +resolved-reading -> expected-prefix mapping is a deployment-declared +table (`Settings.capture_camera_select_prefixes`), never hardcoded +here. Verdict is `match` (OK) when the live selection resolves to the +same prefix `full_file_name` is configured with; `mismatch(...)`, an +unrecognized reading, an empty mapping table, or an unreadable PV are +all BAD, never a silent pass: this check either confirms the two agree +or says plainly that it cannot. + Exit codes: 0 every configured PV connected and decoded clean; 2 anything disconnected, timed out, was access-denied, or a decoder rejected it. """ @@ -163,6 +189,14 @@ _PROGRESS_ROLES = (ROLE_IMAGES_SAVED, ROLE_IMAGES_COLLECTED) +ROLE_CAMERA_SELECTED = "camera_selected" +"""Optional `capture_watch_pvs` role, declared-and-unread by production +exactly like `server_running` (`ControlPortCaptureObserver` builds no +pump for either): the beamline's live camera-selection readback PV. +Read here only, to cross-check against the `full_file_name` role's +configured PV prefix; see `_camera_prefix_check` and this module's +"Camera-prefix cross-check" docstring section.""" + @dataclass class _PvReport: @@ -221,6 +255,7 @@ async def preflight_read_capture_pvs( status_phases: Mapping[str, str], baseline_pvs: Mapping[str, Mapping[str, str]] | None = None, experiment_identity_pvs: Mapping[str, Mapping[str, str]] | None = None, + camera_select_prefixes: Mapping[str, str] | None = None, ) -> _Report: """Read every configured `capture_watch_pvs` role, then every `capture_baseline_pvs` channel (slice 12), then every @@ -232,11 +267,30 @@ async def preflight_read_capture_pvs( against an unchanged config produce line-for-line identical output. Each PV is read independently: one dead or misconfigured PV does not abort the sweep, it reports as its own failed line. + + A code declaring BOTH `full_file_name` and `camera_selected` gets + one further line, `camera_prefix_check`, appended after that code's + own roles (see "Camera-prefix cross-check" in this module's + docstring): reused readings only, no extra `control_port.read()`. """ report = _Report() for code in sorted(capture_pvs): - for role, pv in sorted(capture_pvs[code].items()): - report.lines.append(await _read_one(control_port, code, role, pv, status_phases)) + roles = capture_pvs[code] + role_reports: dict[str, _PvReport] = {} + for role, pv in sorted(roles.items()): + pv_report = await _read_one(control_port, code, role, pv, status_phases) + report.lines.append(pv_report) + role_reports[role] = pv_report + if ROLE_FULL_FILE_NAME in roles and ROLE_CAMERA_SELECTED in roles: + report.lines.append( + _camera_prefix_check( + code=code, + full_file_name_pv=roles[ROLE_FULL_FILE_NAME], + camera_selected_pv=roles[ROLE_CAMERA_SELECTED], + camera_selected_report=role_reports[ROLE_CAMERA_SELECTED], + camera_select_prefixes=camera_select_prefixes or {}, + ) + ) for code in sorted(baseline_pvs or {}): for channel_name, pv in sorted((baseline_pvs or {})[code].items()): report.lines.append(await _read_one_baseline(control_port, code, channel_name, pv)) @@ -368,6 +422,105 @@ def _full_file_name_verdict(value: object) -> tuple[str, bool]: return f"text(len={len(value)})", True +def _configured_pv_prefix(pv: str) -> str: + """The IOC-prefix segment of a configured PV name, up to and + including the first colon (`"2bmSP2:HDF1:FullFileName_RBV"` -> + `"2bmSP2:"`). Plain string parsing, not a lookup against any + facility-specific vocabulary: this reads the STATIC config string, + never a live value. + """ + head, sep, _ = pv.partition(":") + return f"{head}{sep}" + + +def _camera_prefix_check( + *, + code: str, + full_file_name_pv: str, + camera_selected_pv: str, + camera_selected_report: _PvReport, + camera_select_prefixes: Mapping[str, str], +) -> _PvReport: + """The camera-prefix cross-check (see this module's docstring): + compares `full_file_name`'s CONFIGURED PV prefix against the live + `camera_selected` reading, resolved through the deployment-declared + `camera_select_prefixes` table. + + Never reports a clean match on a reading it cannot actually + confirm: an unreadable `camera_selected` PV, a reading the table + does not resolve, or an empty table are each their own distinct BAD + verdict, not a fallback pass. Reuses `camera_selected_report` + (already read by the caller's own `capture_watch_pvs` sweep); makes + no second `control_port.read()` of its own. + """ + configured_prefix = _configured_pv_prefix(full_file_name_pv) + if not camera_selected_report.connected: + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=False, + connected=False, + detail=f"camera_selected PV unreadable: {camera_selected_report.detail}", + ) + if camera_selected_report.value is None: + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=False, + connected=True, + kind="PrefixCheck", + value=configured_prefix, + verdict="unrecognized-reading(camera_selected PV did not decode)", + ) + if not camera_select_prefixes: + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=False, + connected=True, + kind="PrefixCheck", + value=configured_prefix, + verdict="not-configured(CAPTURE_CAMERA_SELECT_PREFIXES empty)", + ) + resolved = str(camera_selected_report.value) + expected_prefix = camera_select_prefixes.get(resolved) + if expected_prefix is None: + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=False, + connected=True, + kind="PrefixCheck", + value=configured_prefix, + verdict=f"unrecognized-reading({resolved!r})", + ) + if expected_prefix != configured_prefix: + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=False, + connected=True, + kind="PrefixCheck", + value=configured_prefix, + verdict=f"mismatch(selected camera expects {expected_prefix!r})", + ) + return _PvReport( + code=code, + pv_key="camera_prefix_check", + pv=camera_selected_pv, + ok=True, + connected=True, + kind="PrefixCheck", + value=configured_prefix, + verdict="match", + ) + + async def _read_one_baseline( control_port: ControlPort, code: str, @@ -594,6 +747,7 @@ async def _run() -> int: status_phases=settings.capture_status_phases, baseline_pvs=settings.capture_baseline_pvs, experiment_identity_pvs=settings.capture_experiment_identity_pvs, + camera_select_prefixes=settings.capture_camera_select_prefixes, ) return _finish(report) finally: diff --git a/apps/api/src/cora/infrastructure/config.py b/apps/api/src/cora/infrastructure/config.py index e9c43895cee..aaf8aea08c8 100644 --- a/apps/api/src/cora/infrastructure/config.py +++ b/apps/api/src/cora/infrastructure/config.py @@ -812,8 +812,50 @@ class Settings(BaseSettings): # never logged in full and never lands on an event; it goes to the # `run_capture_path` PII vault via `RunWitnessRecorder`'s dual-clock # guard. See `_run_witness.py`'s "Capture path pairing" section. + # + # `camera_selected` (optional per code) is a further role, + # declared-and-unread by production exactly like `server_running` + # (`ControlPortCaptureObserver` builds no pump for either): it names + # the beamline's live camera-selection readback PV (2-BM: + # `2bm:MCTOptics:CameraSelected`), read only by + # `capture_watch_preflight`'s camera-prefix cross-check. That check + # exists because `full_file_name`'s PV above is a hardcoded string + # (2-BM's `2bmSP1:` / `2bmSP2:` are two separate cameras) with + # nothing making it follow which camera is actually selected: an + # operator's camera switch (as happened 2026-08-20) leaves it + # reading the idle camera's stale value, which then reaches the + # `run_capture_path` PII vault above unless caught first. See + # `capture_camera_select_prefixes` below and + # `capture_watch_preflight._camera_prefix_check`. capture_watch_pvs: dict[str, dict[str, str]] = {} + # Deployment-declared table the `camera_selected` role's decoded + # reading is looked up in, to resolve the `full_file_name` PV prefix + # it should correspond to (the camera-prefix cross-check above). + # CORA does not know, and must not guess, whether the substrate's + # `CameraSelected` resolves to a bare index or an EPICS ENUM label: + # that vocabulary belongs to one facility's IOC, exactly like + # `capture_status_phases` below, so it is declared here rather than + # hardcoded in the spine. Empty (default) means the cross-check + # reports "not configured" rather than silently passing. Read from + # CAPTURE_CAMERA_SELECT_PREFIXES as JSON: + # + # CAPTURE_CAMERA_SELECT_PREFIXES='{ + # "0": "2bmSP1:", + # "1": "2bmSP2:" + # }' + # + # The example above encodes the ONE fact confirmed in + # `deployments/2-bm/beamline.yaml` (operator-verified 2026-06-19, + # DET-11): camera 0 is the 5 MP `2bmSP1:` unit, camera 1 is the + # 31 MP `2bmSP2:` unit. Whether `CameraSelected` actually reads back + # as the bare literal `"0"` / `"1"` (rather than some other ENUM + # label) is NOT confirmed against the live IOC; deploying this table + # with the wrong keys would only ever produce "unrecognized-reading" + # verdicts, never a false match, so it is safe to try and correct + # once staff confirm the real readback shape. + capture_camera_select_prefixes: dict[str, str] = {} + # Genesis-baseline PVs (slice 12): a deployment-declared set read # ONCE, at the instant a capture promotes to a witnessed Run, and # written as `Observation` rows with `sampling_procedure="baseline"`. diff --git a/apps/api/tests/unit/api/test_capture_watch_preflight.py b/apps/api/tests/unit/api/test_capture_watch_preflight.py index 66d2cf17c84..ab878b71f05 100644 --- a/apps/api/tests/unit/api/test_capture_watch_preflight.py +++ b/apps/api/tests/unit/api/test_capture_watch_preflight.py @@ -69,6 +69,7 @@ async def _preflight( status_phases: dict[str, str] | None = None, baseline_pvs: dict[str, dict[str, str]] | None = None, experiment_identity_pvs: dict[str, dict[str, str]] | None = None, + camera_select_prefixes: dict[str, str] | None = None, ) -> _Report: """`_FakeControlPort` implements `.read()` only (this command never writes or subscribes), so it satisfies `ControlPort` in practice but @@ -80,6 +81,7 @@ async def _preflight( status_phases=status_phases if status_phases is not None else _PHASES, baseline_pvs=baseline_pvs, experiment_identity_pvs=experiment_identity_pvs, + camera_select_prefixes=camera_select_prefixes, ) @@ -287,6 +289,161 @@ async def test_preflight_read_full_file_name_role_non_text_is_bad() -> None: assert line.value == "" +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_matching_camera_is_ok() -> None: + """The 2026-08-20 incident's happy path: `full_file_name` is + configured for `2bmSP2:`, and the live `camera_selected` reading + resolves (via the deployment's own table) to that same prefix.""" + port = _FakeControlPort( + { + "2bmSP2:HDF1:FullFileName_RBV": _reading("/local1/2BM/2026-08-exp/scan_0001.h5"), + "2bm:MCTOptics:CameraSelected": _reading("1", kind="Categorical"), + } + ) + + report = await _preflight( + port, + { + "code": { + "full_file_name": "2bmSP2:HDF1:FullFileName_RBV", + "camera_selected": "2bm:MCTOptics:CameraSelected", + } + }, + camera_select_prefixes={"0": "2bmSP1:", "1": "2bmSP2:"}, + ) + + by_key = {line.pv_key: line for line in report.lines} + check = by_key["camera_prefix_check"] + assert check.ok + assert check.verdict == "match" + + +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_wrong_camera_selected_is_a_mismatch() -> None: + """The actual 2026-08-20 shape: the operator switched to camera 0, + but `full_file_name` is still hardcoded to camera 1's PV prefix.""" + port = _FakeControlPort( + { + "2bmSP2:HDF1:FullFileName_RBV": _reading("/local1/2BM/2026-08-exp/scan_0001.h5"), + "2bm:MCTOptics:CameraSelected": _reading("0", kind="Categorical"), + } + ) + + report = await _preflight( + port, + { + "code": { + "full_file_name": "2bmSP2:HDF1:FullFileName_RBV", + "camera_selected": "2bm:MCTOptics:CameraSelected", + } + }, + camera_select_prefixes={"0": "2bmSP1:", "1": "2bmSP2:"}, + ) + + by_key = {line.pv_key: line for line in report.lines} + check = by_key["camera_prefix_check"] + assert not check.ok + assert check.verdict == "mismatch(selected camera expects '2bmSP1:')" + + +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_unreadable_camera_pv_is_bad_not_skipped() -> None: + port = _FakeControlPort( + { + "2bmSP2:HDF1:FullFileName_RBV": _reading("/local1/2BM/2026-08-exp/scan_0001.h5"), + "2bm:MCTOptics:CameraSelected": ControlNotConnectedError( + "2bm:MCTOptics:CameraSelected" + ), + } + ) + + report = await _preflight( + port, + { + "code": { + "full_file_name": "2bmSP2:HDF1:FullFileName_RBV", + "camera_selected": "2bm:MCTOptics:CameraSelected", + } + }, + camera_select_prefixes={"0": "2bmSP1:", "1": "2bmSP2:"}, + ) + + by_key = {line.pv_key: line for line in report.lines} + check = by_key["camera_prefix_check"] + assert not check.ok + assert not check.connected + + +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_empty_prefix_table_reports_not_configured() -> ( + None +): + """No `CAPTURE_CAMERA_SELECT_PREFIXES` declared yet: must report + that the check cannot confirm anything, never a silent pass.""" + port = _FakeControlPort( + { + "2bmSP2:HDF1:FullFileName_RBV": _reading("/local1/2BM/2026-08-exp/scan_0001.h5"), + "2bm:MCTOptics:CameraSelected": _reading("1", kind="Categorical"), + } + ) + + report = await _preflight( + port, + { + "code": { + "full_file_name": "2bmSP2:HDF1:FullFileName_RBV", + "camera_selected": "2bm:MCTOptics:CameraSelected", + } + }, + ) + + by_key = {line.pv_key: line for line in report.lines} + check = by_key["camera_prefix_check"] + assert not check.ok + assert check.verdict == "not-configured(CAPTURE_CAMERA_SELECT_PREFIXES empty)" + + +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_unrecognized_reading_is_bad() -> None: + """A `camera_selected` reading absent from the deployment's own + table: the mapping is deployment-declared vocabulary, so an + unrecognized reading is reported plainly, never guessed at.""" + port = _FakeControlPort( + { + "2bmSP2:HDF1:FullFileName_RBV": _reading("/local1/2BM/2026-08-exp/scan_0001.h5"), + "2bm:MCTOptics:CameraSelected": _reading("Camera 1", kind="Categorical"), + } + ) + + report = await _preflight( + port, + { + "code": { + "full_file_name": "2bmSP2:HDF1:FullFileName_RBV", + "camera_selected": "2bm:MCTOptics:CameraSelected", + } + }, + camera_select_prefixes={"0": "2bmSP1:", "1": "2bmSP2:"}, + ) + + by_key = {line.pv_key: line for line in report.lines} + check = by_key["camera_prefix_check"] + assert not check.ok + assert check.verdict == "unrecognized-reading('Camera 1')" + + +@pytest.mark.unit +async def test_preflight_read_camera_prefix_check_skipped_without_camera_selected_role() -> None: + """A code with `full_file_name` alone (no `camera_selected` role + declared) gets no cross-check line: this is the pre-existing + behavior for every deployment that has not yet opted in.""" + port = _FakeControlPort({"pv:file": _reading("2bmSP2:HDF1:FullFileName_RBV")}) + + report = await _preflight(port, {"code": {"full_file_name": "pv:file"}}) + + assert "camera_prefix_check" not in {line.pv_key for line in report.lines} + + @pytest.mark.unit async def test_preflight_read_a_mis_keyed_role_with_a_path_value_still_redacts() -> None: """Defense-in-depth: a `capture_watch_pvs` role-key typo diff --git a/docs/deployments/2-bm/questions.md b/docs/deployments/2-bm/questions.md index 2d114b81d62..9bf0bab9c3e 100644 --- a/docs/deployments/2-bm/questions.md +++ b/docs/deployments/2-bm/questions.md @@ -130,7 +130,7 @@ gives a reader no way to tell which writer produced its own timestamp. | ID | Priority | Question | CORA assumes | Already done? | Resolves | | --- | --- | --- | --- | --- | --- | -| DATA-8 | `Nice-to-have` | How often do scans finish with dropped frames? `add_theta()` compares written frames against commanded angles and logs a warning when they disagree, so the condition is detected but not fatal. Knowing whether this is rare-and-alarming or routine decides whether a record of the scan should refuse to be written, or carry the shortfall as an ordinary recorded fact. | rare enough to treat as an exception worth surfacing, not a routine outcome to normalise | not yet, but the question is now askable: CORA could not read the commanded counts at all until 2026-08-12 (2-BM writes them as one-element arrays and the reader understood only plain scalars), so every shortfall check silently compared against nothing. Two data points since: `test_000.h5`, an early smoke test, 3601 commanded and 1 captured with `theta` absent; and `test_005.h5`, the first production scan CORA read end to end, 1501 commanded and 1501 captured, nothing dropped | [Operations](operations.md) | +| DATA-8 | `Nice-to-have` | How often do scans finish with dropped frames? `add_theta()` compares written frames against commanded angles and logs a warning when they disagree, so the condition is detected but not fatal. Knowing whether this is rare-and-alarming or routine decides whether a record of the scan should refuse to be written, or carry the shortfall as an ordinary recorded fact. | rare enough to treat as an exception worth surfacing, not a routine outcome to normalise | not yet, but the question is now askable: CORA could not read the commanded counts at all until 2026-08-12 (2-BM writes them as one-element arrays and the reader understood only plain scalars), so every shortfall check silently compared against nothing. Two data points since: `test_000.h5`, an early smoke test, 3601 commanded and 1 captured with `theta` absent; and `test_005.h5`, the first production scan CORA read end to end, 1501 commanded and 1501 captured, nothing dropped. A third, 2026-08-20: a helical scan of 1501 projections where the saved-frame counter fell behind the collected-frame counter as the disk filled, the write failed, and the file was left unreadable. Opening it still succeeded; reading the frame-index dataset raised an HDF5 address-overflow error with end-of-allocation at 2048 bytes, so nothing past the file header had been flushed. `add_theta()`'s comparison assumes a file complete enough to read a count out of, which this one was not, so a shortfall check has to treat an unreadable file as its own outcome and not only a readable one with a smaller-than-commanded count | [Operations](operations.md) | | DATA-12 | `Blocks-go-live` | From which date are 2-BM scan files written with the corrected `start_date`, and has the `2bmbSP2` IOC been restarted yet? A file gives no way to answer this from its own contents: the client fix overwrites the IOC's stale value instead of preserving it, so a pre-fix and a post-fix file are identical in shape. CORA now reads `start_date`, which is right for new files and silently wrong for old ones, and the ingest policy is that a parseable file timestamp beats an operator's, so a wrong value cannot be corrected after the fact. A date is a complete answer. A marker written into the file, even a one-line `start_date_writer` attribute, would retire the question permanently and would serve every other consumer of these files too. | the client fix (`decarlof/tomoscan@d0025a2`) went live 2026-08-13; the IOC restart that stops the stale write at file open has not happened yet | not yet (the descriptor declares `start_date`; no 2-BM file written after the fix has been read by CORA) | [Operations](operations.md#inside-the-scan-file) | ## Where CORA runs