From e71ce12cf8a66d15c219481859c2bfbfbc17d816 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 27 Jul 2026 23:40:47 -0400 Subject: [PATCH 1/8] fix(benchmark): check the MockMed encounter type as its own field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `verify_encounter_saved` is the arm-independent success criterion for the compiled-vs-agent MockMed benchmark. It matched the saved encounter row as one fuzzy candidate string, ` — `, at min_ratio 0.8. A single similarity ratio over that concatenation is decided by whichever field contributes more characters, so a long correct note outvotes a short wrong type. Measured on a retained MockMed frame (E4 precondition run, episode 4 of 20): the agent saved the correct note under encounter type `Consult` instead of the requested `Triage`, and the check passed it. expected : 'Triage — Follow-up in 2 weeks; BP recheck.' actual : 'Consult — Follow-up in 2 weeks; BP recheck.' ratio : 0.8471 >= 0.8 -> MATCHED A second, independent path made the type unverifiable at all: the row's fallback candidate was the bare `note_text[:40]`, which the banner's own note line satisfies. On that same frame it scored 0.918 while the banner evidence the check *needs* scored only 0.875, so no threshold separates the two: any min_ratio that rejects the wrong-type row also rejects every genuine success. Measured over 61 retained trajectories, sweeping min_ratio leaves both false positives standing up to 0.86 and collapses all 27 true positives to false negatives at 0.88. Raising the threshold is not a fix. The fix is structural. Each field is judged on its own terms: the row line is parsed into its type field and its note field, the type field is CATEGORICAL and must name exactly one member of the application's enumeration (`ENCOUNTER_TYPES`) and that member must be the requested one, and the note field keeps the unchanged min_ratio fuzzy tolerance. A row carrying this run's note under a different known type now sets a new `wrong_type_row` field and fails the run, matching what `verify_hybrid_final` already reports for the hybrid arms. Validation over all 61 retained E4 trajectories (40 carry an independent in-page record-oracle label; 21 are labelled from the rendered encounter list), both directions: before TP=27 FP=2 TN=32 FN=0 after TP=27 FP=0 TN=34 FN=0 Only the two known-bad episodes change verdict; every known-good episode still passes. On the 27 true positives the type field scores 1.0000 against a 0.8 floor and the note field 0.9000, so the fix is not near a false negative. Free-text tolerance is unchanged: every true positive already satisfied the ` — ` row candidate at 0.8974, so removing the type-free fallback costs no genuine success on this corpus. Corrected figures for the E4 precondition run, denominator 20 episodes per model per oracle: 7B screen oracle 14/20 -> 13/20, which now agrees with the in-page record oracle's 13/20; 3B unchanged at 0/20 on both. Tests exercise both directions at four note lengths (11/22/33/60 chars), covering the band where the old ratio crossed the threshold (0.7317, 0.8254, 0.8706, 0.9209 against the expected Triage row form). --- openadapt_flow/benchmark/verify.py | 164 ++++++++++++++++++++++++----- openadapt_flow/vision/__init__.py | 5 +- tests/test_benchmark.py | 120 +++++++++++++++++++-- 3 files changed, 253 insertions(+), 36 deletions(-) diff --git a/openadapt_flow/benchmark/verify.py b/openadapt_flow/benchmark/verify.py index d12a0df0..c549a7db 100644 --- a/openadapt_flow/benchmark/verify.py +++ b/openadapt_flow/benchmark/verify.py @@ -9,7 +9,8 @@ - :func:`verify_encounter_saved` (MockMed): OCR must find (a) the ``Encounter saved — `` banner and (b) the saved encounter row - (`` — ``). MockMed renders both only on the patient screen + (`` — ``), whose categorical type field and free-text note + are checked separately. MockMed renders both only on the patient screen after a successful save, so the check passes only in the navigated-back-to-patient state a user would accept as "done". - :func:`verify_note_saved` (OpenEMR): OCR of the final screen must show @@ -22,27 +23,108 @@ from pydantic import BaseModel -from openadapt_flow.vision import find_text, ocr, upscale_png +from openadapt_flow.vision import find_text, normalize_text, ocr, upscale_png BANNER_PREFIX = "Encounter saved" #: MockMed truncation limits (static/app.js): banner shows note[:40], the #: encounters list row shows note[:60]. BANNER_NOTE_CHARS = 40 ROW_NOTE_CHARS = 60 +#: Every encounter type MockMed can store (the segmented control in +#: ``mockmed/static/app.js``). The type field is CATEGORICAL, so it is +#: decided by which member of this enumeration the row names — never by a +#: similarity score against one expected string. +ENCOUNTER_TYPES = ("Triage", "Consult") +#: Dash-like characters rapidocr emits for the row's em-dash separator +#: (`` — ``). Includes the CJK forms it substitutes when the +#: note is short and the glyph is read in isolation. +_ROW_SEPARATORS = "-‐‑‒–—―−一─－" +#: Leading list decoration rapidocr reads from the ``
  • `` bullet. +_ROW_DECORATION = " \t.·•‣●▪*" class VerifyResult(BaseModel): """Outcome of the shared success check. Attributes: - success: True iff both the banner and the encounter row were found. + success: True iff the banner and a correctly-typed encounter row + were found and no wrong-type row carries this run's note. banner_found: The ``Encounter saved — `` banner was located. - note_found: The `` — `` encounter row was located. + note_found: An encounter row was located whose type field is the + requested type AND whose note field is the requested note. + wrong_type_row: Some encounter row carries this run's note under a + DIFFERENT known encounter type — a silent wrong-target write. """ success: bool banner_found: bool note_found: bool + wrong_type_row: bool = False + + +def _split_row_line(line: str, known_types: tuple[str, ...]) -> list[tuple[str, str]]: + """Candidate ``(type_field, note_field)`` splits of one OCR line. + + The encounters list renders ``
  • —
    • ``, so the + two fields are separated by an em dash. OCR is not reliable about that + glyph (it drops it, or reads it as a hyphen or a CJK character), and + the note itself contains hyphens, so the split is attempted two ways + and every candidate is scored: + + 1. at the first dash-like character, and + 2. positionally, after exactly ``len(t)`` characters, for each known + type ``t`` — which still parses a row whose separator OCR dropped. + + A candidate that does not name a known type simply scores badly and is + discarded by the caller; producing extra candidates cannot weaken the + check because both fields must pass on the SAME candidate. + """ + text = normalize_text(line).lstrip(_ROW_DECORATION) + candidates: list[tuple[str, str]] = [] + for index, char in enumerate(text): + if char in _ROW_SEPARATORS: + candidates.append((text[:index], text[index + 1 :])) + break + for known in known_types: + head, tail = text[: len(known)], text[len(known) :] + candidates.append((head, tail.lstrip(_ROW_DECORATION + _ROW_SEPARATORS))) + return [ + (head.strip(_ROW_DECORATION + _ROW_SEPARATORS), tail.strip(_ROW_DECORATION)) + for head, tail in candidates + ] + + +def _row_type( + line: str, + note_text: str, + known_types: tuple[str, ...], + min_ratio: float, +) -> str | None: + """Return the encounter type of the row this OCR line renders, if any. + + The line is parsed into its two fields and each is judged on its own + terms: the categorical type field must name exactly one member of + ``known_types`` (highest similarity, strictly ahead of every other + member, above ``min_ratio``), and the free-text note field must match + the requested note at ``min_ratio``. A line that is not a row, or + whose note is absent, yields ``None``. + """ + expected_note = normalize_text(note_text[:ROW_NOTE_CHARS]) + for type_field, note_field in _split_row_line(line, known_types): + scores = { + known: difflib.SequenceMatcher( + None, type_field, normalize_text(known) + ).ratio() + for known in known_types + } + best = max(scores, key=lambda known: scores[known]) + runner_up = max((v for k, v in scores.items() if k != best), default=0.0) + if scores[best] < min_ratio or scores[best] <= runner_up: + continue + note_ratio = difflib.SequenceMatcher(None, note_field, expected_note).ratio() + if note_ratio >= min_ratio: + return best + return None def verify_encounter_saved( @@ -50,50 +132,78 @@ def verify_encounter_saved( note_text: str, *, encounter_type: str = "Triage", + known_types: tuple[str, ...] = ENCOUNTER_TYPES, min_ratio: float = 0.8, ) -> VerifyResult: """Check a final-state screenshot for the encounter-saved evidence. - ``find_text`` fuzzy-matches whole OCR lines, and the OCR engine may - segment the banner as one line (``Encounter saved — ``) or as two - (the prefix and the note separately). Each check therefore accepts any - of a small set of candidate line forms; all candidates describe the same - on-screen evidence, so this tolerates OCR line segmentation without - weakening the criterion: the banner prefix exists only after a save, - and both checks must pass. + Two independent pieces of evidence are required: the save banner, and + an encounter row whose fields are the requested ones. + + **The banner** is a save-happened signal with no parameters of its + own. ``find_text`` fuzzy-matches whole OCR lines and the engine may + segment the banner as one line (``Encounter saved — ``) or as + two (the prefix and the note separately), so either form is accepted. + + **The row** is `` — ``, one categorical field and one + free-text field. It is deliberately NOT matched as one fuzzy candidate + string. A single similarity ratio over the concatenation is decided by + whichever field contributes more characters, so a long correct note + outvotes a short wrong type: measured on a real MockMed frame, a + ``Consult`` row carrying the requested 32-character note scored 0.8471 + against the ``Triage`` row form, above this function's 0.8 threshold. + No threshold repairs that — on the same frame the note text alone + scored 0.918 while the banner evidence the check *needs* scored only + 0.875, so any threshold that rejects the wrong-type row also rejects + every genuine success. + + Each field is therefore judged on its own terms. The type field is + categorical: it must name exactly one member of ``known_types``, and + that member must be ``encounter_type``. The note field is free text + and keeps the unchanged ``min_ratio`` fuzzy tolerance. A row that + carries the requested note under a different known type sets + ``wrong_type_row`` and fails the run — a wrong-target write is not a + success, and reporting it is required by the reliability standard. Args: screen_png: Full-frame screenshot of the final state as PNG bytes. note_text: The note the run was asked to enter. encounter_type: Encounter type the run was asked to create. - min_ratio: Fuzzy-match threshold forwarded to ``find_text``. + known_types: Every encounter type the application can store. The + categorical check is a choice among these, so an incomplete + enumeration would let an unlisted type go unrecognized. + min_ratio: Fuzzy-match threshold for the banner, the note field, + and the type field. Returns: - A :class:`VerifyResult`; ``success`` requires both checks to pass. + A :class:`VerifyResult`; ``success`` requires the banner, a + correctly-typed row, and no wrong-type row. """ + if encounter_type not in known_types: + known_types = (*known_types, encounter_type) - def any_found(candidates: tuple[str, ...]) -> bool: - return any( - find_text(screen_png, c, min_ratio=min_ratio) is not None - for c in candidates - ) - - banner_found = any_found( - ( + banner_found = any( + find_text(screen_png, candidate, min_ratio=min_ratio) is not None + for candidate in ( f"{BANNER_PREFIX} — {note_text[:BANNER_NOTE_CHARS]}", f"{BANNER_PREFIX} —", ) ) - note_found = any_found( - ( - f"{encounter_type} — {note_text[:ROW_NOTE_CHARS]}", - note_text[:BANNER_NOTE_CHARS], + row_types = { + found + for found in ( + _row_type(line.text, note_text, known_types, min_ratio) + for line in ocr(screen_png) ) - ) + if found is not None + } + note_found = encounter_type in row_types + wrong_type_row = bool(row_types - {encounter_type}) return VerifyResult( - success=banner_found and note_found, + success=banner_found and note_found and not wrong_type_row, banner_found=banner_found, note_found=note_found, + wrong_type_row=wrong_type_row, ) diff --git a/openadapt_flow/vision/__init__.py b/openadapt_flow/vision/__init__.py index aecc60db..facb66f0 100644 --- a/openadapt_flow/vision/__init__.py +++ b/openadapt_flow/vision/__init__.py @@ -4,7 +4,8 @@ - :class:`Match`, :func:`find_template`, :func:`find_structural_template` - :class:`OcrLine`, :func:`ocr`, :func:`find_text`, - :func:`find_text_candidates`, :func:`text_present`, :func:`upscale_png` + :func:`find_text_candidates`, :func:`normalize_text`, + :func:`text_present`, :func:`upscale_png` - :func:`phash_png`, :func:`phash_distance` - :func:`pixels_changed` - :func:`wait_settled`, :func:`wait_settled_result`, :class:`SettleResult` @@ -24,6 +25,7 @@ OcrResolutionRefused, find_text, find_text_candidates, + normalize_text, ocr, text_present, upscale_png, @@ -45,6 +47,7 @@ "find_template", "find_text", "find_text_candidates", + "normalize_text", "ocr", "phash_distance", "phash_png", diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index a2141276..1a5ab532 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -392,25 +392,36 @@ def test_task_prompt_states_intent_not_coordinates(self) -> None: # -- verify ------------------------------------------------------------------- -def put_line(img: np.ndarray, text: str, y: int) -> None: +def put_line(img: np.ndarray, text: str, y: int, scale: float = 0.7) -> None: cv2.putText( img, text, (40, y), cv2.FONT_HERSHEY_SIMPLEX, - 0.7, + scale, (0, 0, 0), 2, cv2.LINE_AA, ) +def make_encounter_screen(*lines: str, scale: float = 0.7) -> bytes: + """A synthetic MockMed final frame carrying ``lines``. + + ``scale`` shrinks the Hershey render so a long line still fits the + 1280px frame; at the 0.7 default rapidocr drops a ~60-character row + entirely, which is a property of this synthetic renderer and not of + the browser-rendered frames the verifier faces in production. + """ + img = np.full((800, 1280, 3), 245, dtype=np.uint8) + for i, line in enumerate(lines): + put_line(img, line, 200 + i * 60, scale) + return to_png(img) + + class TestVerify: def make_screen(self, *lines: str) -> bytes: - img = np.full((800, 1280, 3), 245, dtype=np.uint8) - for i, line in enumerate(lines): - put_line(img, line, 200 + i * 60) - return to_png(img) + return make_encounter_screen(*lines) def test_saved_state_passes(self) -> None: # cv2 Hershey fonts have no em dash; the fuzzy ratio absorbs the @@ -437,10 +448,12 @@ def test_split_banner_lines_pass(self) -> None: def test_note_in_form_without_banner_fails(self) -> None: # The typed note is visible on the encounter form BEFORE saving; - # without the banner that must not count as success. + # without the banner that must not count as success. The form also + # carries no encounter ROW (only the note in a textarea), so the + # row check fails independently of the banner check. screen = self.make_screen("Note", NOTE[:40], "Save Encounter") verdict = verify_encounter_saved(screen, NOTE) - assert verdict.note_found + assert not verdict.note_found assert not verdict.banner_found assert not verdict.success @@ -466,6 +479,97 @@ def test_wrong_note_fails(self) -> None: assert not verdict.success +# -- verify: the encounter TYPE is categorical and must not be outvoted -------- + +# Notes of increasing length. The defect this guards against is +# length-dependent: one fuzzy ratio over the whole `` — `` row +# is decided by whichever field contributes more characters, so a long +# correct note hides a short wrong type. Similarity of a ``Consult`` row +# against the expected ``Triage`` row form, by note length, against the +# 0.8 threshold: 11 chars -> 0.7317 (caught), 22 -> 0.8254 (MISSED), +# 33 -> 0.8706 (MISSED), 60 -> 0.9209 (MISSED). The benchmark's own note +# is 32 characters, in the missed band. +NOTE_LENGTHS = [ + "BP recheck.", + "BP recheck; see chart.", + "Follow-up in 2 weeks; BP recheck.", + "Follow-up in 2 weeks; BP recheck and repeat basic labs then.", +] + + +class TestVerifyEncounterType: + """The categorical type field must decide on its own terms.""" + + def make_screen(self, *lines: str) -> bytes: + # 0.6 rather than the 0.7 default: the longest note here renders + # past what rapidocr reads back at 0.7 in this synthetic frame. + return make_encounter_screen(*lines, scale=0.6) + + @pytest.mark.parametrize("note", NOTE_LENGTHS) + def test_right_type_passes_at_every_note_length(self, note: str) -> None: + # The over-halt guard: a genuinely correct save must pass at every + # note length. A check strict enough to trip on OCR noise would + # make a working system look broken, which is worse than the + # over-count it replaces. + screen = self.make_screen( + f"Encounter saved - {note[:40]}", + f"Triage - {note[:60]}", + ) + verdict = verify_encounter_saved(screen, note) + assert verdict.banner_found + assert verdict.note_found + assert not verdict.wrong_type_row + assert verdict.success + + @pytest.mark.parametrize("note", NOTE_LENGTHS) + def test_wrong_type_fails_at_every_note_length(self, note: str) -> None: + # The regression: the note is correct and the banner is present, + # but the encounter was saved as Consult. That is a wrong-target + # write, not a success, however long the note is. + screen = self.make_screen( + f"Encounter saved - {note[:40]}", + f"Consult - {note[:60]}", + ) + verdict = verify_encounter_saved(screen, note) + assert verdict.banner_found + assert not verdict.note_found + assert verdict.wrong_type_row + assert not verdict.success + + def test_wrong_type_row_beside_right_row_fails(self) -> None: + # A correct row does not excuse a second row carrying the same + # note under a different type: the run mutated the wrong target. + screen = self.make_screen( + f"Encounter saved - {NOTE[:40]}", + f"Triage - {NOTE[:60]}", + f"Consult - {NOTE[:60]}", + ) + verdict = verify_encounter_saved(screen, NOTE) + assert verdict.note_found + assert verdict.wrong_type_row + assert not verdict.success + + def test_bare_note_line_is_not_an_encounter_row(self) -> None: + # The banner's note line carries no type at all. It must never + # stand in for the encounter row -- accepting it was the second + # path by which a wrong type went unchecked. + screen = self.make_screen("Encounter saved -", NOTE[:40]) + verdict = verify_encounter_saved(screen, NOTE) + assert verdict.banner_found + assert not verdict.note_found + assert not verdict.success + + def test_unlisted_requested_type_is_still_checked(self) -> None: + # A caller may ask for a type outside the default enumeration; it + # is added to the choice set rather than silently unmatchable. + screen = self.make_screen( + f"Encounter saved - {NOTE[:40]}", + f"Referral - {NOTE[:60]}", + ) + assert verify_encounter_saved(screen, NOTE, encounter_type="Referral").success + assert not verify_encounter_saved(screen, NOTE).success + + # -- orchestrator aggregation -------------------------------------------------- From 304024c40e97219f29327464a7691c6d74c9198e Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 01:24:33 -0400 Subject: [PATCH 2/8] test(benchmark): isolate encounter type parser --- tests/test_benchmark.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 1a5ab532..21288c73 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -506,7 +506,9 @@ def make_screen(self, *lines: str) -> bytes: return make_encounter_screen(*lines, scale=0.6) @pytest.mark.parametrize("note", NOTE_LENGTHS) - def test_right_type_passes_at_every_note_length(self, note: str) -> None: + def test_right_type_passes_at_every_note_length( + self, note: str, monkeypatch: pytest.MonkeyPatch + ) -> None: # The over-halt guard: a genuinely correct save must pass at every # note length. A check strict enough to trip on OCR noise would # make a working system look broken, which is worse than the @@ -515,6 +517,14 @@ def test_right_type_passes_at_every_note_length(self, note: str) -> None: f"Encounter saved - {note[:40]}", f"Triage - {note[:60]}", ) + # Keep the field-separation regression independent of rapidocr's + # platform-specific segmentation of the longest synthetic Hershey + # line. TestVerify above retains a real-OCR smoke for the benchmark's + # actual 32-character note; these four cases pin the verifier logic. + monkeypatch.setattr( + "openadapt_flow.benchmark.verify.ocr", + lambda _screen: [SimpleNamespace(text=f"Triage - {note[:60]}")], + ) verdict = verify_encounter_saved(screen, note) assert verdict.banner_found assert verdict.note_found @@ -522,7 +532,9 @@ def test_right_type_passes_at_every_note_length(self, note: str) -> None: assert verdict.success @pytest.mark.parametrize("note", NOTE_LENGTHS) - def test_wrong_type_fails_at_every_note_length(self, note: str) -> None: + def test_wrong_type_fails_at_every_note_length( + self, note: str, monkeypatch: pytest.MonkeyPatch + ) -> None: # The regression: the note is correct and the banner is present, # but the encounter was saved as Consult. That is a wrong-target # write, not a success, however long the note is. @@ -530,6 +542,10 @@ def test_wrong_type_fails_at_every_note_length(self, note: str) -> None: f"Encounter saved - {note[:40]}", f"Consult - {note[:60]}", ) + monkeypatch.setattr( + "openadapt_flow.benchmark.verify.ocr", + lambda _screen: [SimpleNamespace(text=f"Consult - {note[:60]}")], + ) verdict = verify_encounter_saved(screen, note) assert verdict.banner_found assert not verdict.note_found From f441f870a38f79ff1880fc33b032f26159e9c25b Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 01:44:14 -0400 Subject: [PATCH 3/8] fix: require saved-row context in OpenEMR oracle --- openadapt_flow/benchmark/verify.py | 101 +++++++++++++++++---- tests/test_openemr_benchmark.py | 139 +++++++++++++++++++++++------ 2 files changed, 192 insertions(+), 48 deletions(-) diff --git a/openadapt_flow/benchmark/verify.py b/openadapt_flow/benchmark/verify.py index c549a7db..36eccf3d 100644 --- a/openadapt_flow/benchmark/verify.py +++ b/openadapt_flow/benchmark/verify.py @@ -23,7 +23,7 @@ from pydantic import BaseModel -from openadapt_flow.vision import find_text, normalize_text, ocr, upscale_png +from openadapt_flow.vision import OcrLine, find_text, normalize_text, ocr, upscale_png BANNER_PREFIX = "Encounter saved" #: MockMed truncation limits (static/app.js): banner shows note[:40], the @@ -250,6 +250,68 @@ def _score_note(hay: str, needle: str) -> NoteVerifyResult: ) +def _saved_message_note_lines(lines: list[OcrLine]) -> list[OcrLine]: + """Return OCR lines that occupy a saved Patient Messages table row. + + OpenEMR renders the saved note in the ``Content`` column and renders a + ``New`` status in the adjacent ``Status`` column. OCR can return that + status separately or merge it onto the content line. A wrapped continuation + can sit one text line below the status. The entry form is above the table, + so its textarea does not satisfy this geometric row contract. + + The contract uses only pixels and OCR geometry. It does not read the DOM or + trust the runner's internal result. + """ + content_headers = [line for line in lines if _squash(line.text) == "content"] + status_headers = [line for line in lines if _squash(line.text) == "status"] + candidates: list[OcrLine] = [] + for content in content_headers: + cx, cy, cw, ch = content.region + content_center_x = cx + cw / 2 + content_center_y = cy + ch / 2 + for status in status_headers: + sx, sy, sw, sh = status.region + status_center_x = sx + sw / 2 + status_center_y = sy + sh / 2 + if status_center_x <= content_center_x: + continue + if abs(status_center_y - content_center_y) > 2 * max(ch, sh): + continue + + header_bottom = max(cy + ch, sy + sh) + row_status_bands = [ + (ly + lh / 2, lh) + for line in lines + for lx, ly, lw, lh in (line.region,) + if ly > header_bottom + and ( + ( + _squash(line.text) == "new" + and lx + lw / 2 >= sx - sw + ) + or ( + _squash(line.text).endswith("new") + and lx + lw >= sx + ) + ) + ] + for line in lines: + lx, ly, lw, lh = line.region + line_center_x = lx + lw / 2 + line_center_y = ly + lh / 2 + if ly <= header_bottom: + continue + if not (content_center_x - cw <= line_center_x < status_center_x): + continue + if any( + abs(line_center_y - row_status_center_y) + <= 2 * max(lh, status_h) + for row_status_center_y, status_h in row_status_bands + ): + candidates.append(line) + return candidates + + def verify_note_saved( screen_png: bytes, note_text: str, @@ -258,11 +320,12 @@ def verify_note_saved( ) -> NoteVerifyResult: """Check a final-state screenshot for the saved OpenEMR note. - The message list embeds the note inside a longer line (`` - (admin to admin) ``) and wraps it, so whole-line fuzzy matching - misses; and rapidocr drops some dense table lines entirely at - 1280x800, so when the raw frame does not pass, the frame is retried - at 2x resolution, which recovers most dropped lines. + The message list embeds the note in the ``Content`` column of a saved + table row and wraps it. A valid evidence line must be below the table's + ``Content``/``Status`` headers and aligned with that row's ``New`` status. + Thus, the same note in the unsaved entry form does not pass. RapidOCR drops + some dense table lines entirely at 1280x800, so when the raw frame does not + pass, the frame is retried at 2x resolution. The criterion is a **contiguous** matched run of at least ``min_run`` squashed characters between the note and the frame's OCR text. A @@ -293,17 +356,17 @@ def verify_note_saved( best = NoteVerifyResult(success=False, matched_ratio=0.0, longest_run=0) for png in (screen_png, upscale_png(screen_png)): - hay = _squash(" ".join(line.text for line in ocr(png))) - result = _score_note(hay, needle) - if result.success or result.longest_run >= min_run: - return NoteVerifyResult( - success=True, - matched_ratio=result.matched_ratio, - longest_run=result.longest_run, - ) - if (result.longest_run, result.matched_ratio) > ( - best.longest_run, - best.matched_ratio, - ): - best = result + for line in _saved_message_note_lines(ocr(png)): + result = _score_note(_squash(line.text), needle) + if result.success or result.longest_run >= min_run: + return NoteVerifyResult( + success=True, + matched_ratio=result.matched_ratio, + longest_run=result.longest_run, + ) + if (result.longest_run, result.matched_ratio) > ( + best.longest_run, + best.matched_ratio, + ): + best = result return best diff --git a/tests/test_openemr_benchmark.py b/tests/test_openemr_benchmark.py index 3a550c3f..f7407fe2 100644 --- a/tests/test_openemr_benchmark.py +++ b/tests/test_openemr_benchmark.py @@ -1,8 +1,8 @@ """Unit tests for the OpenEMR benchmark pieces (no network anywhere). -The Anthropic client and backend are faked, the orchestrator's run helpers -are monkeypatched, and ``verify_note_saved`` runs real OCR on synthetic -cv2-rendered screenshots — the same testing style as ``test_benchmark.py``. +The Anthropic client and backend are faked, and the orchestrator's run helpers +are monkeypatched. Saved-note tests use exact OCR text and geometry so font and +OCR differences between operating systems cannot change the row contract. """ from __future__ import annotations @@ -14,6 +14,7 @@ import cv2 import numpy as np +import openadapt_flow.benchmark.verify as benchmark_verify from openadapt_flow.benchmark import agent_baseline, openemr_benchmark from openadapt_flow.benchmark.agent_baseline import ( openemr_task_prompt, @@ -27,6 +28,7 @@ write_openemr_outputs, ) from openadapt_flow.benchmark.verify import verify_note_saved +from openadapt_flow.vision import OcrLine NOTE = "Insurance card copied and coverage verified by phone." @@ -60,48 +62,127 @@ def make_screen(*lines: str, thickness: int = 2) -> bytes: class TestVerifyNoteSaved: - def test_note_embedded_in_message_row_passes(self) -> None: - # OpenEMR shows the note inside a longer list line (timestamp + - # "(admin to admin)" prefix). Rendered at thickness=1 (thin strokes, - # like real anti-aliased browser text) rather than the helper's - # bold thickness=2 default: rapidocr's angle classifier mis-detects - # the BOLD cv2-Hershey render of this long, digit-prefixed line as - # 180°-rotated and flips it, garbling the OCR. That is a synthetic- - # render artifact of the blocky bold font — it does NOT occur on the - # real browser-rendered OpenEMR row (which verifies at 100%). The row - # content, the note, and the criterion below are all unchanged; only - # the incidental stroke weight is thinned so OCR reads the frame the - # verifier actually faces in production. - screen = make_screen( - "Patient Messages", - f"2026-07-08 (admin to admin) {NOTE}", - thickness=1, + @staticmethod + def _patch_ocr(monkeypatch: Any, lines: list[OcrLine]) -> None: + monkeypatch.setattr(benchmark_verify, "ocr", lambda _png: lines) + monkeypatch.setattr(benchmark_verify, "upscale_png", lambda png: png) + + @staticmethod + def _table(*rows: OcrLine) -> list[OcrLine]: + return [ + OcrLine(text="Content", region=(420, 420, 70, 20), confidence=1.0), + OcrLine(text="Status", region=(900, 420, 60, 20), confidence=1.0), + *rows, + ] + + def test_note_embedded_in_message_row_passes(self, monkeypatch: Any) -> None: + self._patch_ocr( + monkeypatch, + self._table( + OcrLine( + text=f"2026-07-08 (admin to admin) {NOTE}", + region=(330, 480, 550, 20), + confidence=1.0, + ), + OcrLine(text="New", region=(900, 480, 40, 20), confidence=1.0), + ), ) + screen = BLANK_PNG verdict = verify_note_saved(screen, NOTE) assert verdict.success assert verdict.longest_run >= 16 - def test_wrapped_fragment_passes(self) -> None: - # A wrapped note where OCR only captured one distinctive fragment - # of at least 16 contiguous characters still counts. - screen = make_screen("coverage verified by") - verdict = verify_note_saved(screen, NOTE) + def test_wrapped_fragment_passes(self, monkeypatch: Any) -> None: + # A continuation line can sit below the row's status line. + self._patch_ocr( + monkeypatch, + self._table( + OcrLine( + text="coverage verified by", + region=(420, 505, 180, 20), + confidence=1.0, + ), + OcrLine(text="New", region=(900, 480, 40, 20), confidence=1.0), + ), + ) + verdict = verify_note_saved(BLANK_PNG, NOTE) assert verdict.success assert verdict.longest_run >= 16 + def test_status_merged_onto_content_line_passes(self, monkeypatch: Any) -> None: + # RapidOCR can merge the adjacent New cell onto a long content line. + self._patch_ocr( + monkeypatch, + self._table( + OcrLine( + text="2026-07-08 (admin to admin) " + "Travel vaccine consult booked beforeNew", + region=(330, 480, 610, 20), + confidence=1.0, + ), + ), + ) + verdict = verify_note_saved( + BLANK_PNG, + "Travel vaccine consult booked before the June trip.", + ) + assert verdict.success + assert verdict.longest_run >= 16 + + def test_note_in_unsaved_entry_form_fails(self, monkeypatch: Any) -> None: + # The exact note is visible in the textarea, but no saved row contains + # it. This is the retained compiled_019 false-success shape. + self._patch_ocr( + monkeypatch, + [ + OcrLine( + text="Patient Message", + region=(420, 280, 160, 20), + confidence=1.0, + ), + OcrLine(text=NOTE, region=(460, 340, 440, 20), confidence=1.0), + OcrLine( + text="Save as new message", + region=(460, 380, 180, 20), + confidence=1.0, + ), + *self._table( + OcrLine( + text="A different saved message.", + region=(420, 480, 240, 20), + confidence=1.0, + ), + OcrLine( + text="New", region=(900, 480, 40, 20), confidence=1.0 + ), + ), + ], + ) + verdict = verify_note_saved(BLANK_PNG, NOTE) + assert not verdict.success + assert verdict.longest_run < 16 + def test_blank_screen_fails(self) -> None: verdict = verify_note_saved(BLANK_PNG, NOTE) assert not verdict.success assert verdict.longest_run < 16 - def test_wrong_note_fails(self) -> None: + def test_wrong_note_fails(self, monkeypatch: Any) -> None: # A different run's note visible on screen must not satisfy this # run's check (contiguous-run criterion, dissimilar note texts). - screen = make_screen( - "2026-07-08 (admin to admin) " - "Dermatology biopsy site healing cleanly, no drainage.", + self._patch_ocr( + monkeypatch, + self._table( + OcrLine( + text="2026-07-08 (admin to admin) " + "Dermatology biopsy site healing cleanly, no drainage.", + region=(330, 480, 550, 20), + confidence=1.0, + ), + OcrLine(text="New", region=(900, 480, 40, 20), confidence=1.0), + ), ) - verdict = verify_note_saved(screen, NOTE) + verdict = verify_note_saved(BLANK_PNG, NOTE) assert not verdict.success def test_empty_note_fails(self) -> None: From 7a76a7d801e9a4ddc4e06242244bc1a8e830144f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 02:00:09 -0400 Subject: [PATCH 4/8] fix: correct OpenEMR saved-row benchmark claims --- README.md | 7 +- benchmark/comparison_artifact/README.md | 11 +-- benchmark/comparison_artifact/comparison.html | 19 ++-- benchmark/comparison_artifact/comparison.json | 29 +++++- benchmark/comparison_artifact/generate.py | 66 +++++++------ benchmark/openemr/BENCHMARK.md | 42 ++++----- benchmark/openemr/results.json | 35 ++++++- claims.yaml | 20 +++- docs/VERIFICATION.md | 8 +- docs/verification.json | 18 +++- openadapt_flow/benchmark/openemr_benchmark.py | 93 +++++++++++++------ openadapt_flow/benchmark/verify.py | 6 +- paper/sections/05_results.tex | 18 ++-- tests/test_comparison_artifact.py | 5 +- tests/test_openemr_benchmark.py | 21 ++++- tests/test_openemr_retained_finals.py | 70 ++++++++++++++ 16 files changed, 336 insertions(+), 132 deletions(-) create mode 100644 tests/test_openemr_retained_finals.py diff --git a/README.md b/README.md index 6b24d95f..c9165275 100644 --- a/README.md +++ b/README.md @@ -492,10 +492,13 @@ a note) with a distinct note value each run and the same OCR success check on both arms: 20 compiled replays against 10 runs of a claude-sonnet-5 computer-use agent, measured on 2026-07-08 from a pre-v0.2.0 source checkout declaring openadapt-flow 0.1.0. Compiled went -20/20 at 39.2s (p50) +19/20 at 39.2s (p50) with zero model calls; the agent went 10/10 at 70.4s (p50), about $0.55 per run at list price ($5.52 total for the 10 runs, with prompt caching -and hard cost caps enforced in the harness). It is a shared public demo +and hard cost caps enforced in the harness). The corrected OCR check requires +the note in a saved Patient Messages row; it rejects one compiled run where the +note remained in the unsaved entry form. This is screen-row evidence, not a +system-of-record read. It is a shared public demo that other users mutate and that resets daily, so it is not CI-reproducible, and the sample is small. Correctness alone (no agent arm, 5/5 fresh browsers, zero model calls, closed-loop scrolling) is in diff --git a/benchmark/comparison_artifact/README.md b/benchmark/comparison_artifact/README.md index 9ce225da..f4f0a08d 100644 --- a/benchmark/comparison_artifact/README.md +++ b/benchmark/comparison_artifact/README.md @@ -2,8 +2,8 @@ A self-contained, theme-aware HTML page that packages the core wedge in one place: for a task you have **already demonstrated**, a compiled replay is -**model-free**, **~$0 per run**, and **faster** than a computer-use agent — at -**parity** success on these tasks. +**model-free**, **~$0 per run**, and **faster** than a computer-use agent. The +artifact reports the measured success of each arm; it does not assume parity. It is **generated from the repo's real benchmark results**, not hand-typed. The generator reads the two existing `results.json` files and lays their figures out @@ -34,7 +34,7 @@ Leads with the **real third-party result** (OpenEMR public demo, an 18-step add-patient-note workflow on a live EMR), then the **CI-reproducible anchor** (MockMed, the bundled demo clinic). For each benchmark: -- **Success parity** — both arms pass the same arm-independent OCR check. +- **Measured success** — both arms use the same arm-independent OCR contract. - **Measured model API cost** — `$0` for compiled vs `$/run` for the agent, as an inline-SVG bar chart plus an explicitly limited arithmetic projection. - **Latency** — p50/p95 wall-clock for both arms as grouped inline-SVG bars. @@ -45,9 +45,8 @@ labels) — no screenshots, no external assets, no base64 needed. It ends with an **honest "Read before quoting these numbers"** panel (mirroring the gallery's "What still slips" tone): small N and wide error bars; the lead is a field result on a shared, daily-resetting public demo (not CI-reproducible); -list-price costs with hard caps; the single OCR success check that errs -conservative on both arms; and the fact that this measures cost/latency at -**parity** success — **not** a general capability claim. +list-price costs with hard caps; the corrected saved-row OCR contract; and the +fact that this is **not** a general capability claim. ## Regenerate diff --git a/benchmark/comparison_artifact/comparison.html b/benchmark/comparison_artifact/comparison.html index 61b4e3be..b7a2b6a9 100644 --- a/benchmark/comparison_artifact/comparison.html +++ b/benchmark/comparison_artifact/comparison.html @@ -159,10 +159,11 @@

    Compiled Replay vs. Computer-Use Agent

    One clinical task, two ways to automate it, one success check. The wedge in one line: for a task you have already demonstrated, a compiled replay is model-free, ~$0 per run, and - faster — at parity success. Generated + faster on these measured trials. Each arm's success is + reported separately. Generated straight from the repo's real benchmark results.json files; no number here is hand-typed.

    -
    Same task, same success (20/20 vs 10/10) — compiled replay costs $0 and runs 1.8× faster than the claude-sonnet-5 agent
    +
    Same task, measured success (19/20 vs 10/10) — compiled replay costs $0 and runs 1.8× faster than the claude-sonnet-5 agent

    What this is. The economics of the 500th run of a known task: once a workflow has been demonstrated once, replaying @@ -186,7 +187,7 @@

    OpenEMR public demo — a real third-party EMR

    OpenEMR public demo: log in, search the demo patient, open the chart, scroll the dashboard to the Messages card, open Patient Messages, add a parameterized note, save · 18 compiled steps

    -
    success — parity
    20/20 · 10/10
    compiled · agent — both pass the same arm-independent OCR check
    model cost / run
    $0 vs $0.5522
    agent arm total $5.52 over 10 runs
    latency p50
    39.2 s vs 70.4 s
    compiled is 1.8× faster at the median
    +
    measured success
    19/20 · 10/10
    compiled · agent — same arm-independent screen contract
    model cost / run
    $0 vs $0.5522
    agent arm total $5.52 over 10 runs
    latency p50
    39.2 s vs 70.4 s
    compiled is 1.8× faster at the median
    Model cost per run
    $0$0.2000$0.4000$0.6000USD / run (list price)compiled replay$0computer-use agent$0.5522
    In this measured sample, compiled replay made zero model calls and incurred $0 in model API charges.
    Latency (wall-clock)
    0255075100secondscompiled p5039.2 scompiled p9541.0 sagent p5070.4 sagent p9582.6 s
    Per-run wall-clock around the replay / agent loop only.
    @@ -199,7 +200,7 @@

    OpenEMR public demo — a real third-party EMR

    arm-independent OCR check applied to both arms — neither arm's self-report is trusted. Not CI-reproducible: a single shared public instance that every internet visitor mutates and that resets daily. Treat as a field result. figures: benchmark/openemr/results.json

    -

    One compiled run (#19) self-flagged expected-screen drift at step_017 and aborted — yet the arm-independent OCR check confirmed the note saved, so it counts as a success. On a shared instance the message list grows under every visitor, so a postcondition can drift after the write lands; the self-flag halting instead of improvising is the point.

    +

    One compiled run (#20) self-flagged expected-screen drift at step_017 and aborted. The corrected saved-row OCR check also rejects it: the retained final frame shows the note in the unsaved entry form, not in a saved message row. It counts as a failure.

    @@ -212,7 +213,7 @@

    MockMed — the bundled demo clinic (CI-reproducible anchor)

    MockMed triage: login, open first referral, create a Triage encounter, enter the note, save

    -
    success — parity
    100/100 · 20/20
    compiled · agent — both pass the same arm-independent OCR check
    model cost / run
    $0 vs $0.2716
    agent arm total $5.43 over 20 runs
    latency p50
    4.9 s vs 37.5 s
    compiled is 7.6× faster at the median
    +
    measured success
    100/100 · 20/20
    compiled · agent — same arm-independent screen contract
    model cost / run
    $0 vs $0.2716
    agent arm total $5.43 over 20 runs
    latency p50
    4.9 s vs 37.5 s
    compiled is 7.6× faster at the median
    Model cost per run
    $0$0.1000$0.2000$0.3000USD / run (list price)compiled replay$0computer-use agent$0.2716
    In this measured sample, compiled replay made zero model calls and incurred $0 in model API charges.
    Latency (wall-clock)
    0204060secondscompiled p504.9 scompiled p955.1 sagent p5037.5 sagent p9543.4 s
    Per-run wall-clock around the replay / agent loop only.
    @@ -239,11 +240,11 @@

    Read before quoting these numbers

  • Small N, wide error bars.

    The agent arm is N=10 on OpenEMR and N=20 on MockMed, because agent runs cost real money, real minutes, and real load on a shared public service. A 100% success rate over ten runs is not a five-nines claim — its confidence interval is wide. The compiled arm (N=20 / 100) is cheap to repeat, so its bars are tighter, but the honest comparison is still small-sample on the agent side.

  • The lead result is a field result, not CI-reproducible.

    OpenEMR is a single shared public demo that anyone on the internet can mutate and that resets daily; every successful run also appends a message that grows the dashboard for the next run. Numbers depend on that instance's state and load on the day. The MockMed row is the reproducible anchor — treat OpenEMR as a real-world sighting, not a repeatable measurement.

  • Cost is list price with hard caps — billed cost is lower.

    Costs are computed from API token counts at list pricing. list price; an introductory $2/$10 rate applies through 2026-08-31, so the amount actually billed today is about a third lower than the figures shown. Every agent run is capped at $1.50 and the whole agent arm at $8.00 (list price); the caps stop the arm and the truncation is disclosed in the source data rather than hidden.

    • -
  • Success is one OCR check on both arms — and it errs conservative.

    A run passes only if the run's own note text is read back out of the final screenshot by OCR, identically for compiled and agent. On dense EMR text the OCR sometimes drops the exact line it is looking for, so a 'failed' verification can be a measurement miss with the note plainly on screen. The check is identical for both arms, so it cannot favour one — but it under-counts rather than over-counts success.

    • -
  • This measures cost and latency at PARITY success — not capability.

    Both arms passed every task here, so the story is purely how much cheaper and faster the compiled path is at the same outcome. It is NOT a claim that compiled replay is more capable, more robust to novel situations, or a substitute for an agent on a task it has never seen. MockMed is also a deliberately simple app; harder surfaces would slow and likely degrade both arms, plausibly at different rates.

    • +
  • Success is corrected saved-row screen evidence, not a record read.

    The legacy whole-frame OCR check accepted one compiled run whose note was still in the unsaved entry form. The corrected contract requires the note in a saved Patient Messages row, and replay of all 30 retained final frames changes only that run. OCR can still miss dense text. This field result does not read the OpenEMR system of record out of band.

    • +
  • This measures bounded outcomes, cost, and latency — not capability.

    The OpenEMR outcomes are 19/20 and 10/10 under one corrected screen contract. The result shows the measured cost and latency difference. It is NOT a claim that compiled replay is more capable, more robust to novel situations, or a substitute for an agent on a task it has never seen. MockMed is also a deliberately simple app; harder surfaces would slow and likely degrade both arms, plausibly at different rates.

    • -

    Bottom line: at parity success on these tasks, the compiled path - removes the model from the loop — $0 and +

    Bottom line: on these measured trials, the compiled path removes + the model from the loop — $0 and 1.8× faster on the real EMR, and a tighter, reproducible version of the same gap on MockMed. That is a cost/latency result on known tasks, disclosed with its limits — not a general capability claim.

    diff --git a/benchmark/comparison_artifact/comparison.json b/benchmark/comparison_artifact/comparison.json index 399f7976..93e8f6cc 100644 --- a/benchmark/comparison_artifact/comparison.json +++ b/benchmark/comparison_artifact/comparison.json @@ -23,8 +23,8 @@ "arms": { "compiled": { "n": 20, - "success_count": 20, - "success_rate": 1.0, + "success_count": 19, + "success_rate": 0.95, "wall_s_p50": 39.170917561999886, "wall_s_p95": 41.04867044315233, "cost_usd_per_run": 0.0, @@ -49,8 +49,31 @@ "pricing_note": "list price; an introductory $2/$10 rate applies through 2026-08-31", "compiled_self_flag": { "i": 19, - "success": true, + "success": false, "first_failure_step": "step_017" + }, + "success_contract": { + "kind": "screen_saved_message_row", + "note_min_contiguous_squashed_chars": 16, + "requires": [ + "Patient Messages Content and Status headers", + "note fragment in the Content column below the headers", + "an aligned New status for the same saved row" + ], + "rejects": "note text visible only in the unsaved entry form" + }, + "oracle_adjudication": { + "corrected_at": "2026-07-28", + "contract": "The final screenshot must show a contiguous note fragment of at least 16 squashed characters inside a saved Patient Messages Content row aligned with its New status. Note text in the unsaved entry form is not saved-row evidence.", + "retained_final_path": "benchmark/openemr/finals/compiled_019.png", + "retained_final_sha256": "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f", + "old_behavior": "The legacy verifier accepted the requested note anywhere in full-frame OCR and returned success for compiled run 20.", + "new_behavior": "The saved-row verifier rejects compiled run 20 because the note remains in the open textarea and the Save as new message button remains visible; no saved row contains the note.", + "changed_runs": [ + "compiled:19" + ], + "denominators_changed": false, + "latency_aggregates_changed": false } } }, diff --git a/benchmark/comparison_artifact/generate.py b/benchmark/comparison_artifact/generate.py index c2f76ebb..e2051705 100644 --- a/benchmark/comparison_artifact/generate.py +++ b/benchmark/comparison_artifact/generate.py @@ -7,8 +7,9 @@ - ``benchmark/openemr/results.json`` — the LEAD result: the same task run against a real third-party application (the official OpenEMR public demo), - 20 compiled replays vs 10 ``claude-sonnet-5`` computer-use agent runs, both - 100%. This is a field result, not CI-reproducible (shared public instance). + 20 compiled replays vs 10 ``claude-sonnet-5`` computer-use agent runs. The + corrected saved-row oracle counts 19/20 and 10/10. This is a field result, + not CI-reproducible (shared public instance). - ``benchmark/results.json`` — the CI-reproducible MockMed anchor: 100 compiled vs 20 agent on the bundled demo clinic, both 100%, same orchestrator and same arm-independent OCR success check. @@ -32,8 +33,8 @@ their provenance), so the page is verifiable without eyeballing it. The page leads with the honest wedge: compiled replay is model-free, ~$0/run, -and faster, at *parity* success on these tasks — with the small-n, shared-demo, -and not-a-capability-claim caveats stated up front, not buried. +and faster on these trials. It reports each arm's measured success and states +the small-n, shared-demo, and not-a-capability-claim caveats up front. """ from __future__ import annotations @@ -139,8 +140,8 @@ def load_openemr(path: Path = OPENEMR_RESULTS) -> Benchmark: model = str(d.get("model", "claude-sonnet-5")) compiled = _arm_from_json("compiled", arms["compiled"], "0 model calls") agent = _arm_from_json("agent", arms["agent"], model) - # A single compiled run self-flagged expected-screen drift and aborted; the - # arm-independent OCR check confirmed the write landed. Report it honestly. + # A single compiled run self-flagged expected-screen drift and aborted. The + # corrected saved-row OCR check also rejects that run. Report both signals. self_flag = None for run in d.get("runs", {}).get("compiled", []): if run.get("replayer_success") is False: @@ -166,6 +167,8 @@ def load_openemr(path: Path = OPENEMR_RESULTS) -> Benchmark: "cost_caps_usd": d.get("cost_caps_usd"), "pricing_note": (d.get("pricing_usd_per_mtok") or {}).get("note"), "compiled_self_flag": self_flag, + "success_contract": d.get("success_contract"), + "oracle_adjudication": d.get("oracle_adjudication"), }, ) @@ -425,10 +428,10 @@ def _stat_tiles(b: Benchmark) -> str: speed = _speedup(b) tiles = [ ( - "success — parity", + "measured success", f"{b.compiled.success_count}/{b.compiled.n} · " f"{b.agent.success_count}/{b.agent.n}", - "compiled · agent — both pass the same arm-independent OCR check", + "compiled · agent — same arm-independent screen contract", "ok", ), ( @@ -484,14 +487,21 @@ def _self_flag_note(b: Benchmark) -> str: sf = b.extras.get("compiled_self_flag") if not sf: return "" + run_number = int(sf.get("i", -1)) + 1 + if sf.get("success") is False: + return ( + '

    One compiled run (#{i}) self-flagged ' + 'expected-screen drift at {step} and aborted. The ' + 'corrected saved-row OCR check also rejects it: the retained final ' + 'frame shows the note in the unsaved entry form, not in a saved ' + 'message row. It counts as a failure.

    ' + ).format(i=_e(run_number), step=_e(sf.get("first_failure_step"))) return ( '

    One compiled run (#{i}) self-flagged ' - 'expected-screen drift at {step} and aborted — yet the ' - 'arm-independent OCR check confirmed the note saved, so it counts as a ' - 'success. On a shared instance the message list grows under every ' - "visitor, so a postcondition can drift after the write lands; " - 'the self-flag halting instead of improvising is the point.

    ' - ).format(i=_e(sf.get("i")), step=_e(sf.get("first_failure_step"))) + 'expected-screen drift at {step} and aborted, while the ' + 'saved-row OCR check accepted its final frame. It counts according to ' + 'that shared screen contract.

    ' + ).format(i=_e(run_number), step=_e(sf.get("first_failure_step"))) def _drift_note(b: Benchmark) -> str: @@ -594,18 +604,17 @@ def _caveats(oe: Benchmark, mm: Benchmark) -> list[tuple[str, str]]: "in the source data rather than hidden.", ), ( - "Success is one OCR check on both arms — and it errs conservative.", - "A run passes only if the run's own note text is read back out of the " - "final screenshot by OCR, identically for compiled and agent. On dense " - "EMR text the OCR sometimes drops the exact line it is looking for, so " - "a 'failed' verification can be a measurement miss with the note " - "plainly on screen. The check is identical for both arms, so it cannot " - "favour one — but it under-counts rather than over-counts success.", + "Success is corrected saved-row screen evidence, not a record read.", + "The legacy whole-frame OCR check accepted one compiled run whose note " + "was still in the unsaved entry form. The corrected contract requires " + "the note in a saved Patient Messages row, and replay of all 30 retained " + "final frames changes only that run. OCR can still miss dense text. This " + "field result does not read the OpenEMR system of record out of band.", ), ( - "This measures cost and latency at PARITY success — not capability.", - "Both arms passed every task here, so the story is purely how much " - "cheaper and faster the compiled path is at the same outcome. It is " + "This measures bounded outcomes, cost, and latency — not capability.", + "The OpenEMR outcomes are 19/20 and 10/10 under one corrected screen " + "contract. The result shows the measured cost and latency difference. It is " "NOT a claim that compiled replay is more capable, more robust to novel " "situations, or a substitute for an agent on a task it has never seen. " "MockMed is also a deliberately simple app; harder surfaces would slow " @@ -617,7 +626,7 @@ def _caveats(oe: Benchmark, mm: Benchmark) -> list[tuple[str, str]]: def render_html(oe: Benchmark, mm: Benchmark) -> str: speed_oe = _speedup(oe) headline = ( - f"Same task, same success ({oe.compiled.success_count}/{oe.compiled.n} " + f"Same task, measured success ({oe.compiled.success_count}/{oe.compiled.n} " f"vs {oe.agent.success_count}/{oe.agent.n}) — compiled replay costs " f"{fmt_usd(oe.compiled.cost_per_run)} and runs {speed_oe:.1f}× faster " f"than the {oe.model} agent" @@ -642,7 +651,8 @@ def render_html(oe: Benchmark, mm: Benchmark) -> str:

    One clinical task, two ways to automate it, one success check. The wedge in one line: for a task you have already demonstrated, a compiled replay is model-free, ~$0 per run, and - faster — at parity success. Generated + faster on these measured trials. Each arm's success is + reported separately. Generated straight from the repo's real benchmark results.json files; no number here is hand-typed.

    {headline}
    @@ -671,8 +681,8 @@ def render_html(oe: Benchmark, mm: Benchmark) -> str:
      {caveats}
    -

    Bottom line: at parity success on these tasks, the compiled path - removes the model from the loop — {fmt_usd(oe.compiled.cost_per_run)} and +

    Bottom line: on these measured trials, the compiled path removes + the model from the loop — {fmt_usd(oe.compiled.cost_per_run)} and {speed_oe:.1f}× faster on the real EMR, and a tighter, reproducible version of the same gap on MockMed. That is a cost/latency result on known tasks, disclosed with its limits — not a general capability claim.

    diff --git a/benchmark/openemr/BENCHMARK.md b/benchmark/openemr/BENCHMARK.md index 9858a64f..477233e5 100644 --- a/benchmark/openemr/BENCHMARK.md +++ b/benchmark/openemr/BENCHMARK.md @@ -17,7 +17,7 @@ BOTH arms), save. | | compiled replay | computer-use agent | |---|---|---| | runs | 20 | 10 | -| success rate | 100% (20/20) | 100% (10/10) | +| success rate | 95% (19/20) | 100% (10/10) | | latency p50 | 39.2 s | 70.4 s | | latency p95 | 41.0 s | 82.6 s | | model cost / run | $0 | $0.5522 | @@ -35,13 +35,7 @@ Failed runs, reported honestly: Compiled arm: -- none - -Compiled runs that self-flagged, also reported honestly (success is judged by the arm-independent OCR check both arms share, not the replayer's self-report): - -- compiled run 20: postconditions flagged expected-screen drift at step_017 and the replayer aborted; the arm-independent OCR check verified the note saved (matched 100%) - -On a shared instance every run — ours and other visitors' — grows the message list, so a postcondition can drift after the action lands. The self-flag is the point: the replayer detects the drift and halts instead of improvising. +- compiled run 20: saved-row oracle failed; replayer halted at step_017 — The legacy whole-frame OCR verifier matched the note in the unsaved entry textarea. The retained final frame has no saved row for this note. Agent arm: @@ -76,15 +70,17 @@ below. target patient, the exact note text — not steps or coordinates. Every executed action returns a settled screenshot. - **Same success criterion, implemented once.** After each run, the final - screenshot is checked by `verify_note_saved` (OCR): a contiguous run of - at least 16 characters of the run's note must appear in the frame's - OCR text (whitespace-squashed; retried at 2x resolution when the raw - frame does not pass, because rapidocr drops dense table lines at - 1280x800). Neither arm's self-reported success is used. + screenshot is checked by `verify_note_saved` (OCR). A contiguous run of + at least 16 characters of the run's note must appear in a saved Patient + Messages `Content` row aligned with its `New` status. The same note in the + unsaved entry form does not pass. OCR is whitespace-squashed and retried at + 2x resolution when the raw frame does not pass. Neither arm's self-reported + success is used. This is screen-row evidence, not an out-of-band read of the + OpenEMR system of record. - **Distinct, mutually dissimilar note per run in BOTH arms** (no two notes share a 16-character squashed substring — unit-tested), so - success proves parameter substitution against live state and one run's - note cannot satisfy another run's check. + a saved-row success proves parameter substitution in the visible message + list and one run's note cannot satisfy another run's check. - **Pacing.** Runs are spaced ~30s apart as public-demo courtesy; the pacing gap is excluded from latency. - **Latency** is wall-clock around the replay / agent loop only. @@ -133,15 +129,13 @@ below. - **The compiled arm needs a demonstration first.** The one-time record + compile step (about a minute of human demonstration) is the price of the fast replays; the agent needs only the prompt. -- **OCR verification on dense EMR text under-counts.** rapidocr sometimes - drops the exact table line containing the note (a known limitation - documented in - [docs/showcase-openemr/FINDINGS.md](../../docs/showcase-openemr/FINDINGS.md)), - so a "failed" verification can be a measurement miss with the note - plainly visible in the final screenshot. The check errs conservative - and is identical for both arms. Every run's final screenshot is saved - to `benchmark/openemr/finals/` (local only, not committed) so failed - verdicts can be audited against what was actually on screen. +- **The success oracle is bounded screen evidence.** rapidocr can drop dense + table lines and cause an over-halt. The legacy whole-frame check could also + accept note text in the unsaved form; retained compiled run 20 exposed that + false success. The corrected check requires saved-row context. All 30 final + frames were replayed under it: only compiled run 20 changed. Every final + screenshot stays in `benchmark/openemr/finals/` (local only, not committed) + for audit. This check does not read the OpenEMR system of record. - Single machine (macOS-15.7.3-arm64-arm-64bit). ## Reproduce diff --git a/benchmark/openemr/results.json b/benchmark/openemr/results.json index 93354594..368c2857 100644 --- a/benchmark/openemr/results.json +++ b/benchmark/openemr/results.json @@ -17,7 +17,30 @@ "parent_before_artifact_commit": "099eac0759440a86eceef38530797e8d5b765a15", "note": "These SHAs identify when the retained rows entered repository history; neither is attributed as the exact runtime HEAD." }, + "oracle_adjudication": { + "corrected_at": "2026-07-28", + "contract": "The final screenshot must show a contiguous note fragment of at least 16 squashed characters inside a saved Patient Messages Content row aligned with its New status. Note text in the unsaved entry form is not saved-row evidence.", + "retained_final_path": "benchmark/openemr/finals/compiled_019.png", + "retained_final_sha256": "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f", + "old_behavior": "The legacy verifier accepted the requested note anywhere in full-frame OCR and returned success for compiled run 20.", + "new_behavior": "The saved-row verifier rejects compiled run 20 because the note remains in the open textarea and the Save as new message button remains visible; no saved row contains the note.", + "changed_runs": [ + "compiled:19" + ], + "denominators_changed": false, + "latency_aggregates_changed": false + }, "task": "OpenEMR public demo: log in, search the demo patient, open the chart, scroll the dashboard to the Messages card, open Patient Messages, add a parameterized note, save", + "success_contract": { + "kind": "screen_saved_message_row", + "note_min_contiguous_squashed_chars": 16, + "requires": [ + "Patient Messages Content and Status headers", + "note fragment in the Content column below the headers", + "an aligned New status for the same saved row" + ], + "rejects": "note text visible only in the unsaved entry form" + }, "target": "https://demo.openemr.io/openemr/index.php", "workflow_steps": 18, "model": "claude-sonnet-5", @@ -35,8 +58,8 @@ "arms": { "compiled": { "n": 20, - "success_count": 20, - "success_rate": 1.0, + "success_count": 19, + "success_rate": 0.95, "wall_s_p50": 39.170917561999886, "wall_s_p95": 41.04867044315233, "wall_s_mean": 39.51734407485037, @@ -449,7 +472,7 @@ { "arm": "compiled", "wall_s": 43.10221470899705, - "success": true, + "success": false, "replayer_success": false, "heal_count": 1, "actions": 18, @@ -464,8 +487,10 @@ "cache_read_input_tokens": 0, "cost_usd": 0.0, "error": null, - "matched_ratio": 1.0, - "longest_run": 47, + "matched_ratio": 0.0, + "longest_run": 0, + "legacy_screen_success": true, + "oracle_correction": "The legacy whole-frame OCR verifier matched the note in the unsaved entry textarea. The retained final frame has no saved row for this note.", "i": 19, "note": "Orthopedic pillow suggestion discussed for neck pain." } diff --git a/claims.yaml b/claims.yaml index f784d818..9a0e3755 100644 --- a/claims.yaml +++ b/claims.yaml @@ -630,7 +630,7 @@ claims: # -------------------------------- OpenEMR head-to-head (field, not CI-reprod.) - id: openemr-field-benchmark claim: >- - On the real third-party OpenEMR public demo, compiled replay went 20/20 + On the real third-party OpenEMR public demo, compiled replay went 19/20 versus 10/10 for a computer-use agent, faster and with zero model calls. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build. surfaces: [README.md, docs, website] @@ -639,19 +639,29 @@ claims: evidence: - path: tests/test_openemr_benchmark.py proves: >- - The CI-reproducible half: the note-saved verifier and the - intent-not-coordinates task prompt that both benchmark arms use. + The CI-reproducible half: the saved-message-row verifier, its + unsaved-entry-form rejection, and the intent-not-coordinates task + prompt that both benchmark arms use. + - path: tests/test_openemr_retained_finals.py + kind: test + proves: >- + When the local-only retained final frames are mounted, the guard + replays all 30 with the current verifier and requires 19/20 compiled, + 10/10 agent, and exactly one corrected legacy false success. - path: benchmark/openemr/BENCHMARK.md kind: benchmark proves: >- - The field-test numbers, methodology, and cost caps for the 20-vs-10 - head-to-head run. + The corrected field-test numbers, saved-row screen-oracle contract, + methodology, and cost caps for the 20-vs-10 head-to-head run. - path: docs/showcase-openemr/FINDINGS.md kind: doc proves: >- The correctness-only field findings (fresh browsers, zero model calls, closed-loop scrolling). caveats: + - >- + The historical result uses OCR evidence from a visible saved message + row. It does not use an out-of-band OpenEMR system-of-record read. - >- FIELD TEST, NOT CI-reproducible: the head-to-head ran against a SHARED public demo that other users mutate and that resets daily; the sample diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 496a09a0..27602991 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -309,7 +309,7 @@ ### `openemr-field-benchmark` — validating — opt-in / infra-gated or field test -> On the real third-party OpenEMR public demo, compiled replay went 20/20 versus 10/10 for a computer-use agent, faster and with zero model calls. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build. +> On the real third-party OpenEMR public demo, compiled replay went 19/20 versus 10/10 for a computer-use agent, faster and with zero model calls. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build. - Reproducibility: **field** - Surfaces: README.md, docs, website @@ -317,11 +317,13 @@ | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| -| `tests/test_openemr_benchmark.py` | test | ci (required PR gate (test)) | supported | The CI-reproducible half: the note-saved verifier and the intent-not-coordinates task prompt that both benchmark arms use. | -| `benchmark/openemr/BENCHMARK.md` | benchmark | artifact (doc/benchmark) | roadmap | The field-test numbers, methodology, and cost caps for the 20-vs-10 head-to-head run. | +| `tests/test_openemr_benchmark.py` | test | ci (required PR gate (test)) | supported | The CI-reproducible half: the saved-message-row verifier, its unsaved-entry-form rejection, and the intent-not-coordinates task prompt that both benchmark arms use. | +| `tests/test_openemr_retained_finals.py` | test | opt-in (OPENADAPT_OPENEMR_FINALS_DIR) | validating | When the local-only retained final frames are mounted, the guard replays all 30 with the current verifier and requires 19/20 compiled, 10/10 agent, and exactly one corrected legacy false success. | +| `benchmark/openemr/BENCHMARK.md` | benchmark | artifact (doc/benchmark) | roadmap | The corrected field-test numbers, saved-row screen-oracle contract, methodology, and cost caps for the 20-vs-10 head-to-head run. | | `docs/showcase-openemr/FINDINGS.md` | doc | artifact (doc/benchmark) | roadmap | The correctness-only field findings (fresh browsers, zero model calls, closed-loop scrolling). | **Caveats (honest limits):** +- The historical result uses OCR evidence from a visible saved message row. It does not use an out-of-band OpenEMR system-of-record read. - FIELD TEST, NOT CI-reproducible: the head-to-head ran against a SHARED public demo that other users mutate and that resets daily; the sample is small (10 agent runs). Only the verifier + task-prompt units run in CI. `reproducibility: field` forbids ever labeling this `supported`. diff --git a/docs/verification.json b/docs/verification.json index f1931eda..d369ed25 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -927,7 +927,7 @@ }, { "id": "openemr-field-benchmark", - "claim": "On the real third-party OpenEMR public demo, compiled replay went 20/20 versus 10/10 for a computer-use agent, faster and with zero model calls. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build.", + "claim": "On the real third-party OpenEMR public demo, compiled replay went 19/20 versus 10/10 for a computer-use agent, faster and with zero model calls. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build.", "tier": "validating", "reproducibility": "field", "surfaces": [ @@ -937,6 +937,7 @@ ], "strongest_evidence": "supported", "caveats": [ + "The historical result uses OCR evidence from a visible saved message row. It does not use an out-of-band OpenEMR system-of-record read.", "FIELD TEST, NOT CI-reproducible: the head-to-head ran against a SHARED public demo that other users mutate and that resets daily; the sample is small (10 agent runs). Only the verifier + task-prompt units run in CI. `reproducibility: field` forbids ever labeling this `supported`." ], "evidence": [ @@ -949,7 +950,18 @@ "node": null, "node_found": null, "junit_status": null, - "proves": "The CI-reproducible half: the note-saved verifier and the intent-not-coordinates task prompt that both benchmark arms use." + "proves": "The CI-reproducible half: the saved-message-row verifier, its unsaved-entry-form rejection, and the intent-not-coordinates task prompt that both benchmark arms use." + }, + { + "path": "tests/test_openemr_retained_finals.py", + "kind": "test", + "exists": true, + "strength": "validating", + "gating": "opt-in (OPENADAPT_OPENEMR_FINALS_DIR)", + "node": null, + "node_found": null, + "junit_status": null, + "proves": "When the local-only retained final frames are mounted, the guard replays all 30 with the current verifier and requires 19/20 compiled, 10/10 agent, and exactly one corrected legacy false success." }, { "path": "benchmark/openemr/BENCHMARK.md", @@ -960,7 +972,7 @@ "node": null, "node_found": null, "junit_status": null, - "proves": "The field-test numbers, methodology, and cost caps for the 20-vs-10 head-to-head run." + "proves": "The corrected field-test numbers, saved-row screen-oracle contract, methodology, and cost caps for the 20-vs-10 head-to-head run." }, { "path": "docs/showcase-openemr/FINDINGS.md", diff --git a/openadapt_flow/benchmark/openemr_benchmark.py b/openadapt_flow/benchmark/openemr_benchmark.py index 440b954a..f2482f23 100644 --- a/openadapt_flow/benchmark/openemr_benchmark.py +++ b/openadapt_flow/benchmark/openemr_benchmark.py @@ -185,6 +185,16 @@ def aggregate_openemr_results( "the chart, scroll the dashboard to the Messages card, open " "Patient Messages, add a parameterized note, save" ), + "success_contract": { + "kind": "screen_saved_message_row", + "note_min_contiguous_squashed_chars": 16, + "requires": [ + "Patient Messages Content and Status headers", + "note fragment in the Content column below the headers", + "an aligned New status for the same saved row", + ], + "rejects": "note text visible only in the unsaved entry form", + }, "target": DEMO_URL, "workflow_steps": 18, "model": agent_baseline.MODEL, @@ -227,6 +237,29 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: total_cap = caps.get("total", MAX_TOTAL_COST_USD) note = results.get("agent_arm_note") note_block = f"\n> **Agent arm disclosure:** {note}\n" if note else "" + flow = results.get("flow") or {} + artifact = results.get("artifact_provenance") or {} + flow_version = flow.get("flow_version") + engine_line = ( + f" Engine: a pre-`v0.2.0` source checkout declaring\n" + f"openadapt-flow {flow_version}." + if flow_version + else "" + ) + provenance_block = ( + f"**Measured on Flow {flow_version}, {date}.** The measurement used a " + "pre-`v0.2.0`\n" + "development source checkout; its exact runtime HEAD was not retained. " + "The rows\n" + "were first committed in " + f"`{str(artifact.get('first_committed_in', 'unknown'))[:8]}` after parent " + f"`{str(artifact.get('parent_before_artifact_commit', 'unknown'))[:8]}`; " + "those two SHAs\n" + "describe artifact history, not the runtime used for the measurement. " + "Not\nre-measured on a later release.\n" + if flow_version and artifact + else "" + ) agent_errors = [r for r in results["runs"]["agent"] if not r["success"]] failure_lines = ( "".join( @@ -243,16 +276,23 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: compiled_failures = [r for r in results["runs"]["compiled"] if not r["success"]] compiled_failure_lines = ( "".join( - f"- compiled run {r['i'] + 1}: " - + ( - r["error"] - if r.get("error") - else f"{r.get('actions', '?')} steps executed, " - f"replayer_success={r.get('replayer_success')}, " - f"first_failure={r.get('first_failure')}, " - f"OCR matched {r.get('matched_ratio', 0):.0%} of the note" + ( + f"- compiled run {r['i'] + 1}: saved-row oracle failed; " + f"replayer halted at " + f"{(r.get('first_failure') or {}).get('step', '?')} — " + f"{r['oracle_correction']}\n" + if r.get("oracle_correction") + else f"- compiled run {r['i'] + 1}: " + + ( + r["error"] + if r.get("error") + else f"{r.get('actions', '?')} steps executed, " + f"replayer_success={r.get('replayer_success')}, " + f"first_failure={r.get('first_failure')}, " + f"OCR matched {r.get('matched_ratio', 0):.0%} of the note" + ) + + "\n" ) - + "\n" for r in compiled_failures ) or "- none\n" @@ -283,7 +323,7 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: ) return f"""# Benchmark: compiled replay vs. computer-use agent — OpenEMR (real app) -Date: {date}. Same head-to-head as the [MockMed benchmark](../BENCHMARK.md), +Date: {date}.{engine_line} Same head-to-head as the [MockMed benchmark](../BENCHMARK.md), run against a real third-party application: the official OpenEMR public demo (`{results["target"]}`, fake patients only, instance resets daily). One task, two ways to automate it, one success check. @@ -311,6 +351,7 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: {a["cache_creation_input_tokens_total"]:,} / \ {a["cache_read_input_tokens_total"]:,} | {note_block} +{provenance_block} Failed runs, reported honestly: Compiled arm: @@ -348,15 +389,17 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: target patient, the exact note text — not steps or coordinates. Every executed action returns a settled screenshot. - **Same success criterion, implemented once.** After each run, the final - screenshot is checked by `verify_note_saved` (OCR): a contiguous run of - at least 16 characters of the run's note must appear in the frame's - OCR text (whitespace-squashed; retried at 2x resolution when the raw - frame does not pass, because rapidocr drops dense table lines at - 1280x800). Neither arm's self-reported success is used. + screenshot is checked by `verify_note_saved` (OCR). A contiguous run of + at least 16 characters of the run's note must appear in a saved Patient + Messages `Content` row aligned with its `New` status. The same note in the + unsaved entry form does not pass. OCR is whitespace-squashed and retried at + 2x resolution when the raw frame does not pass. Neither arm's self-reported + success is used. This is screen-row evidence, not an out-of-band read of the + OpenEMR system of record. - **Distinct, mutually dissimilar note per run in BOTH arms** (no two notes share a 16-character squashed substring — unit-tested), so - success proves parameter substitution against live state and one run's - note cannot satisfy another run's check. + a saved-row success proves parameter substitution in the visible message + list and one run's note cannot satisfy another run's check. - **Pacing.** Runs are spaced ~{results.get("pace_s", 30):.0f}s apart as public-demo courtesy; the pacing gap is excluded from latency. - **Latency** is wall-clock around the replay / agent loop only. @@ -405,15 +448,13 @@ def render_openemr_markdown(results: dict[str, Any]) -> str: - **The compiled arm needs a demonstration first.** The one-time record + compile step (about a minute of human demonstration) is the price of the fast replays; the agent needs only the prompt. -- **OCR verification on dense EMR text under-counts.** rapidocr sometimes - drops the exact table line containing the note (a known limitation - documented in - [docs/showcase-openemr/FINDINGS.md](../../docs/showcase-openemr/FINDINGS.md)), - so a "failed" verification can be a measurement miss with the note - plainly visible in the final screenshot. The check errs conservative - and is identical for both arms. Every run's final screenshot is saved - to `benchmark/openemr/finals/` (local only, not committed) so failed - verdicts can be audited against what was actually on screen. +- **The success oracle is bounded screen evidence.** rapidocr can drop dense + table lines and cause an over-halt. The legacy whole-frame check could also + accept note text in the unsaved form; retained compiled run 20 exposed that + false success. The corrected check requires saved-row context. All 30 final + frames were replayed under it: only compiled run 20 changed. Every final + screenshot stays in `benchmark/openemr/finals/` (local only, not committed) + for audit. This check does not read the OpenEMR system of record. - Single machine ({results["platform"]}). ## Reproduce diff --git a/openadapt_flow/benchmark/verify.py b/openadapt_flow/benchmark/verify.py index 36eccf3d..980785a7 100644 --- a/openadapt_flow/benchmark/verify.py +++ b/openadapt_flow/benchmark/verify.py @@ -211,11 +211,9 @@ class NoteVerifyResult(BaseModel): """Outcome of the OpenEMR saved-note check. Attributes: - success: True iff the note evidence was found on the final screen. + success: True iff the note evidence was found in a saved message row. matched_ratio: Fraction of the note's squashed characters that OCR - matched somewhere in the frame's squashed text (diagnostic - only — non-contiguous matches accumulate from unrelated text - on a dense screen, so this never decides success). + matched in the best eligible saved-row OCR line (diagnostic only). longest_run: Longest contiguous matched character run (this is the criterion). """ diff --git a/paper/sections/05_results.tex b/paper/sections/05_results.tex index 3d6de204..3f72ae8d 100644 --- a/paper/sections/05_results.tex +++ b/paper/sections/05_results.tex @@ -200,10 +200,10 @@ \section{Results} \paragraph{Compiled replay removes per-run reasoning on demonstrated tasks.} Table~\ref{tab:comparison} shows that, on these two already-demonstrated tasks, -compiled replay completed every measured run with lower median latency and no -model calls. The unequal, small samples do not establish superior reliability. -The correct inference is narrower: repeated reasoning was unnecessary for the -healthy executions measured here. +compiled replay completed 19 of 20 OpenEMR runs and every measured MockMed run +with lower median latency and no model calls. The unequal, small samples do not +establish superior reliability. The correct inference is narrower: repeated +reasoning was unnecessary for the healthy executions measured here. \begin{table}[t] \centering @@ -215,7 +215,7 @@ \section{Results} \toprule Task & Arm & Success & $n$ & Median (s) & Model cost/run \\ \midrule -OpenEMR & Compiled & 20/20 & 20 & 39.2 & \$0.00 \\ +OpenEMR & Compiled & 19/20 & 20 & 39.2 & \$0.00 \\ OpenEMR & Agent & 10/10 & 10 & 70.4 & \$0.55 \\ MockMed & Compiled & 100/100 & 100 & 4.9 & \$0.00 \\ MockMed & Agent & 20/20 & 20 & 37.5 & \$0.27 \\ @@ -224,10 +224,10 @@ \section{Results} \end{table} \paragraph{Repair under drift: the compiler heals what breaks the agent's day.} -The comparison above is the healthy path. To exercise the ``govern every -repair'' half of the thesis, we forced the MockMed interface to re-render with a -dark palette, invalidating every recorded template crop. As a single observation -per arm, compiled replay self-healed in 9.7\,s with 8 target repairs and zero +The comparison above uses nominal interface conditions. To exercise the +``govern every repair'' half of the thesis, we forced the MockMed interface to +re-render with a dark palette, invalidating every recorded template crop. As a +single observation per arm, compiled replay self-healed in 9.7\,s with 8 target repairs and zero model calls, while the same computer-use agent under the same drift took 87.4\,s and \$0.63 across 24 model calls. One run is not a reliability claim, but it shows the mechanism working end to end: the resolution ladder re-resolved changed diff --git a/tests/test_comparison_artifact.py b/tests/test_comparison_artifact.py index 9ec29732..78c7b2c9 100644 --- a/tests/test_comparison_artifact.py +++ b/tests/test_comparison_artifact.py @@ -42,7 +42,8 @@ def test_openemr_figures_match_source_exactly() -> None: # The lead result's shape: compiled is free, agent is not. assert b.compiled.cost_per_run == 0.0 assert b.agent.cost_per_run > 0.0 - assert b.compiled.success_count == b.compiled.n == 20 + assert b.compiled.n == 20 + assert b.compiled.success_count == 19 assert b.agent.success_count == b.agent.n == 10 @@ -103,7 +104,7 @@ def test_build_emits_html_and_json_with_real_figures(tmp_path) -> None: # The real OpenEMR headline figures must appear verbatim in the page. for needle in ( - f"{oe.compiled.success_count}/{oe.compiled.n}", # 20/20 + f"{oe.compiled.success_count}/{oe.compiled.n}", # 19/20 f"{oe.agent.success_count}/{oe.agent.n}", # 10/10 gen.fmt_s(oe.compiled.p50_s), # 39.2 s gen.fmt_s(oe.agent.p50_s), # 70.4 s diff --git a/tests/test_openemr_benchmark.py b/tests/test_openemr_benchmark.py index f7407fe2..6a7bcd1e 100644 --- a/tests/test_openemr_benchmark.py +++ b/tests/test_openemr_benchmark.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace from typing import Any @@ -412,6 +413,22 @@ def test_aggregates(self) -> None: assert a["cost_usd_total"] == 5.0 assert results["model"] == agent_baseline.MODEL assert results["target"].startswith("https://demo.openemr.io") + assert results["success_contract"]["kind"] == "screen_saved_message_row" + + def test_committed_result_preserves_oracle_adjudication(self) -> None: + path = Path(__file__).resolve().parents[1] / "benchmark/openemr/results.json" + results = json.loads(path.read_text()) + compiled = results["arms"]["compiled"] + agent = results["arms"]["agent"] + assert (compiled["success_count"], compiled["n"]) == (19, 20) + assert (agent["success_count"], agent["n"]) == (10, 10) + assert compiled["wall_s_p50"] == 39.170917561999886 + assert agent["wall_s_p50"] == 70.44607112499943 + assert results["oracle_adjudication"]["denominators_changed"] is False + corrected = results["runs"]["compiled"][19] + assert corrected["success"] is False + assert corrected["legacy_screen_success"] is True + assert corrected["i"] == 19 def test_markdown_names_anchor_and_caveats(self) -> None: md = render_openemr_markdown(self.make_results()) @@ -443,7 +460,7 @@ def test_markdown_discloses_compiled_self_flags(self) -> None: results = aggregate_openemr_results(compiled, [agent_row(i) for i in range(10)]) results["pace_s"] = 30.0 md = render_openemr_markdown(results) - assert "100% (20/20)" in md # headline unchanged + assert f"100% ({len(compiled)}/{len(compiled)})" in md assert "self-flagged" in md assert "compiled run 20" in md assert "step_017" in md @@ -453,8 +470,6 @@ def test_markdown_discloses_compiled_self_flags(self) -> None: assert "self-flagged" not in render_openemr_markdown(self.make_results()) def test_write_outputs(self, tmp_path: Path) -> None: - import json - write_openemr_outputs(self.make_results(), tmp_path) loaded = json.loads((tmp_path / "results.json").read_text()) assert loaded["arms"]["agent"]["success_count"] == 8 diff --git a/tests/test_openemr_retained_finals.py b/tests/test_openemr_retained_finals.py new file mode 100644 index 00000000..af7a59bf --- /dev/null +++ b/tests/test_openemr_retained_finals.py @@ -0,0 +1,70 @@ +"""Replay the local-only retained OpenEMR benchmark final frames. + +The frames contain synthetic public-demo data and stay outside the repository. +Set ``OPENADAPT_OPENEMR_FINALS_DIR`` when they are mounted somewhere other than +``benchmark/openemr/finals``. A checkout without the retained frames skips this +field-evidence guard; a checkout with them must supply the complete 30-frame +set and reproduce the corrected 19/20 compiled and 10/10 agent result. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from openadapt_flow.benchmark.verify import verify_note_saved + +REPO_ROOT = Path(__file__).resolve().parents[1] +RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") +FINALS_DIR = Path( + RUN_RETAINED_FINALS or REPO_ROOT / "benchmark" / "openemr" / "finals" +) +RESULTS_PATH = REPO_ROOT / "benchmark" / "openemr" / "results.json" +RETAINED_FALSE_SUCCESS_SHA256 = ( + "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f" +) +pytestmark = pytest.mark.skipif( + not RUN_RETAINED_FINALS, + reason="set OPENADAPT_OPENEMR_FINALS_DIR to replay retained final frames", +) + + +def test_retained_final_frames_reproduce_corrected_result() -> None: + results = json.loads(RESULTS_PATH.read_text()) + rows = [ + row + for arm in ("compiled", "agent") + for row in results["runs"][arm] + ] + assert len(results["runs"]["compiled"]) == 20 + assert len(results["runs"]["agent"]) == 10 + assert len(rows) == 30 + + retained_false_success = FINALS_DIR / "compiled_019.png" + assert hashlib.sha256(retained_false_success.read_bytes()).hexdigest() == ( + RETAINED_FALSE_SUCCESS_SHA256 + ) + + replayed: dict[str, list[bool]] = {"compiled": [], "agent": []} + changed_from_legacy: list[tuple[str, int]] = [] + for row in rows: + frame = FINALS_DIR / f"{row['arm']}_{row['i']:03d}.png" + assert frame.is_file(), f"missing retained final frame: {frame.name}" + verdict = verify_note_saved(frame.read_bytes(), row["note"]) + replayed[row["arm"]].append(verdict.success) + assert verdict.success is row["success"], (row["arm"], row["i"], verdict) + if row.get("legacy_screen_success") is not None: + assert row["legacy_screen_success"] is not verdict.success + changed_from_legacy.append((row["arm"], row["i"])) + + assert sum(replayed["compiled"]) == 19 + assert sum(replayed["agent"]) == 10 + assert changed_from_legacy == [("compiled", 19)] + assert results["arms"]["compiled"]["n"] == 20 + assert results["arms"]["compiled"]["success_count"] == 19 + assert results["arms"]["agent"]["n"] == 10 + assert results["arms"]["agent"]["success_count"] == 10 From e66eb1d079b36d846a08252c616f730ba818f936 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 03:54:20 -0400 Subject: [PATCH 5/8] chore: refresh public artifact inventory --- public-artifacts.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public-artifacts.json b/public-artifacts.json index adfd6548..4a272cf1 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -157,11 +157,11 @@ }, { "path": "benchmark/comparison_artifact/comparison.html", - "sha256": "750221fc2301b360c9ec22d1c2f78640371b4ee72fd0a8794bb087f77d553489" + "sha256": "dcf4ca2df1b06c222341c16e968b0abe18b71a01868431ec8dfb23c9b504ccbf" }, { "path": "benchmark/comparison_artifact/comparison.json", - "sha256": "e9e4f64c4cd699a7d0e6889b8610e9d1bc1afa430a372786cb40b12309a675ba" + "sha256": "c5eb58d234ecbd5b03a687745fb6afeb53d0965459b0e42779942441b19dbbd6" }, { "path": "benchmark/cosmetic_drift/results.json", @@ -337,7 +337,7 @@ }, { "path": "benchmark/openemr/results.json", - "sha256": "16e02e4ad045c3cb1e4bc2755268bebb2d5e1a8832c5da9e2b03019e9b96fb0c" + "sha256": "30de6dbb43ad0db4fd8b55737d765d06747b896ffae04522b6957d567dcfd89c" }, { "path": "benchmark/openemr_live/docker-compose.yml", @@ -589,7 +589,7 @@ }, { "path": "claims.yaml", - "sha256": "c62ac4d9a816e367f044dba4a9250cc0db2f310b61d80669433c78ee7aa545f9" + "sha256": "1a185fc7d9c377a3ca984d9144089057532d1c9a6536e4ca8d6f4f617ad498b4" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1797,7 +1797,7 @@ }, { "path": "docs/verification.json", - "sha256": "c962bd5e9e302307209ec7d2c276348787e59840c28b631d5833adc703537b0b" + "sha256": "520cf27d3a531092579b2e456ed61d449a6173f3ca52d0f4fd91763a068f8f2f" }, { "path": "openadapt_flow/console/static/console.css", From db9684d70969bd73add80f9ae4498b9b72cd439c Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 03:55:59 -0400 Subject: [PATCH 6/8] style: format benchmark oracle changes --- openadapt_flow/benchmark/verify.py | 13 +++---------- tests/test_openemr_benchmark.py | 4 +--- tests/test_openemr_retained_finals.py | 10 ++-------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/openadapt_flow/benchmark/verify.py b/openadapt_flow/benchmark/verify.py index 980785a7..da85d41c 100644 --- a/openadapt_flow/benchmark/verify.py +++ b/openadapt_flow/benchmark/verify.py @@ -283,14 +283,8 @@ def _saved_message_note_lines(lines: list[OcrLine]) -> list[OcrLine]: for lx, ly, lw, lh in (line.region,) if ly > header_bottom and ( - ( - _squash(line.text) == "new" - and lx + lw / 2 >= sx - sw - ) - or ( - _squash(line.text).endswith("new") - and lx + lw >= sx - ) + (_squash(line.text) == "new" and lx + lw / 2 >= sx - sw) + or (_squash(line.text).endswith("new") and lx + lw >= sx) ) ] for line in lines: @@ -302,8 +296,7 @@ def _saved_message_note_lines(lines: list[OcrLine]) -> list[OcrLine]: if not (content_center_x - cw <= line_center_x < status_center_x): continue if any( - abs(line_center_y - row_status_center_y) - <= 2 * max(lh, status_h) + abs(line_center_y - row_status_center_y) <= 2 * max(lh, status_h) for row_status_center_y, status_h in row_status_bands ): candidates.append(line) diff --git a/tests/test_openemr_benchmark.py b/tests/test_openemr_benchmark.py index 6a7bcd1e..be815f11 100644 --- a/tests/test_openemr_benchmark.py +++ b/tests/test_openemr_benchmark.py @@ -153,9 +153,7 @@ def test_note_in_unsaved_entry_form_fails(self, monkeypatch: Any) -> None: region=(420, 480, 240, 20), confidence=1.0, ), - OcrLine( - text="New", region=(900, 480, 40, 20), confidence=1.0 - ), + OcrLine(text="New", region=(900, 480, 40, 20), confidence=1.0), ), ], ) diff --git a/tests/test_openemr_retained_finals.py b/tests/test_openemr_retained_finals.py index af7a59bf..8fbdc229 100644 --- a/tests/test_openemr_retained_finals.py +++ b/tests/test_openemr_retained_finals.py @@ -20,9 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] RUN_RETAINED_FINALS = os.environ.get("OPENADAPT_OPENEMR_FINALS_DIR") -FINALS_DIR = Path( - RUN_RETAINED_FINALS or REPO_ROOT / "benchmark" / "openemr" / "finals" -) +FINALS_DIR = Path(RUN_RETAINED_FINALS or REPO_ROOT / "benchmark" / "openemr" / "finals") RESULTS_PATH = REPO_ROOT / "benchmark" / "openemr" / "results.json" RETAINED_FALSE_SUCCESS_SHA256 = ( "8c504ba15bab9cdca8b5987dd1d1ab7b0ba7ae77f67fac5e93ba8481492ae18f" @@ -35,11 +33,7 @@ def test_retained_final_frames_reproduce_corrected_result() -> None: results = json.loads(RESULTS_PATH.read_text()) - rows = [ - row - for arm in ("compiled", "agent") - for row in results["runs"][arm] - ] + rows = [row for arm in ("compiled", "agent") for row in results["runs"][arm]] assert len(results["runs"]["compiled"]) == 20 assert len(results["runs"]["agent"]) == 10 assert len(rows) == 30 From 25fb36edbf55d9883d37eb18b36be1ae35e3886f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 04:16:59 -0400 Subject: [PATCH 7/8] fix(paper): bind the corrected OpenEMR count --- paper/check_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper/check_artifacts.py b/paper/check_artifacts.py index cf8048bb..39eec6d7 100644 --- a/paper/check_artifacts.py +++ b/paper/check_artifacts.py @@ -77,7 +77,7 @@ def main() -> None: ) require_equal(openemr["compiled"]["n"], 20, "OpenEMR compiled n") - require_equal(openemr["compiled"]["success_count"], 20, "OpenEMR compiled success") + require_equal(openemr["compiled"]["success_count"], 19, "OpenEMR compiled success") require_close(openemr["compiled"]["wall_s_p50"], 39.2, "OpenEMR compiled p50") require_equal(openemr["agent"]["n"], 10, "OpenEMR agent n") require_equal(openemr["agent"]["success_count"], 10, "OpenEMR agent success") From 2d9c73607508001a6ccf6928b0363467ee8b302b Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 28 Jul 2026 11:54:00 -0400 Subject: [PATCH 8/8] fix(benchmark): refuse ambiguous bare note rows --- openadapt_flow/benchmark/verify.py | 21 +++++++++++++++++- tests/test_benchmark.py | 35 +++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/openadapt_flow/benchmark/verify.py b/openadapt_flow/benchmark/verify.py index da85d41c..099f6e68 100644 --- a/openadapt_flow/benchmark/verify.py +++ b/openadapt_flow/benchmark/verify.py @@ -110,6 +110,12 @@ def _row_type( whose note is absent, yields ``None``. """ expected_note = normalize_text(note_text[:ROW_NOTE_CHARS]) + normalized_line = normalize_text(line).lstrip(_ROW_DECORATION) + bare_note_ratio = difflib.SequenceMatcher( + None, + normalized_line, + expected_note, + ).ratio() for type_field, note_field in _split_row_line(line, known_types): scores = { known: difflib.SequenceMatcher( @@ -122,7 +128,13 @@ def _row_type( if scores[best] < min_ratio or scores[best] <= runner_up: continue note_ratio = difflib.SequenceMatcher(None, note_field, expected_note).ratio() - if note_ratio >= min_ratio: + # A banner can segment its bare note into a separate OCR line. When a + # note itself begins with a known type, positional splitting can make + # that line look like ```` even though no row exists. The + # row hypothesis must therefore explain the line better than the bare + # note hypothesis. A real row includes an additional type prefix, so + # its parsed note field wins; an isolated note line does not. + if note_ratio >= min_ratio and note_ratio > bare_note_ratio: return best return None @@ -179,6 +191,13 @@ def verify_encounter_saved( A :class:`VerifyResult`; ``success`` requires the banner, a correctly-typed row, and no wrong-type row. """ + if not normalize_text(note_text): + return VerifyResult( + success=False, + banner_found=False, + note_found=False, + wrong_type_row=False, + ) if encounter_type not in known_types: known_types = (*known_types, encounter_type) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 21288c73..6f903b79 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -17,7 +17,6 @@ from openadapt_flow.benchmark import agent_baseline from openadapt_flow.benchmark.agent_baseline import ( - AgentRunResult, _truncate_screenshots, compute_cost, load_api_key, @@ -575,6 +574,40 @@ def test_bare_note_line_is_not_an_encounter_row(self) -> None: assert not verdict.note_found assert not verdict.success + def test_empty_note_contract_never_verifies(self) -> None: + screen = self.make_screen("Encounter saved -", "Triage -") + verdict = verify_encounter_saved(screen, "") + assert not verdict.banner_found + assert not verdict.note_found + assert not verdict.wrong_type_row + assert not verdict.success + + @pytest.mark.parametrize( + "note", + [ + "Triage follow-up remains stable after medication review.", + "Consult - follow-up remains stable after medication review.", + ], + ) + def test_bare_note_starting_with_known_type_is_not_a_row( + self, + note: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Positional parsing tolerates a dropped row separator. It must not + # reinterpret a banner's isolated note line as that row merely because + # the note itself starts with a known categorical value. + screen = self.make_screen(f"Encounter saved - {note[:40]}", note[:60]) + monkeypatch.setattr( + "openadapt_flow.benchmark.verify.ocr", + lambda _screen: [SimpleNamespace(text=note[:60])], + ) + verdict = verify_encounter_saved(screen, note) + assert verdict.banner_found + assert not verdict.note_found + assert not verdict.wrong_type_row + assert not verdict.success + def test_unlisted_requested_type_is_still_checked(self) -> None: # A caller may ask for a type outside the default enumeration; it # is added to the choice set rather than silently unmatchable.