From e61980a9817d4537e264a5942c610e590ef611ed Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 28 Jul 2026 04:04:19 -0400 Subject: [PATCH] fix: preserve benchmark agent failures --- openadapt_ml/benchmarks/agent.py | 12 ++--- openadapt_ml/experiments/waa_demo/runner.py | 6 +-- tests/test_agent_failure_actions.py | 53 +++++++++++++++++++++ 3 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 tests/test_agent_failure_actions.py diff --git a/openadapt_ml/benchmarks/agent.py b/openadapt_ml/benchmarks/agent.py index 3c42287..7d27985 100644 --- a/openadapt_ml/benchmarks/agent.py +++ b/openadapt_ml/benchmarks/agent.py @@ -325,7 +325,7 @@ def act( try: response = adapter.generate(sample, max_new_tokens=self.max_tokens) except Exception as e: - return BenchmarkAction(type="done", raw_action={"error": str(e)}) + return BenchmarkAction(type="error", raw_action={"error": str(e)}) return self._parse_response(response, observation) @@ -451,7 +451,7 @@ def _parse_response( if not action_line: raw_action["parse_error"] = "No action pattern found" - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) # Parse CLICK([id]) click_match = re.match( @@ -557,7 +557,7 @@ def _parse_response( ) raw_action["parse_error"] = f"Unknown action format: {action_line}" - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) def reset(self) -> None: """Reset agent state.""" @@ -661,7 +661,7 @@ def act( except Exception as e: if self.verbose: print(f"[UnifiedBaselineAgent] Adapter error: {e}") - return BenchmarkAction(type="done", raw_action={"error": str(e)}) + return BenchmarkAction(type="error", raw_action={"error": str(e)}) return self._parsed_to_benchmark_action(parsed_action, observation) @@ -693,7 +693,7 @@ def _parsed_to_benchmark_action( if not parsed_action.is_valid: raw_action["parse_error"] = parsed_action.parse_error - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) action_type = parsed_action.action_type @@ -743,7 +743,7 @@ def _parsed_to_benchmark_action( ) raw_action["unknown_action"] = action_type - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) def reset(self) -> None: """Reset agent state.""" diff --git a/openadapt_ml/experiments/waa_demo/runner.py b/openadapt_ml/experiments/waa_demo/runner.py index 93e1fbd..2d1791a 100644 --- a/openadapt_ml/experiments/waa_demo/runner.py +++ b/openadapt_ml/experiments/waa_demo/runner.py @@ -318,7 +318,7 @@ def act( except Exception as e: logger.error(f"API error: {e}") return BenchmarkAction( - type="done", + type="error", raw_action={"error": str(e)}, ) @@ -473,7 +473,7 @@ def _parse_response( if not action_line: raw_action["parse_error"] = "No action pattern found" - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) # Parse CLICK with element ID click_id_match = re.match( @@ -538,7 +538,7 @@ def _parse_response( return BenchmarkAction(type="done", raw_action=raw_action) raw_action["parse_error"] = f"Unknown action format: {action_line}" - return BenchmarkAction(type="done", raw_action=raw_action) + return BenchmarkAction(type="error", raw_action=raw_action) def reset(self) -> None: """Reset agent state between episodes.""" diff --git a/tests/test_agent_failure_actions.py b/tests/test_agent_failure_actions.py new file mode 100644 index 0000000..38d4ccc --- /dev/null +++ b/tests/test_agent_failure_actions.py @@ -0,0 +1,53 @@ +"""Agent infrastructure and parse failures must not look like task completion.""" + +from __future__ import annotations + +from openadapt_types import BenchmarkObservation, BenchmarkTask + +from openadapt_ml.baselines import ParsedAction +from openadapt_ml.benchmarks.agent import APIBenchmarkAgent, UnifiedBaselineAgent +from openadapt_ml.experiments.waa_demo.runner import DemoConditionedAgent + + +class _FailingAdapter: + def generate(self, *args, **kwargs): + raise RuntimeError("provider unavailable") + + def predict(self, *args, **kwargs): + raise RuntimeError("provider unavailable") + + +def _observation_and_task(): + return BenchmarkObservation(), BenchmarkTask( + task_id="task-1", instruction="Complete the task", domain="desktop" + ) + + +def test_api_agent_preserves_provider_and_parse_failures(): + observation, task = _observation_and_task() + agent = APIBenchmarkAgent() + agent._adapter = _FailingAdapter() + + assert agent.act(observation, task).type == "error" + assert agent._parse_response("not an action").type == "error" + assert agent._parse_response("ACTION: UNKNOWN(foo)").type == "error" + + +def test_unified_agent_preserves_provider_and_parse_failures(): + observation, task = _observation_and_task() + agent = UnifiedBaselineAgent() + agent._adapter = _FailingAdapter() + + assert agent.act(observation, task).type == "error" + invalid = ParsedAction(action_type="unknown", parse_error="not an action") + assert agent._parsed_to_benchmark_action(invalid).type == "error" + + +def test_demo_agent_preserves_provider_and_parse_failures(): + observation, task = _observation_and_task() + agent = DemoConditionedAgent() + agent._adapter = _FailingAdapter() + + assert agent.act(observation, task).type == "error" + assert agent._parse_response("not an action").type == "error" + assert agent._parse_response("ACTION: UNKNOWN(foo)").type == "error"