fix: preserve benchmark agent failures

openadapt_ml/benchmarks/agent.py

@@ -325,7 +325,7 @@ def act(
         try:
             response = adapter.generate(sample, max_new_tokens=self.max_tokens)
         except Exception as e:
-            return BenchmarkAction(type="done", raw_action={"error": str(e)})
+            return BenchmarkAction(type="error", raw_action={"error": str(e)})
 
         return self._parse_response(response, observation)
 
@@ -451,7 +451,7 @@ def _parse_response(
 
         if not action_line:
             raw_action["parse_error"] = "No action pattern found"
-            return BenchmarkAction(type="done", raw_action=raw_action)
+            return BenchmarkAction(type="error", raw_action=raw_action)
 
         # Parse CLICK([id])
         click_match = re.match(
@@ -557,7 +557,7 @@ def _parse_response(
             )
 
         raw_action["parse_error"] = f"Unknown action format: {action_line}"
-        return BenchmarkAction(type="done", raw_action=raw_action)
+        return BenchmarkAction(type="error", raw_action=raw_action)
 
     def reset(self) -> None:
         """Reset agent state."""
@@ -661,7 +661,7 @@ def act(
         except Exception as e:
             if self.verbose:
                 print(f"[UnifiedBaselineAgent] Adapter error: {e}")
-            return BenchmarkAction(type="done", raw_action={"error": str(e)})
+            return BenchmarkAction(type="error", raw_action={"error": str(e)})
 
         return self._parsed_to_benchmark_action(parsed_action, observation)
 
@@ -693,7 +693,7 @@ def _parsed_to_benchmark_action(
 
         if not parsed_action.is_valid:
             raw_action["parse_error"] = parsed_action.parse_error
-            return BenchmarkAction(type="done", raw_action=raw_action)
+            return BenchmarkAction(type="error", raw_action=raw_action)
 
         action_type = parsed_action.action_type
 
@@ -743,7 +743,7 @@ def _parsed_to_benchmark_action(
             )
 
         raw_action["unknown_action"] = action_type
-        return BenchmarkAction(type="done", raw_action=raw_action)
+        return BenchmarkAction(type="error", raw_action=raw_action)
 
     def reset(self) -> None:
         """Reset agent state."""

openadapt_ml/experiments/waa_demo/runner.py

@@ -318,7 +318,7 @@ def act(
         except Exception as e:
             logger.error(f"API error: {e}")
             return BenchmarkAction(
-                type="done",
+                type="error",
                 raw_action={"error": str(e)},
             )
 
@@ -473,7 +473,7 @@ def _parse_response(
 
         if not action_line:
             raw_action["parse_error"] = "No action pattern found"
-            return BenchmarkAction(type="done", raw_action=raw_action)
+            return BenchmarkAction(type="error", raw_action=raw_action)
 
         # Parse CLICK with element ID
         click_id_match = re.match(
@@ -538,7 +538,7 @@ def _parse_response(
             return BenchmarkAction(type="done", raw_action=raw_action)
 
         raw_action["parse_error"] = f"Unknown action format: {action_line}"
-        return BenchmarkAction(type="done", raw_action=raw_action)
+        return BenchmarkAction(type="error", raw_action=raw_action)
 
     def reset(self) -> None:
         """Reset agent state between episodes."""

tests/test_agent_failure_actions.py

@@ -0,0 +1,53 @@
+"""Agent infrastructure and parse failures must not look like task completion."""
+
+from __future__ import annotations
+
+from openadapt_types import BenchmarkObservation, BenchmarkTask
+
+from openadapt_ml.baselines import ParsedAction
+from openadapt_ml.benchmarks.agent import APIBenchmarkAgent, UnifiedBaselineAgent
+from openadapt_ml.experiments.waa_demo.runner import DemoConditionedAgent
+
+
+class _FailingAdapter:
+    def generate(self, *args, **kwargs):
+        raise RuntimeError("provider unavailable")
+
+    def predict(self, *args, **kwargs):
+        raise RuntimeError("provider unavailable")
+
+
+def _observation_and_task():
+    return BenchmarkObservation(), BenchmarkTask(
+        task_id="task-1", instruction="Complete the task", domain="desktop"
+    )
+
+
+def test_api_agent_preserves_provider_and_parse_failures():
+    observation, task = _observation_and_task()
+    agent = APIBenchmarkAgent()
+    agent._adapter = _FailingAdapter()
+
+    assert agent.act(observation, task).type == "error"
+    assert agent._parse_response("not an action").type == "error"
+    assert agent._parse_response("ACTION: UNKNOWN(foo)").type == "error"
+
+
+def test_unified_agent_preserves_provider_and_parse_failures():
+    observation, task = _observation_and_task()
+    agent = UnifiedBaselineAgent()
+    agent._adapter = _FailingAdapter()
+
+    assert agent.act(observation, task).type == "error"
+    invalid = ParsedAction(action_type="unknown", parse_error="not an action")
+    assert agent._parsed_to_benchmark_action(invalid).type == "error"
+
+
+def test_demo_agent_preserves_provider_and_parse_failures():
+    observation, task = _observation_and_task()
+    agent = DemoConditionedAgent()
+    agent._adapter = _FailingAdapter()
+
+    assert agent.act(observation, task).type == "error"
+    assert agent._parse_response("not an action").type == "error"
+    assert agent._parse_response("ACTION: UNKNOWN(foo)").type == "error"