diff --git a/benchmarks/README.md b/benchmarks/README.md index e7b78c94..d9bd203b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -215,12 +215,17 @@ uv run python benchmarks/scripts/run_eval.py \ --tools dev-browser ``` +Use `--controller pi --model openai/gpt-5.6-sol` to run the same lane through +Pi. The Pi sidecar mounts the installed `dev-browser` skill and enables only +its `bash` and `read` tools. + `dev-browser install` is required because it installs the daemon's Playwright and QuickJS dependencies. It may also download dev-browser's Chromium, but the benchmark does not use that browser: the task-private shim always connects dev-browser to the task's dedicated CloakBrowser CDP endpoint. -For Libretto Browser Tools with Codex and one dedicated CloakBrowser per task: +For Libretto Browser Tools with Codex or Pi and one dedicated CloakBrowser per +task: ```bash npm ci @@ -234,10 +239,12 @@ uv run python benchmarks/scripts/run_eval.py \ --tools libretto ``` -Libretto is currently Codex-only. The harness injects its pinned stdio MCP -server per attempt and exposes `browser_open`, `browser_exec`, -`browser_snapshot`, `browser_status`, and `browser_close`. `browser_connect` -is disabled so the agent cannot leave the task's dedicated CloakBrowser. +With Codex, the harness injects its pinned stdio MCP server per attempt. With +Pi, it registers the equivalent native Pi custom tools directly; use +`--controller pi --model openai/gpt-5.6-sol`. Both expose only `browser_open`, +`browser_exec`, `browser_snapshot`, `browser_status`, and `browser_close`. +`browser_connect` is disabled so the agent cannot leave the task's dedicated +CloakBrowser. Use `--stealth-view official` only with `Stealth_Bench_V1`. Never publish `results/`. diff --git a/benchmarks/scripts/pi_controller.mjs b/benchmarks/scripts/pi_controller.mjs index 7ee37a8a..27888bd8 100644 --- a/benchmarks/scripts/pi_controller.mjs +++ b/benchmarks/scripts/pi_controller.mjs @@ -9,6 +9,8 @@ import { import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; +import { createPiToolConfiguration } from "./pi_toolkit.mjs"; + const SDK_PACKAGE = JSON.parse( readFileSync( new URL("../package.json", import.meta.resolve("@earendil-works/pi-coding-agent")), @@ -62,7 +64,18 @@ function selectedThinkingLevel(args) { return level; } -function selectedSkillPaths(args) { +function selectedTool(args) { + const index = args.indexOf("--tool"); + if (index < 0) return "webcmd"; + const tool = args[index + 1]; + if (!tool || tool.startsWith("--")) throw new Error("--tool requires a value"); + if (!new Set(["webcmd", "dev-browser", "libretto"]).has(tool)) { + throw new Error(`Unsupported Pi benchmark tool: ${tool}`); + } + return tool; +} + +function selectedSkillPaths(args, required) { const paths = []; for (let index = 0; index < args.length; index += 1) { if (args[index] !== "--skill-path") continue; @@ -78,7 +91,7 @@ function selectedSkillPaths(args) { paths.push(path); index += 1; } - if (paths.length === 0) { + if (required && paths.length === 0) { throw new Error("--skill-path requires a value"); } return paths; @@ -93,7 +106,8 @@ async function main() { const { provider, modelId, selector } = selectedModel(args); const thinkingLevel = selectedThinkingLevel(args); - const skillPaths = selectedSkillPaths(args); + const tool = selectedTool(args); + const skillPaths = selectedSkillPaths(args, tool !== "libretto"); const authStorage = AuthStorage.inMemory(); const modelRegistry = ModelRegistry.inMemory(authStorage); const model = modelRegistry.find(provider, modelId); @@ -123,30 +137,34 @@ async function main() { }); await resourceLoader.reload(); - const { session } = await createAgentSession({ - cwd, - agentDir, - model, - thinkingLevel, - authStorage, - modelRegistry, - resourceLoader, - settingsManager, - sessionManager: SessionManager.inMemory(cwd), - tools: ["bash", "read"], - }); - const unsubscribe = session.subscribe((event) => { - if ( - event.type === "message_end" || - event.type === "tool_execution_start" || - event.type === "tool_execution_end" - ) { - emit(event); - } - }); - const startedAt = Date.now(); - + const toolConfiguration = await createPiToolConfiguration(tool); + let session; + let unsubscribe = () => {}; try { + ({ session } = await createAgentSession({ + cwd, + agentDir, + model, + thinkingLevel, + authStorage, + modelRegistry, + resourceLoader, + settingsManager, + sessionManager: SessionManager.inMemory(cwd), + tools: toolConfiguration.tools, + noTools: toolConfiguration.noTools, + customTools: toolConfiguration.customTools, + })); + unsubscribe = session.subscribe((event) => { + if ( + event.type === "message_end" || + event.type === "tool_execution_start" || + event.type === "tool_execution_end" + ) { + emit(event); + } + }); + const startedAt = Date.now(); await session.prompt(readFileSync(0, "utf8")); emit({ type: "result", @@ -155,7 +173,8 @@ async function main() { }); } finally { unsubscribe(); - session.dispose(); + session?.dispose(); + await toolConfiguration.dispose(); } } diff --git a/benchmarks/scripts/pi_toolkit.mjs b/benchmarks/scripts/pi_toolkit.mjs new file mode 100644 index 00000000..17a1665e --- /dev/null +++ b/benchmarks/scripts/pi_toolkit.mjs @@ -0,0 +1,32 @@ +import { createPiBrowserTools } from "libretto-browser-tools/pi"; + +import { createFixedCdpProvider } from "./libretto_mcp.mjs"; + + +const LIBRETTO_TOOLS = new Set([ + "browser_open", + "browser_exec", + "browser_snapshot", + "browser_status", + "browser_close", +]); + + +export async function createPiToolConfiguration(tool, env = process.env) { + if (tool !== "libretto") { + return { + tools: ["bash", "read"], + customTools: [], + async dispose() {}, + }; + } + + const cdpEndpoint = env.LIBRETTO_CDP_URL; + if (!cdpEndpoint) throw new Error("LIBRETTO_CDP_URL is required"); + const toolkit = createPiBrowserTools(createFixedCdpProvider(cdpEndpoint)); + return { + noTools: "builtin", + customTools: toolkit.tools.filter(({ name }) => LIBRETTO_TOOLS.has(name)), + dispose: () => toolkit.dispose(), + }; +} diff --git a/benchmarks/scripts/run_controller.py b/benchmarks/scripts/run_controller.py index 44338f3d..aaa2e0a9 100644 --- a/benchmarks/scripts/run_controller.py +++ b/benchmarks/scripts/run_controller.py @@ -3,6 +3,7 @@ import base64 import hashlib import json +import math import os import re import shlex @@ -77,9 +78,11 @@ WEBCMD_BROWSER_SKILL = Path.home() / ".codex/skills/webcmd-browser" WEBCMD_BROWSER_SKILL_FILE = WEBCMD_BROWSER_SKILL / "SKILL.md" WEBCMD_BROWSER_SKILL_ROOT = WEBCMD_BROWSER_SKILL.resolve() -WEBCMD_SETUP_SKILL_FILES = frozenset( - {WEBCMD_BROWSER_SKILL_FILE.resolve()} -) +DEV_BROWSER_SKILL = Path.home() / ".codex/skills/dev-browser" +PI_SETUP_SKILL_FILES = { + "webcmd": frozenset({WEBCMD_BROWSER_SKILL_FILE.resolve()}), + "dev-browser": frozenset({(DEV_BROWSER_SKILL / "SKILL.md").resolve()}), +} GPT_5_6_SOL_PRICES_PER_MILLION = { "input": 5.0, "cached_input": 0.5, @@ -226,6 +229,7 @@ class ParsedEvents: provider_turns: int | None provider_duration_seconds: float | None provider_api_duration_seconds: float | None + agent_turns: int | None mcp_calls: list[tuple[str, str]] screenshot_images: list[bytes] @@ -347,9 +351,22 @@ def _controller_command( str(PI_CONTROLLER), "--model", model, - "--skill-path", - str(WEBCMD_BROWSER_SKILL), ] + pi_tool = tool or "webcmd" + if pi_tool == "webcmd": + command.extend(["--skill-path", str(WEBCMD_BROWSER_SKILL)]) + elif pi_tool == "dev-browser": + command.extend( + ["--tool", pi_tool, "--skill-path", str(DEV_BROWSER_SKILL)] + ) + elif pi_tool == "libretto": + if not (runtime_env or {}).get("LIBRETTO_CDP_URL"): + raise ValueError( + "Pi Libretto requires a task-private LIBRETTO_CDP_URL" + ) + command.extend(["--tool", pi_tool]) + else: + raise ValueError(f"Pi support is not configured for {pi_tool}") if reasoning_effort is not None: command.extend(["--thinking", reasoning_effort]) return command, prompt.encode() @@ -434,10 +451,13 @@ def _parse_events( reasoning_output = 0 usage_seen = False estimated_api_cost_usd = 0.0 - cost_complete = controller == "codex" and model in GPT_5_6_SOL_MODELS + cost_complete = controller == "pi" or ( + controller == "codex" and model in GPT_5_6_SOL_MODELS + ) provider_turns = None provider_duration_seconds = None provider_api_duration_seconds = None + agent_turns = 0 if controller == "pi" else None mcp_calls: list[tuple[str, str]] = [] screenshot_images: list[bytes] = [] for line in lines: @@ -531,6 +551,7 @@ def _parse_events( message = event.get("message") or {} if message.get("role") != "assistant": continue + agent_turns += 1 usage = message.get("usage") or {} if usage: ordinary_input += int(usage.get("input") or 0) @@ -538,6 +559,17 @@ def _parse_events( cache_creation_input += int(usage.get("cacheWrite") or 0) output_tokens += int(usage.get("output") or 0) usage_seen = True + turn_cost = (usage.get("cost") or {}).get("total") + if ( + isinstance(turn_cost, (int, float)) + and not isinstance(turn_cost, bool) + and math.isfinite(turn_cost) + ): + estimated_api_cost_usd += float(turn_cost) + else: + cost_complete = False + else: + cost_complete = False for block in message.get("content", []) or []: block_type = block.get("type") if block_type == "text": @@ -564,9 +596,15 @@ def _parse_events( elif ( name == "read" and Path(str(arguments.get("path") or "")).expanduser().resolve() - in WEBCMD_SETUP_SKILL_FILES + in PI_SETUP_SKILL_FILES.get(tool or "webcmd", frozenset()) ): steps.append(_short(f"setup_tool: {name} {_short(arguments)}")) + elif tool == "libretto": + event_types.append("mcp_tool_call") + mcp_calls.append(("libretto", name)) + tool_calls += 1 + steps_count += 1 + steps.append(_short(f"tool: {name} {_short(arguments)}")) else: event_types.append("mcp_tool_call") steps_count += 1 @@ -575,6 +613,30 @@ def _parse_events( result = event.get("result") or {} content = result.get("content") if isinstance(result, dict) else result steps.append(_short(f"tool_result: {_short(content or '')}")) + if tool == "libretto" and event.get("toolName") == "browser_snapshot": + encoded = None + for item in content if isinstance(content, list) else []: + if ( + isinstance(item, dict) + and item.get("type") == "image" + and item.get("mimeType") == "image/png" + and isinstance(item.get("data"), str) + and not item["data"].startswith("[omitted ") + ): + encoded = item["data"] + break + details = result.get("details") if isinstance(result, dict) else None + screenshot = details.get("screenshot") if isinstance(details, dict) else None + if encoded is None and isinstance(screenshot, dict): + if screenshot.get("mimeType") == "image/png": + encoded = screenshot.get("base64") + if isinstance(encoded, str): + try: + screenshot_images.append( + base64.b64decode(encoded, validate=True) + ) + except ValueError: + pass elif controller == "pi" and event_type == "result": text = str(event.get("result") or "") if text: @@ -663,6 +725,7 @@ def _parse_events( provider_turns=provider_turns, provider_duration_seconds=provider_duration_seconds, provider_api_duration_seconds=provider_api_duration_seconds, + agent_turns=agent_turns, mcp_calls=mcp_calls, screenshot_images=screenshot_images, ) @@ -1373,7 +1436,9 @@ async def run_controller(controller: Controller, model: str, tool: Tool, task: s provider_duration_seconds=parsed.provider_duration_seconds, provider_api_duration_seconds=parsed.provider_api_duration_seconds, agent_turns=( - turn_collector.agent_turns if turn_collector is not None else None + turn_collector.agent_turns + if turn_collector is not None + else parsed.agent_turns ), ) return ExecutionEvidence(final_answer=final_answer, steps=parsed.steps, screenshot_paths=sorted(shots_dir.glob("*.png")), controller_exit_code=process.returncode if process.returncode is not None else -9, termination=termination, metrics=metrics) diff --git a/benchmarks/scripts/run_eval.py b/benchmarks/scripts/run_eval.py index 937a1743..8d9a8d23 100644 --- a/benchmarks/scripts/run_eval.py +++ b/benchmarks/scripts/run_eval.py @@ -90,8 +90,18 @@ def validate_args(args: argparse.Namespace) -> None: raise ValueError("--reasoning-effort is supported only for Codex or Pi") if args.controller == "pi" and args.reasoning_effort not in PI_THINKING_LEVELS | {None}: raise ValueError(f"Pi does not support --reasoning-effort {args.reasoning_effort}") - if args.tools == "libretto" and args.controller != "codex": - raise ValueError("Libretto is currently supported only with the Codex controller") + if args.controller == "pi" and args.tools not in { + "webcmd", + "dev-browser", + "libretto", + }: + raise ValueError( + "Pi currently supports only Webcmd, dev-browser, or Libretto" + ) + if args.tools == "libretto" and args.controller not in {"codex", "pi"}: + raise ValueError( + "Libretto is currently supported only with the Codex or Pi controller" + ) def _validate_output_dir(output_dir: Path) -> Path: diff --git a/benchmarks/tests/pi_controller.test.mjs b/benchmarks/tests/pi_controller.test.mjs index 25e3f719..b3deac8b 100644 --- a/benchmarks/tests/pi_controller.test.mjs +++ b/benchmarks/tests/pi_controller.test.mjs @@ -6,6 +6,8 @@ import { join } from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; +import { createPiToolConfiguration } from "../scripts/pi_toolkit.mjs"; + const controller = fileURLToPath( new URL("../scripts/pi_controller.mjs", import.meta.url), ); @@ -78,3 +80,25 @@ test("Pi sidecar validates every explicitly selected skill", () => { rmSync(first, { recursive: true, force: true }); } }); + +test("Pi Libretto configuration exposes only the benchmark browser tools", async () => { + const configuration = await createPiToolConfiguration("libretto", { + LIBRETTO_CDP_URL: "http://127.0.0.1:43210", + }); + + try { + assert.equal(configuration.noTools, "builtin"); + assert.deepEqual( + configuration.customTools.map((tool) => tool.name), + [ + "browser_open", + "browser_exec", + "browser_snapshot", + "browser_status", + "browser_close", + ], + ); + } finally { + await configuration.dispose(); + } +}); diff --git a/benchmarks/tests/test_controller.py b/benchmarks/tests/test_controller.py index b60ea017..61a6c615 100644 --- a/benchmarks/tests/test_controller.py +++ b/benchmarks/tests/test_controller.py @@ -472,6 +472,49 @@ def test_codex_libretto_collects_snapshot_from_wrapped_mcp_result(): assert parsed.screenshot_images == [png] +def test_pi_libretto_custom_tool_counts_once_and_collects_snapshot_image(): + png = b"\x89PNG\r\npi-libretto" + events = [ + { + "type": "tool_execution_start", + "toolName": "browser_snapshot", + "args": {"sessionId": "ses-1", "screenshot": True}, + }, + { + "type": "tool_execution_end", + "toolName": "browser_snapshot", + "result": { + "content": [ + { + "type": "image", + "data": "[omitted 12000 characters]", + "mimeType": "image/png", + } + ], + "details": { + "ok": True, + "screenshot": { + "base64": base64.b64encode(png).decode(), + "mimeType": "image/png", + }, + }, + }, + "isError": False, + }, + ] + + parsed = _parse_events( + "pi", [json.dumps(event) for event in events], tool="libretto" + ) + + assert parsed.tool_calls == 1 + assert parsed.mcp_calls == [("libretto", "browser_snapshot")] + assert parsed.screenshot_images == [png] + assert not _policy_violation( + "libretto", parsed.commands, parsed.event_types, parsed.mcp_calls + ) + + @pytest.mark.parametrize( ("server", "tool"), [ @@ -548,6 +591,14 @@ def test_pi_events_normalize_commands_results_usage_and_final_text(): "cacheRead": 70, "cacheWrite": 10, "output": 5, + "totalTokens": 105, + "cost": { + "input": 0.01, + "output": 0.02, + "cacheRead": 0.003, + "cacheWrite": 0.004, + "total": 0.037, + }, }, "content": [{"type": "text", "text": "working"}], }, @@ -579,6 +630,14 @@ def test_pi_events_normalize_commands_results_usage_and_final_text(): "cacheRead": 0, "cacheWrite": 0, "output": 2, + "totalTokens": 12, + "cost": { + "input": 0.005, + "output": 0.006, + "cacheRead": 0.0, + "cacheWrite": 0.0, + "total": 0.011, + }, }, "content": [{"type": "text", "text": "FINAL ANSWER: 42"}], }, @@ -603,11 +662,58 @@ def test_pi_events_normalize_commands_results_usage_and_final_text(): assert parsed.tokens.non_cached_input == 30 assert parsed.tokens.output == 7 assert parsed.tokens.total == 117 + assert parsed.tokens.estimated_api_cost_usd == pytest.approx(0.048) + assert parsed.agent_turns == 2 assert parsed.provider_duration_seconds == 1.0 assert _extract_final_answer(parsed.final_text) == "42" assert not _policy_violation("webcmd", parsed.commands, parsed.event_types) +def test_pi_cost_is_unavailable_when_any_usage_turn_lacks_cost(): + events = [ + { + "type": "message_end", + "message": { + "role": "assistant", + "usage": { + "input": 10, + "cacheRead": 0, + "cacheWrite": 0, + "output": 2, + "totalTokens": 12, + "cost": { + "input": 0.005, + "output": 0.006, + "cacheRead": 0.0, + "cacheWrite": 0.0, + "total": 0.011, + }, + }, + "content": [{"type": "text", "text": "working"}], + }, + }, + { + "type": "message_end", + "message": { + "role": "assistant", + "usage": { + "input": 5, + "cacheRead": 0, + "cacheWrite": 0, + "output": 1, + "totalTokens": 6, + }, + "content": [{"type": "text", "text": "done"}], + }, + }, + ] + + parsed = _parse_events("pi", [json.dumps(event) for event in events]) + + assert parsed.agent_turns == 2 + assert parsed.tokens.estimated_api_cost_usd is None + + def test_pi_non_bash_tool_is_a_policy_violation(): event = { "type": "tool_execution_start", @@ -662,6 +768,26 @@ def test_pi_may_read_only_the_registered_webcmd_browser_skill(): ) +def test_pi_dev_browser_skill_read_is_setup_not_a_foreign_tool(): + event = { + "type": "tool_execution_start", + "toolName": "read", + "args": { + "path": str(Path.home() / ".codex/skills/dev-browser/SKILL.md") + }, + } + + parsed = _parse_events( + "pi", [json.dumps(event)], tool="dev-browser" + ) + + assert parsed.tool_calls == 0 + assert parsed.steps_count == 0 + assert not _policy_violation( + "dev-browser", parsed.commands, parsed.event_types + ) + + def test_claude_result_usage_is_used_when_messages_have_no_usage(): event = { "type": "result", @@ -1391,6 +1517,46 @@ def test_controller_commands_are_noninteractive(): assert pi_input == b"prompt" +def test_pi_dev_browser_command_mounts_only_the_dev_browser_skill(): + command, stdin = _controller_command( + "pi", "openai/gpt-5.6-sol", "prompt", tool="dev-browser" + ) + + assert command == [ + "node", + str(run_controller.PI_CONTROLLER), + "--model", + "openai/gpt-5.6-sol", + "--tool", + "dev-browser", + "--skill-path", + str(Path.home() / ".codex/skills/dev-browser"), + ] + assert str(WEBCMD_BROWSER_SKILL) not in command + assert stdin == b"prompt" + + +def test_pi_libretto_command_uses_direct_tools_without_a_skill_path(): + command, stdin = _controller_command( + "pi", + "openai/gpt-5.6-sol", + "prompt", + tool="libretto", + runtime_env={"LIBRETTO_CDP_URL": "http://127.0.0.1:43210"}, + ) + + assert command == [ + "node", + str(run_controller.PI_CONTROLLER), + "--model", + "openai/gpt-5.6-sol", + "--tool", + "libretto", + ] + assert "--skill-path" not in command + assert stdin == b"prompt" + + def test_codex_controller_command_applies_reasoning_effort_override(): command, stdin = _controller_command("codex", "gpt-5.6-sol", "prompt", "high") @@ -1697,6 +1863,60 @@ def fake_command(*args, otel_endpoint=None, **kwargs): assert evidence.metrics.agent_turns == 1 +def test_pi_execution_records_agent_turns_and_controller_cost(tmp_path, monkeypatch): + usage_event = json.dumps( + { + "type": "message_end", + "message": { + "role": "assistant", + "usage": { + "input": 20, + "cacheRead": 5, + "cacheWrite": 0, + "output": 4, + "totalTokens": 29, + "cost": { + "input": 0.02, + "output": 0.04, + "cacheRead": 0.001, + "cacheWrite": 0.0, + "total": 0.061, + }, + }, + "content": [ + {"type": "text", "text": "FINAL ANSWER: 42"} + ], + }, + } + ) + result_event = json.dumps( + { + "type": "result", + "result": "FINAL ANSWER: 42", + "duration_ms": 100, + } + ) + + _fake_controller( + monkeypatch, f"print({usage_event!r}); print({result_event!r})" + ) + monkeypatch.setattr(run_controller, "_close_session", _no_close) + + evidence = asyncio.run( + execute_controller( + "pi", + "openai/gpt-5.6-sol", + "webcmd", + "task", + tmp_path / "attempt", + 5, + ) + ) + + assert evidence.metrics.agent_turns == 1 + assert evidence.metrics.tokens.estimated_api_cost_usd == pytest.approx(0.061) + + def test_axi_execution_passes_private_runtime_env_and_closes_runtime(tmp_path, monkeypatch): command_event = json.dumps({"type": "item.completed", "item": {"type": "command_execution", "command": "npx -y chrome-devtools-axi snapshot -i", "aggregated_output": "page"}}) answer_event = json.dumps({"type": "item.completed", "item": {"type": "agent_message", "text": "FINAL ANSWER: 42"}}) diff --git a/benchmarks/tests/test_run_eval.py b/benchmarks/tests/test_run_eval.py index 83125b6d..4ff806c4 100644 --- a/benchmarks/tests/test_run_eval.py +++ b/benchmarks/tests/test_run_eval.py @@ -171,12 +171,51 @@ def test_cli_accepts_libretto_for_codex(): assert args.tools == "libretto" -@pytest.mark.parametrize("controller", ["claude", "pi"]) -def test_cli_rejects_libretto_for_non_codex_controllers(controller): +def test_cli_accepts_libretto_for_pi(): args = run_eval.parse_args( [ "--controller", - controller, + "pi", + "--model", + "openai/gpt-5.6-sol", + "--benchmark", + "BU_Bench_V1", + "--tasks", + "1", + "--tools", + "libretto", + ] + ) + + run_eval.validate_args(args) + + +@pytest.mark.parametrize("tool", ["chrome-devtools-axi", "agent-browser"]) +def test_cli_rejects_pi_tools_without_pi_integration(tool): + args = run_eval.parse_args( + [ + "--controller", + "pi", + "--model", + "openai/gpt-5.6-sol", + "--benchmark", + "BU_Bench_V1", + "--tasks", + "1", + "--tools", + tool, + ] + ) + + with pytest.raises(ValueError, match="Pi.*Webcmd, dev-browser, or Libretto"): + run_eval.validate_args(args) + + +def test_cli_rejects_libretto_for_claude(): + args = run_eval.parse_args( + [ + "--controller", + "claude", "--model", "model", "--benchmark", @@ -188,7 +227,7 @@ def test_cli_rejects_libretto_for_non_codex_controllers(controller): ] ) - with pytest.raises(ValueError, match="Libretto.*Codex"): + with pytest.raises(ValueError, match="Libretto.*Codex or Pi"): run_eval.validate_args(args)