pi support

benchmarks/README.md

@@ -215,12 +215,17 @@ uv run python benchmarks/scripts/run_eval.py \
   --tools dev-browser
 ```
 
+Use `--controller pi --model openai/gpt-5.6-sol` to run the same lane through
+Pi. The Pi sidecar mounts the installed `dev-browser` skill and enables only
+its `bash` and `read` tools.
+
 `dev-browser install` is required because it installs the daemon's Playwright
 and QuickJS dependencies. It may also download dev-browser's Chromium, but the
 benchmark does not use that browser: the task-private shim always connects
 dev-browser to the task's dedicated CloakBrowser CDP endpoint.
 
-For Libretto Browser Tools with Codex and one dedicated CloakBrowser per task:
+For Libretto Browser Tools with Codex or Pi and one dedicated CloakBrowser per
+task:
 
 ```bash
 npm ci
@@ -234,10 +239,12 @@ uv run python benchmarks/scripts/run_eval.py \
   --tools libretto
 ```
 
-Libretto is currently Codex-only. The harness injects its pinned stdio MCP
-server per attempt and exposes `browser_open`, `browser_exec`,
-`browser_snapshot`, `browser_status`, and `browser_close`. `browser_connect`
-is disabled so the agent cannot leave the task's dedicated CloakBrowser.
+With Codex, the harness injects its pinned stdio MCP server per attempt. With
+Pi, it registers the equivalent native Pi custom tools directly; use
+`--controller pi --model openai/gpt-5.6-sol`. Both expose only `browser_open`,
+`browser_exec`, `browser_snapshot`, `browser_status`, and `browser_close`.
+`browser_connect` is disabled so the agent cannot leave the task's dedicated
+CloakBrowser.
 
 Use `--stealth-view official` only with `Stealth_Bench_V1`. Never publish `results/`.
 

benchmarks/scripts/pi_controller.mjs

@@ -9,6 +9,8 @@ import {
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 
+import { createPiToolConfiguration } from "./pi_toolkit.mjs";
+
 const SDK_PACKAGE = JSON.parse(
   readFileSync(
     new URL("../package.json", import.meta.resolve("@earendil-works/pi-coding-agent")),
@@ -62,7 +64,18 @@ function selectedThinkingLevel(args) {
   return level;
 }
 
-function selectedSkillPaths(args) {
+function selectedTool(args) {
+  const index = args.indexOf("--tool");
+  if (index < 0) return "webcmd";
+  const tool = args[index + 1];
+  if (!tool || tool.startsWith("--")) throw new Error("--tool requires a value");
+  if (!new Set(["webcmd", "dev-browser", "libretto"]).has(tool)) {
+    throw new Error(`Unsupported Pi benchmark tool: ${tool}`);
+  }
+  return tool;
+}
+
+function selectedSkillPaths(args, required) {
   const paths = [];
   for (let index = 0; index < args.length; index += 1) {
     if (args[index] !== "--skill-path") continue;
@@ -78,7 +91,7 @@ function selectedSkillPaths(args) {
     paths.push(path);
     index += 1;
   }
-  if (paths.length === 0) {
+  if (required && paths.length === 0) {
     throw new Error("--skill-path requires a value");
   }
   return paths;
@@ -93,7 +106,8 @@ async function main() {
 
   const { provider, modelId, selector } = selectedModel(args);
   const thinkingLevel = selectedThinkingLevel(args);
-  const skillPaths = selectedSkillPaths(args);
+  const tool = selectedTool(args);
+  const skillPaths = selectedSkillPaths(args, tool !== "libretto");
   const authStorage = AuthStorage.inMemory();
   const modelRegistry = ModelRegistry.inMemory(authStorage);
   const model = modelRegistry.find(provider, modelId);
@@ -123,30 +137,34 @@ async function main() {
   });
   await resourceLoader.reload();
 
-  const { session } = await createAgentSession({
-    cwd,
-    agentDir,
-    model,
-    thinkingLevel,
-    authStorage,
-    modelRegistry,
-    resourceLoader,
-    settingsManager,
-    sessionManager: SessionManager.inMemory(cwd),
-    tools: ["bash", "read"],
-  });
-  const unsubscribe = session.subscribe((event) => {
-    if (
-      event.type === "message_end" ||
-      event.type === "tool_execution_start" ||
-      event.type === "tool_execution_end"
-    ) {
-      emit(event);
-    }
-  });
-  const startedAt = Date.now();
-
+  const toolConfiguration = await createPiToolConfiguration(tool);
+  let session;
+  let unsubscribe = () => {};
   try {
+    ({ session } = await createAgentSession({
+      cwd,
+      agentDir,
+      model,
+      thinkingLevel,
+      authStorage,
+      modelRegistry,
+      resourceLoader,
+      settingsManager,
+      sessionManager: SessionManager.inMemory(cwd),
+      tools: toolConfiguration.tools,
+      noTools: toolConfiguration.noTools,
+      customTools: toolConfiguration.customTools,
+    }));
+    unsubscribe = session.subscribe((event) => {
+      if (
+        event.type === "message_end" ||
+        event.type === "tool_execution_start" ||
+        event.type === "tool_execution_end"
+      ) {
+        emit(event);
+      }
+    });
+    const startedAt = Date.now();
     await session.prompt(readFileSync(0, "utf8"));
     emit({
       type: "result",
@@ -155,7 +173,8 @@ async function main() {
     });
   } finally {
     unsubscribe();
-    session.dispose();
+    session?.dispose();
+    await toolConfiguration.dispose();
   }
 }
 

benchmarks/scripts/pi_toolkit.mjs

@@ -0,0 +1,32 @@
+import { createPiBrowserTools } from "libretto-browser-tools/pi";
+
+import { createFixedCdpProvider } from "./libretto_mcp.mjs";
+
+
+const LIBRETTO_TOOLS = new Set([
+  "browser_open",
+  "browser_exec",
+  "browser_snapshot",
+  "browser_status",
+  "browser_close",
+]);
+
+
+export async function createPiToolConfiguration(tool, env = process.env) {
+  if (tool !== "libretto") {
+    return {
+      tools: ["bash", "read"],
+      customTools: [],
+      async dispose() {},
+    };
+  }
+
+  const cdpEndpoint = env.LIBRETTO_CDP_URL;
+  if (!cdpEndpoint) throw new Error("LIBRETTO_CDP_URL is required");
+  const toolkit = createPiBrowserTools(createFixedCdpProvider(cdpEndpoint));
+  return {
+    noTools: "builtin",
+    customTools: toolkit.tools.filter(({ name }) => LIBRETTO_TOOLS.has(name)),
+    dispose: () => toolkit.dispose(),
+  };
+}

benchmarks/scripts/run_controller.py

@@ -3,6 +3,7 @@
 import base64
 import hashlib
 import json
+import math
 import os
 import re
 import shlex
@@ -77,9 +78,11 @@
 WEBCMD_BROWSER_SKILL = Path.home() / ".codex/skills/webcmd-browser"
 WEBCMD_BROWSER_SKILL_FILE = WEBCMD_BROWSER_SKILL / "SKILL.md"
 WEBCMD_BROWSER_SKILL_ROOT = WEBCMD_BROWSER_SKILL.resolve()
-WEBCMD_SETUP_SKILL_FILES = frozenset(
-    {WEBCMD_BROWSER_SKILL_FILE.resolve()}
-)
+DEV_BROWSER_SKILL = Path.home() / ".codex/skills/dev-browser"
+PI_SETUP_SKILL_FILES = {
+    "webcmd": frozenset({WEBCMD_BROWSER_SKILL_FILE.resolve()}),
+    "dev-browser": frozenset({(DEV_BROWSER_SKILL / "SKILL.md").resolve()}),
+}
 GPT_5_6_SOL_PRICES_PER_MILLION = {
     "input": 5.0,
     "cached_input": 0.5,
@@ -226,6 +229,7 @@ class ParsedEvents:
     provider_turns: int | None
     provider_duration_seconds: float | None
     provider_api_duration_seconds: float | None
+    agent_turns: int | None
     mcp_calls: list[tuple[str, str]]
     screenshot_images: list[bytes]
 
@@ -347,9 +351,22 @@ def _controller_command(
             str(PI_CONTROLLER),
             "--model",
             model,
-            "--skill-path",
-            str(WEBCMD_BROWSER_SKILL),
         ]
+        pi_tool = tool or "webcmd"
+        if pi_tool == "webcmd":
+            command.extend(["--skill-path", str(WEBCMD_BROWSER_SKILL)])
+        elif pi_tool == "dev-browser":
+            command.extend(
+                ["--tool", pi_tool, "--skill-path", str(DEV_BROWSER_SKILL)]
+            )
+        elif pi_tool == "libretto":
+            if not (runtime_env or {}).get("LIBRETTO_CDP_URL"):
+                raise ValueError(
+                    "Pi Libretto requires a task-private LIBRETTO_CDP_URL"
+                )
+            command.extend(["--tool", pi_tool])
+        else:
+            raise ValueError(f"Pi support is not configured for {pi_tool}")
         if reasoning_effort is not None:
             command.extend(["--thinking", reasoning_effort])
         return command, prompt.encode()
@@ -434,10 +451,13 @@ def _parse_events(
     reasoning_output = 0
     usage_seen = False
     estimated_api_cost_usd = 0.0
-    cost_complete = controller == "codex" and model in GPT_5_6_SOL_MODELS
+    cost_complete = controller == "pi" or (
+        controller == "codex" and model in GPT_5_6_SOL_MODELS
+    )
     provider_turns = None
     provider_duration_seconds = None
     provider_api_duration_seconds = None
+    agent_turns = 0 if controller == "pi" else None
     mcp_calls: list[tuple[str, str]] = []
     screenshot_images: list[bytes] = []
     for line in lines:
@@ -531,13 +551,25 @@ def _parse_events(
             message = event.get("message") or {}
             if message.get("role") != "assistant":
                 continue
+            agent_turns += 1
             usage = message.get("usage") or {}
             if usage:
                 ordinary_input += int(usage.get("input") or 0)
                 cache_read_input += int(usage.get("cacheRead") or 0)
                 cache_creation_input += int(usage.get("cacheWrite") or 0)
                 output_tokens += int(usage.get("output") or 0)
                 usage_seen = True
+                turn_cost = (usage.get("cost") or {}).get("total")
+                if (
+                    isinstance(turn_cost, (int, float))
+                    and not isinstance(turn_cost, bool)
+                    and math.isfinite(turn_cost)
+                ):
+                    estimated_api_cost_usd += float(turn_cost)
+                else:
+                    cost_complete = False
+            else:
+                cost_complete = False
             for block in message.get("content", []) or []:
                 block_type = block.get("type")
                 if block_type == "text":
@@ -564,9 +596,15 @@ def _parse_events(
             elif (
                 name == "read"
                 and Path(str(arguments.get("path") or "")).expanduser().resolve()
-                in WEBCMD_SETUP_SKILL_FILES
+                in PI_SETUP_SKILL_FILES.get(tool or "webcmd", frozenset())
             ):
                 steps.append(_short(f"setup_tool: {name} {_short(arguments)}"))
+            elif tool == "libretto":
+                event_types.append("mcp_tool_call")
+                mcp_calls.append(("libretto", name))
+                tool_calls += 1
+                steps_count += 1
+                steps.append(_short(f"tool: {name} {_short(arguments)}"))
             else:
                 event_types.append("mcp_tool_call")
                 steps_count += 1
@@ -575,6 +613,30 @@ def _parse_events(
             result = event.get("result") or {}
             content = result.get("content") if isinstance(result, dict) else result
             steps.append(_short(f"tool_result: {_short(content or '')}"))
+            if tool == "libretto" and event.get("toolName") == "browser_snapshot":
+                encoded = None
+                for item in content if isinstance(content, list) else []:
+                    if (
+                        isinstance(item, dict)
+                        and item.get("type") == "image"
+                        and item.get("mimeType") == "image/png"
+                        and isinstance(item.get("data"), str)
+                        and not item["data"].startswith("[omitted ")
+                    ):
+                        encoded = item["data"]
+                        break
+                details = result.get("details") if isinstance(result, dict) else None
+                screenshot = details.get("screenshot") if isinstance(details, dict) else None
+                if encoded is None and isinstance(screenshot, dict):
+                    if screenshot.get("mimeType") == "image/png":
+                        encoded = screenshot.get("base64")
+                if isinstance(encoded, str):
+                    try:
+                        screenshot_images.append(
+                            base64.b64decode(encoded, validate=True)
+                        )
+                    except ValueError:
+                        pass
         elif controller == "pi" and event_type == "result":
             text = str(event.get("result") or "")
             if text:
@@ -663,6 +725,7 @@ def _parse_events(
         provider_turns=provider_turns,
         provider_duration_seconds=provider_duration_seconds,
         provider_api_duration_seconds=provider_api_duration_seconds,
+        agent_turns=agent_turns,
         mcp_calls=mcp_calls,
         screenshot_images=screenshot_images,
     )
@@ -1373,7 +1436,9 @@ async def run_controller(controller: Controller, model: str, tool: Tool, task: s
         provider_duration_seconds=parsed.provider_duration_seconds,
         provider_api_duration_seconds=parsed.provider_api_duration_seconds,
         agent_turns=(
-            turn_collector.agent_turns if turn_collector is not None else None
+            turn_collector.agent_turns
+            if turn_collector is not None
+            else parsed.agent_turns
         ),
     )
     return ExecutionEvidence(final_answer=final_answer, steps=parsed.steps, screenshot_paths=sorted(shots_dir.glob("*.png")), controller_exit_code=process.returncode if process.returncode is not None else -9, termination=termination, metrics=metrics)

benchmarks/scripts/run_eval.py

@@ -90,8 +90,18 @@ def validate_args(args: argparse.Namespace) -> None:
         raise ValueError("--reasoning-effort is supported only for Codex or Pi")
     if args.controller == "pi" and args.reasoning_effort not in PI_THINKING_LEVELS | {None}:
         raise ValueError(f"Pi does not support --reasoning-effort {args.reasoning_effort}")
-    if args.tools == "libretto" and args.controller != "codex":
-        raise ValueError("Libretto is currently supported only with the Codex controller")
+    if args.controller == "pi" and args.tools not in {
+        "webcmd",
+        "dev-browser",
+        "libretto",
+    }:
+        raise ValueError(
+            "Pi currently supports only Webcmd, dev-browser, or Libretto"
+        )
+    if args.tools == "libretto" and args.controller not in {"codex", "pi"}:
+        raise ValueError(
+            "Libretto is currently supported only with the Codex or Pi controller"
+        )
 
 
 def _validate_output_dir(output_dir: Path) -> Path: