Suspend sandbox execution deadlines while a tool dispatch awaits the host

.changeset/suspend-private-sandbox-deadlines.md

@@ -0,0 +1,6 @@
+---
+"@executor-js/runtime-dynamic-worker": patch
+"@executor-js/runtime-deno-subprocess": patch
+---
+
+Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.

.changeset/suspend-sandbox-deadlines.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/runtime-quickjs": patch
+---
+
+Suspend sandbox execution deadlines while tool calls await the host, and reset the autonomous-compute budget after each dispatch returns.

e2e/scenarios/resume-after-sandbox-deadline.test.ts

@@ -0,0 +1,150 @@
+// Cross-target: approval durability against the sandbox clock — the "a human
+// is allowed to take a few minutes" promise. A paused execution suspends the
+// sandbox mid-call, but the sandbox's execution deadline (5 minutes) used to
+// keep ticking while the human decided. Each pause advertises its own
+// approval window, so with chained approvals a later window legitimately
+// extends past the sandbox's absolute clock: the human answers every window
+// on time, yet the fiber is already dead and the final approval lands on an
+// unknown execution.
+//
+// The journey drives exactly that shape: ONE execution with TWO approval
+// gates. The first approval is granted late in its window (~3.5 min), so the
+// second pause's window reaches well past the sandbox's 5-minute mark. The
+// second approval arrives ~5.75 min after execution start — inside its OWN
+// advertised window, but past the old absolute deadline. Deliberately slow
+// (~6 min): the elapsed time IS the subject under test. A single-pause
+// variant cannot express this cross-target — hosts that advertise a
+// 4-minute window would expire it legitimately before the sandbox clock
+// even matters.
+//
+// The gate is `policies.create`'s own `requiresApproval` annotation
+// (hermetic, same device as policy-tool-approval.test.ts); both approvals
+// happen in the same MCP session, so the only variable is elapsed time.
+import { randomUUID } from "node:crypto";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { composePluginApi } from "@executor-js/api/server";
+
+import { scenario } from "../src/scenario";
+import { Api, Mcp, Target } from "../src/services";
+import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts";
+
+const coreApi = composePluginApi([] as const);
+
+// Grant the first approval at 3.5 min — late but inside its 4-minute window.
+// The second pause then opens a fresh window reaching ~7.5 min.
+const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000;
+// Grant the second approval 2.25 min later: ~5.75 min after execution start,
+// past the sandbox's 5-minute budget but inside the second window.
+const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000;
+
+/** Sandbox code that creates two policies through the approval-gated core
+ *  tool. Patterns are unique-per-run and match no real tool, so the rules are
+ *  inert even if leaked. */
+const createPoliciesCode = (firstPattern: string, secondPattern: string) => `
+const first = await tools.executor.coreTools.policies.create({
+  owner: "user",
+  pattern: ${JSON.stringify(firstPattern)},
+  action: "block",
+});
+const second = await tools.executor.coreTools.policies.create({
+  owner: "user",
+  pattern: ${JSON.stringify(secondPattern)},
+  action: "block",
+});
+return JSON.stringify({ first: first.ok, second: second.ok });
+`;
+
+// The journey spans ~6 real minutes of paused waiting, so the host must keep
+// the paused session alive that long. The suite's default e2e override shrinks
+// the paused-session idle teardown to seconds (to keep teardown tests fast),
+// which would evict the session mid-scenario for reasons unrelated to the
+// clock under test — require the production-like window instead.
+const PAUSED_IDLE_WINDOW_TOO_SHORT =
+  configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000
+    ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it`
+    : undefined;
+
+scenario(
+  "MCP · chained approvals granted within their windows survive the sandbox clock",
+  { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT },
+  Effect.gen(function* () {
+    const target = yield* Target;
+    const apiSurface = yield* Api;
+    const mcp = yield* Mcp;
+    const identity = yield* target.newIdentity();
+    const client = yield* apiSurface.client(coreApi, identity);
+    const runId = randomUUID().slice(0, 8);
+    const firstPattern = `resume-deadline-a-${runId}.*`;
+    const secondPattern = `resume-deadline-b-${runId}.*`;
+
+    const cleanup = client.policies.list().pipe(
+      Effect.flatMap((list) =>
+        Effect.forEach(
+          list.filter((p) => p.pattern === firstPattern || p.pattern === secondPattern),
+          (p) =>
+            client.policies
+              .remove({ params: { policyId: p.id }, payload: { owner: "user" } })
+              .pipe(Effect.ignore),
+        ),
+      ),
+      Effect.ignore,
+    );
+
+    yield* Effect.gen(function* () {
+      const session = mcp.session(identity);
+      const tools = yield* session.listTools();
+      expect(tools).toContain("execute");
+
+      const paused = yield* session.call("execute", {
+        code: createPoliciesCode(firstPattern, secondPattern),
+      });
+      expect(paused.text, "the first gated call paused for approval").toContain("Execution paused");
+      const firstMatch = /\bexecutionId:\s*(\S+)/.exec(paused.text);
+      expect(firstMatch, "the first pause carries an executionId").not.toBeNull();
+
+      // The human takes most of the first approval window before deciding.
+      yield* Effect.sleep(`${FIRST_APPROVAL_DELAY_MS} millis`);
+
+      const secondPaused = yield* session.call("resume", {
+        executionId: firstMatch![1]!,
+        action: "accept",
+        content: JSON.stringify({}),
+      });
+      expect(
+        secondPaused.text,
+        "the on-time first approval reaches its pause and surfaces the second gate",
+      ).toContain("Execution paused");
+      const secondMatch = /\bexecutionId:\s*(\S+)/.exec(secondPaused.text);
+      expect(secondMatch, "the second pause carries an executionId").not.toBeNull();
+
+      // The second decision lands past the sandbox's 5-minute budget (counted
+      // from execution start) but well inside the second advertised window.
+      yield* Effect.sleep(`${SECOND_APPROVAL_DELAY_MS} millis`);
+
+      const resumed = yield* session.call("resume", {
+        executionId: secondMatch![1]!,
+        action: "accept",
+        content: JSON.stringify({}),
+      });
+
+      expect(
+        resumed.text,
+        "the second on-time approval reaches a live pause instead of a dead or unknown execution",
+      ).not.toMatch(/Paused execution is unknown|Paused execution expired|timed out after/);
+      expect(resumed.ok, "the fully approved execution completed without error").toBe(true);
+
+      // Both approvals took effect: the gated tool ran twice.
+      const afterApproval = yield* client.policies.list();
+      expect(
+        afterApproval.some((p) => p.pattern === firstPattern),
+        "the first gated tool ran after its approval",
+      ).toBe(true);
+      expect(
+        afterApproval.some((p) => p.pattern === secondPattern),
+        "the second gated tool ran after the late-but-in-window approval",
+      ).toBe(true);
+    }).pipe(Effect.ensuring(cleanup));
+  }),
+);

packages/core/execution/src/tool-invoker.test.ts

@@ -1743,28 +1743,37 @@ describe("pause/resume with multiple elicitations", () => {
     { timeout: 10000 },
   );
 
-  // live clock: the sandbox timeout is a real timer, so the test must
-  // actually wait for it rather than suspend on the virtual TestClock.
+  // Live clock: the failing executor delays long enough for its detached tool
+  // invocation to publish the pause before the executor settles on its own.
   it.live(
     "a pause abandoned by a failing sandbox is dropped and its resume replays the failure outcome",
     () =>
       Effect.gen(function* () {
         const executor = yield* makeElicitingExecutor();
-        // Sandbox times out while suspended on the elicitation, so the fiber
-        // settles without a resume ever arriving.
+        const failingCodeExecutor = {
+          execute: (_code, toolInvoker) =>
+            Effect.gen(function* () {
+              yield* Effect.forkChild(
+                toolInvoker
+                  .invoke({ path: "api.org.main.singleApproval", args: {} })
+                  .pipe(Effect.ignore),
+              );
+              yield* Effect.sleep("100 millis");
+              return { result: null, error: "sandbox failed after pausing" };
+            }),
+        } satisfies typeof codeExecutor;
         const engine = createExecutionEngine({
           executor,
-          codeExecutor: makeQuickJsExecutor({ timeoutMs: 250 }),
+          codeExecutor: failingCodeExecutor,
         });
-        const code = "return await tools.api.org.main.singleApproval({});";
 
-        const outcome1 = yield* engine.executeWithPause(code);
+        const outcome1 = yield* engine.executeWithPause("ignored");
         expect(outcome1.status).toBe("paused");
         if (outcome1.status !== "paused") return;
         const executionId = outcome1.execution.id;
 
-        // Wait for the sandbox timeout to settle the detached fiber.
-        yield* Effect.sleep("600 millis");
+        // Wait for the sandbox failure to settle the detached fiber.
+        yield* Effect.sleep("300 millis");
 
         // The dead pause must no longer be reported as live...
         const stillPaused = yield* engine.getPausedExecution(executionId);
@@ -1774,7 +1783,7 @@ describe("pause/resume with multiple elicitations", () => {
         const late = yield* engine.resume(executionId, { action: "accept" });
         expect(late?.status).toBe("completed");
         const completed = late as Extract<NonNullable<typeof late>, { status: "completed" }>;
-        expect(completed.result.error).toContain("timed out");
+        expect(completed.result.error).toBe("sandbox failed after pausing");
       }),
     { timeout: 10000 },
   );

packages/kernel/runtime-deno-subprocess/src/index.test.ts

@@ -234,19 +234,51 @@ describe.skipIf(!isDenoAvailable())("runtime-deno-subprocess", () => {
     }),
   );
 
-  it.effect("respects timeout", () =>
-    Effect.gen(function* () {
-      const executor = makeDenoSubprocessExecutor({
-        timeoutMs: 500,
-      });
-      const toolInvoker = makeTestInvoker({});
+  it("suspends the execution deadline while a tool dispatch is in flight", async () => {
+    const timeoutMs = 300;
+    const executor = makeDenoSubprocessExecutor({ timeoutMs });
+    const toolInvoker: SandboxToolInvoker = {
+      invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("slow result")),
+    };
+
+    const output = await Effect.runPromise(
+      executor.execute("return await tools.slow.wait({});", toolInvoker),
+    );
+
+    expect(output.error).toBeUndefined();
+    expect(output.result).toBe("slow result");
+  });
+
+  it("still times out continuous autonomous compute", async () => {
+    const timeoutMs = 300;
+    const executor = makeDenoSubprocessExecutor({ timeoutMs });
+    const toolInvoker = makeTestInvoker({});
 
-      const output = yield* executor.execute("await new Promise(() => {}); return 1;", toolInvoker);
+    const output = await Effect.runPromise(
+      executor.execute("await new Promise(() => {}); return 1;", toolInvoker),
+    );
 
-      expect(output.result).toBeNull();
-      expect(output.error).toContain("timed out");
-    }),
-  );
+    expect(output.result).toBeNull();
+    expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
+  });
+
+  it("resets the execution deadline after a tool dispatch returns", async () => {
+    const timeoutMs = 300;
+    const executor = makeDenoSubprocessExecutor({ timeoutMs });
+    const toolInvoker: SandboxToolInvoker = {
+      invoke: () => Effect.sleep(timeoutMs * 3).pipe(Effect.as("done")),
+    };
+
+    const output = await Effect.runPromise(
+      executor.execute(
+        "await tools.slow.wait({}); await new Promise(() => {}); return 1;",
+        toolInvoker,
+      ),
+    );
+
+    expect(output.result).toBeNull();
+    expect(output.error).toBe(`Deno subprocess execution timed out after ${timeoutMs}ms`);
+  });
 
   it.effect("network access is denied by default", () =>
     Effect.gen(function* () {

packages/kernel/runtime-deno-subprocess/src/index.ts

@@ -230,19 +230,29 @@ const executeInDeno = (
       catch: (cause) => new DenoSpawnError({ executable: denoExecutable, reason: cause }),
     });
 
+    const start = Date.now();
+    let inFlight = 0;
+    let lastReturnedAt = start;
+
     // Send code to the subprocess
     writeMessage(worker.stdin, { type: "start", code: recoveredBody, nonce });
 
-    // Set up timeout — kills process and completes the deferred
-    const timer = setTimeout(() => {
+    // Measure only continuous autonomous subprocess compute. Tool dispatches
+    // suspend the clock and each return grants a fresh timeout budget.
+    const timer = setInterval(() => {
+      const now = Date.now();
+      if (inFlight > 0 || now - Math.max(start, lastReturnedAt) < timeoutMs) {
+        return;
+      }
+
       worker.dispose();
       runSync(
         completeWith({
           result: null,
           error: `Deno subprocess execution timed out after ${timeoutMs}ms`,
         }),
       );
-    }, timeoutMs);
+    }, 1_000);
 
     // -----------------------------------------------------------------------
     // Message processing fiber — tool calls happen here, inside Effect
@@ -255,30 +265,39 @@ const executeInDeno = (
 
           switch (msg.type) {
             case "tool_call": {
-              const toolResult = yield* toolInvoker
-                .invoke({ path: msg.toolPath, args: msg.args })
-                .pipe(
-                  Effect.map(
-                    (value): HostToWorkerMessage => ({
-                      type: "tool_result",
-                      nonce,
-                      requestId: msg.requestId,
-                      ok: true,
-                      value,
-                    }),
-                  ),
-                  Effect.catchCause((cause) =>
-                    Effect.succeed<HostToWorkerMessage>({
-                      type: "tool_result",
-                      nonce,
-                      requestId: msg.requestId,
-                      ok: false,
-                      error: causeMessage(cause),
-                    }),
-                  ),
-                );
-
-              writeMessage(worker.stdin, toolResult);
+              // Symmetric bracket (mirrors ToolDispatcher.call): the decrement
+              // must run even if the invoke or write dies, or the watchdog's
+              // in-flight guard would suspend the timeout forever.
+              inFlight += 1;
+              yield* toolInvoker.invoke({ path: msg.toolPath, args: msg.args }).pipe(
+                Effect.map(
+                  (value): HostToWorkerMessage => ({
+                    type: "tool_result",
+                    nonce,
+                    requestId: msg.requestId,
+                    ok: true,
+                    value,
+                  }),
+                ),
+                Effect.catchCause((cause) =>
+                  Effect.succeed<HostToWorkerMessage>({
+                    type: "tool_result",
+                    nonce,
+                    requestId: msg.requestId,
+                    ok: false,
+                    error: causeMessage(cause),
+                  }),
+                ),
+                Effect.flatMap((toolResult) =>
+                  Effect.sync(() => writeMessage(worker.stdin, toolResult)),
+                ),
+                Effect.ensuring(
+                  Effect.sync(() => {
+                    inFlight -= 1;
+                    lastReturnedAt = Date.now();
+                  }),
+                ),
+              );
               break;
             }
 
@@ -318,7 +337,7 @@ const executeInDeno = (
     const output = yield* Deferred.await(result).pipe(
       Effect.ensuring(
         Effect.gen(function* () {
-          clearTimeout(timer);
+          clearInterval(timer);
           yield* Fiber.interrupt(processFiber);
           worker.dispose();
         }),