feat(agent): OpenUI bindings — library/fragment builders, toUIOutput, getUiStream() (DEV-773)

.changeset/openui-bindings.md

@@ -0,0 +1,74 @@
+---
+'@openrouter/agent': minor
+---
+
+OpenUI bindings: a component-library model (`defineComponent`, `createLibrary`, `componentProps`), a typed fragment builder (`fragment`, `uiRef`, `uiState`, `uiBuiltin`), the `openui` plugin helper, `serializeExpr`/`OPENUI_LANG_DIALECT` for emitting OpenUI Lang, a `toUiOutput` tool option that renders a tool's result as UI, and `ModelResult.getUiStream()` for consuming fragments as they arrive.
+
+A tool declares how its output renders, and the caller streams the fragments:
+
+```ts
+import {
+  callModel,
+  createLibrary,
+  defineComponent,
+  fragment,
+  openui,
+  tool,
+} from '@openrouter/agent';
+import { z } from 'zod/v4';
+
+const library = createLibrary([
+  defineComponent({
+    name: 'Card',
+    description: 'Container with a title',
+    props: z.object({
+      title: z.string(),
+      children: z.array(z.unknown()).optional(),
+    }),
+  }),
+  defineComponent({
+    name: 'Text',
+    props: z.object({
+      value: z.string(),
+    }),
+  }),
+]);
+
+const ui = fragment(library);
+
+const weather = tool({
+  name: 'weather',
+  inputSchema: z.object({
+    city: z.string(),
+  }),
+  outputSchema: z.object({
+    summary: z.string(),
+  }),
+  execute: ({ city }) => ({
+    summary: `Clear in ${city}`,
+  }),
+  // Renders the tool's result instead of leaving the model to describe it.
+  toUiOutput: ({ input, output }) =>
+    ui.Card(input.city, [
+      ui.Text(output.summary),
+    ]),
+});
+
+const result = callModel(client, {
+  model: 'anthropic/claude-sonnet-4.5',
+  input: 'What is the weather in Lisbon?',
+  tools: [
+    weather,
+  ],
+  plugins: [
+    openui(library),
+  ],
+});
+
+for await (const event of result.getUiStream()) {
+  if (event.type === 'fragment') {
+    // source: 'root = Card("Lisbon", [Text("Clear in Lisbon")])'
+    console.log(event.source);
+  }
+}
+```

packages/agent/README.md

@@ -114,6 +114,7 @@ What each stream emits:
 | `getToolStream()` | tool-call **argument deltas**; `preliminary_result` events for generator tools — *not* execution results |
 | `getToolCallsStream()` | parsed tool calls as they complete |
 | `getItemsStream()` | all output items (messages, function calls, …) — output items **only**, no usage/response metadata |
+| `getUiStream()` | OpenUI events — `statement` / `fragment` / `document` — from tools declaring `toUiOutput` and from the `openui` plugin |
 | `getFullResponsesStream()` | every response event, including `tool.result` / `tool.call_output` execution events, and each round's `response.completed` (with that round's usage block) |
 
 #### Usage across a multi-round tool loop
@@ -150,7 +151,6 @@ Same `SessionUsageTotals` shape and numbers as the `SessionEnd` hook's
 `totalUsage`. For **per-call** granularity use the `PostModelCall` hook (one
 emit per model call, with `turnType`/`turnNumber`) or read each round's
 `response.completed` off `getFullResponsesStream()`.
-
 ### Tool Types
 
 The `tool()` factory creates type-safe tools with full Zod schema inference. In addition to the legacy kinds below, the unified `run` interface with `lifecycle: 'sync' | 'background' | 'deferred'` covers [async tools](#async-tools) whose results arrive after the tool round, and `tool.agent()` creates [subagent tools](#agent-tools-subagents).

packages/agent/src/index.ts

@@ -201,6 +201,42 @@ export {
   buildNextTurnParamsContext,
   executeNextTurnParamsFunctions,
 } from './lib/next-turn-params.js';
+export type {
+  ComponentDefinition,
+  CreateLibraryOptions,
+  FragmentArg,
+  FragmentBuilder,
+  FragmentNode,
+  OpenUiPlugin,
+  OpenUiWireComponent,
+  PropSignature,
+  UiBuiltinOptions,
+  UiDocumentEvent,
+  UiExpr,
+  UiFragment,
+  UiFragmentEvent,
+  UiLibrary,
+  UiLiteralValue,
+  UiStatementEvent,
+  UiStreamEvent,
+} from './lib/openui/index.js';
+// OpenUI (generative UI) bindings: library model, fragment builder, plugin helper
+export {
+  componentProps,
+  createLibrary,
+  defineComponent,
+  fragment,
+  OPENUI_BUILTIN_COMPONENTS,
+  OPENUI_LANG_DIALECT,
+  OPENUI_ROOT_REF,
+  OPENUI_WIRE_EVENT,
+  openui,
+  serializeExpr,
+  translateUiEvent,
+  uiBuiltin,
+  uiRef,
+  uiState,
+} from './lib/openui/index.js';
 // Stop condition helpers
 export {
   finishReasonIs,
@@ -294,8 +330,10 @@ export type {
   ToolStreamEvent,
   ToolTaskHandle,
   ToolTaskStatus,
+  ToolUiFragmentEvent,
   ToolWithExecute,
   ToolWithGenerator,
+  ToUiOutputFunction,
   TurnContext,
   TurnEndEvent,
   TurnStartEvent,
@@ -325,6 +363,7 @@ export {
   isToolCallOutputEvent,
   isToolPreliminaryResultEvent,
   isToolResultEvent,
+  isToolUiFragmentEvent,
   isTurnEndEvent,
   isTurnStartEvent,
   isUnifiedTool,

packages/agent/src/inner-loop/resume-tool-results.ts

@@ -221,6 +221,12 @@ export async function resumeToolResults<TTools extends readonly Tool[]>(
   const settledIds = new Set(state.settledAsyncCallIds ?? []);
 
   const envelopes: models.BaseInputsUnion[] = [];
+  const uiToolResults: Array<{
+    callId: string;
+    name: string;
+    input: Record<string, unknown>;
+    output: unknown;
+  }> = [];
   /** callId → the terminal lifecycle status persisted for that entry. */
   const settledNow = new Map<string, ToolTaskStatus>();
 
@@ -249,6 +255,7 @@ export async function resumeToolResults<TTools extends readonly Tool[]>(
     const envelope = buildResumeEnvelope(entry, task, request.tools);
 
     envelopes.push(buildTaskResultMessage(envelope));
+    collectUiToolResult(uiToolResults, envelope, task);
     // Persist the entry's real terminal status. 'expired' / 'timed_out'
     // have no ToolTaskStatus member — they persist as 'failed'.
     settledNow.set(
@@ -313,7 +320,7 @@ export async function resumeToolResults<TTools extends readonly Tool[]>(
 
   // Continue the conversation: the envelopes are already in persisted
   // history, so no fresh input is supplied.
-  return callModel(
+  const result = callModel(
     client,
     {
       ...request.run,
@@ -325,6 +332,28 @@ export async function resumeToolResults<TTools extends readonly Tool[]>(
     },
     options,
   );
+  result.queueUiToolResults(uiToolResults);
+  return result;
+}
+
+function collectUiToolResult(
+  results: Array<{
+    callId: string;
+    name: string;
+    input: Record<string, unknown>;
+    output: unknown;
+  }>,
+  envelope: ToolTaskResultEnvelope,
+  task: PendingAsyncTool,
+): void {
+  if (envelope.status === 'completed' && task.input !== undefined) {
+    results.push({
+      callId: task.callId,
+      name: task.name,
+      input: task.input,
+      output: envelope.result,
+    });
+  }
 }
 
 /**

packages/agent/src/lib/async-params.ts

@@ -4,6 +4,7 @@ import type { DoomLoopOption } from './doom-loop.js';
 import type { HooksManager } from './hooks-manager.js';
 import type { InlineHookConfig } from './hooks-types.js';
 import type { Item } from './item-types.js';
+import type { OpenUiPlugin } from './openui/plugin.js';
 import type { ContextInput } from './tool-context.js';
 import type {
   ParsedToolCall,
@@ -56,11 +57,16 @@ type BaseCallModelInput<
   TTools extends readonly Tool[] = readonly Tool[],
   TShared extends Record<string, unknown> = Record<string, never>,
 > = {
-  [K in keyof Omit<models.ResponsesRequest, 'stream' | 'tools' | 'input'>]?: FieldOrAsyncFunction<
-    models.ResponsesRequest[K]
-  >;
+  [K in keyof Omit<
+    models.ResponsesRequest,
+    'stream' | 'tools' | 'input' | 'plugins'
+  >]?: FieldOrAsyncFunction<models.ResponsesRequest[K]>;
 } & {
   input: FieldOrAsyncFunction<Item[]> | string;
+  /** Responses plugins, including the OpenUI binding pending SDK schema regeneration. */
+  plugins?: FieldOrAsyncFunction<
+    Array<NonNullable<models.ResponsesRequest['plugins']>[number] | OpenUiPlugin>
+  >;
   tools?: TTools;
   stopWhen?: StopWhen<TTools>;
   /** Typed context data passed to tools via contextSchema. Includes optional `shared` key. */

packages/agent/src/lib/async-tool-registry.ts

@@ -147,6 +147,7 @@ export class AsyncToolRegistry {
     callId: string;
     taskId: string;
     name: string;
+    input: Record<string, unknown>;
     expiresAt?: number;
     pollAfterMs?: number;
   }): ToolTask {
@@ -155,6 +156,7 @@ export class AsyncToolRegistry {
       callId: entry.callId,
       toolName: entry.name,
       mode: 'defer',
+      input: entry.input,
       ...(entry.expiresAt !== undefined && {
         expiresAt: entry.expiresAt,
       }),
@@ -301,6 +303,9 @@ export class AsyncToolRegistry {
       mode: t.mode,
       status: t.status,
       startedAt: t.startedAt,
+      ...(t.input !== undefined && {
+        input: t.input,
+      }),
       ...(t.expiresAt !== undefined && {
         expiresAt: t.expiresAt,
       }),