feat(eval): add read-only batch-evaluation CLI (get, list)

package.json

@@ -51,11 +51,13 @@
   "dependencies": {
     "@aws-sdk/client-bedrock-agentcore": "^3.1092.0",
     "@aws-sdk/client-bedrock-agentcore-control": "^3.1079.0",
+    "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",
     "@aws-sdk/client-iam": "^3.1080.0",
     "@smithy/core": "3.29.3",
     "@tanstack/react-query": "^5.101.2",
     "cli-truncate": "^6.1.1",
     "commander": "^15.0.0",
+    "handlebars": "^4.7.9",
     "ink": "^7.1.0",
     "ink-scroll-view": "^0.3.7",
     "lodash": "^4.18.1",
@@ -65,7 +67,6 @@
     "string-width": "^8.2.2",
     "winston": "^3.19.0",
     "winston-daily-rotate-file": "^5.0.0",
-    "handlebars": "^4.7.9",
     "zod": "^4.4.3"
   }
 }

src/core/batchEvaluationResults.test.ts

@@ -0,0 +1,167 @@
+import { test, expect } from "bun:test";
+import type { CloudWatchLogsClient, OutputLogEvent } from "@aws-sdk/client-cloudwatch-logs";
+import { createSilentLogger } from "../testing";
+import {
+  isTerminalStatus,
+  parseEvaluationLogEvent,
+  readEvaluationResults,
+} from "./batchEvaluationResults";
+
+// fakeLogs returns a CloudWatchLogsClient that serves `events` as a single page,
+// then signals exhaustion by echoing the same nextForwardToken on the next call —
+// exactly how GetLogEvents ends pagination. Records the tokens it was called with.
+function fakeLogs(events: OutputLogEvent[]): CloudWatchLogsClient {
+  let served = false;
+  return {
+    send: async () => {
+      if (!served) {
+        served = true;
+        return { events, nextForwardToken: "t-end" };
+      }
+      return { events: [], nextForwardToken: "t-end" }; // token unchanged → done
+    },
+  } as unknown as CloudWatchLogsClient;
+}
+
+// fakePagedLogs serves each element of `pages` on successive calls, advancing the
+// forward token per page and repeating the last token once to end. Captures every
+// nextToken the caller sent, so a test can assert the loop paged correctly.
+function fakePagedLogs(pages: OutputLogEvent[][]): {
+  client: CloudWatchLogsClient;
+  tokens: (string | undefined)[];
+} {
+  const tokens: (string | undefined)[] = [];
+  let call = 0;
+  const client = {
+    send: async (command: { input: { nextToken?: string } }) => {
+      tokens.push(command.input.nextToken);
+      const i = call++;
+      if (i < pages.length) return { events: pages[i], nextForwardToken: `t-${i}` };
+      return { events: [], nextForwardToken: `t-${pages.length - 1}` }; // repeat last → done
+    },
+  } as unknown as CloudWatchLogsClient;
+  return { client, tokens };
+}
+
+// A realistic stream shaped after the real `gen_ai.evaluation.result` records
+// (see the recorded fixture below): the level is
+// attributes["aws.bedrock_agentcore.evaluation_level"] (Title-case), session.id
+// sits under attributes, and the trace id is the top-level camelCase `traceId`.
+// One SESSION-level and one TRACE-level record, plus a non-JSON control line.
+const EVENTS: OutputLogEvent[] = [
+  {
+    message: JSON.stringify({
+      attributes: {
+        "gen_ai.evaluation.name": "Builtin.Helpfulness",
+        "aws.bedrock_agentcore.evaluation_level": "Session",
+        "session.id": "session-orders-123",
+        "gen_ai.evaluation.score.value": 5,
+        "gen_ai.evaluation.score.label": "helpful",
+        "gen_ai.evaluation.explanation": "Directly answered with tracking detail.",
+      },
+    }),
+  },
+  {
+    message: JSON.stringify({
+      traceId: "4bf92f3577b34da6a3ce929d0e0e4736",
+      attributes: {
+        "gen_ai.evaluation.name": "Builtin.Faithfulness",
+        "aws.bedrock_agentcore.evaluation_level": "Trace",
+        "session.id": "session-orders-123",
+        "gen_ai.evaluation.score.value": 4,
+        "gen_ai.evaluation.score.label": "faithful",
+        "gen_ai.evaluation.explanation": "Grounded in the tool result.",
+      },
+    }),
+  },
+  { message: "AWS log control line, not JSON" },
+];
+
+test("isTerminalStatus recognizes the terminal arm only", () => {
+  for (const s of ["COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "STOPPED"]) {
+    expect(isTerminalStatus(s)).toBe(true);
+  }
+  for (const s of ["IN_PROGRESS", "PENDING", "STOPPING", "DELETING", undefined]) {
+    expect(isTerminalStatus(s)).toBe(false);
+  }
+});
+
+test("readEvaluationResults keeps level + scope so sessions and traces are distinguishable", async () => {
+  const results = await readEvaluationResults(fakeLogs(EVENTS), "lg", "ls", createSilentLogger());
+
+  // The non-JSON control line is skipped; the two evaluation records parse.
+  expect(results).toHaveLength(2);
+  expect(results[0]).toMatchObject({
+    evaluatorId: "Builtin.Helpfulness",
+    level: "Session",
+    sessionId: "session-orders-123",
+    score: 5,
+    label: "helpful",
+  });
+  expect(results[0]?.traceId).toBeUndefined();
+  expect(results[1]).toMatchObject({
+    evaluatorId: "Builtin.Faithfulness",
+    level: "Trace",
+    sessionId: "session-orders-123",
+    traceId: "4bf92f3577b34da6a3ce929d0e0e4736",
+  });
+  expect(results.map((r) => r.level)).toEqual(["Session", "Trace"]);
+});
+
+test("readEvaluationResults follows pagination until the forward token stops advancing", async () => {
+  const page = (name: string): OutputLogEvent => ({
+    message: JSON.stringify({
+      attributes: {
+        "gen_ai.evaluation.name": name,
+        "aws.bedrock_agentcore.evaluation_level": "Trace",
+        "session.id": "s1",
+      },
+    }),
+  });
+  const { client, tokens } = fakePagedLogs([
+    [page("Builtin.Correctness")],
+    [page("Builtin.Helpfulness")],
+    [page("Builtin.Faithfulness")],
+  ]);
+
+  const results = await readEvaluationResults(client, "lg", "ls", createSilentLogger());
+
+  // All three pages' records are collected.
+  expect(results.map((r) => r.evaluatorId)).toEqual([
+    "Builtin.Correctness",
+    "Builtin.Helpfulness",
+    "Builtin.Faithfulness",
+  ]);
+  // First call has no token; later calls carry the prior page's forward token; a
+  // final call detects the repeated token and stops.
+  expect(tokens).toEqual([undefined, "t-0", "t-1", "t-2"]);
+});
+
+// Real-log-shape validation lives in the fixture-backed command-flow test
+// (batch-evaluation.fixture.test.tsx), where RECORD=1 captures a live GetLogEvents
+// response and matchGolden pins the parsed output. This file stays a pure unit
+// test over inline synthetic events, matching the rest of src/core.
+
+test("readEvaluationResults skips lines without an evaluation name", async () => {
+  const results = await readEvaluationResults(
+    fakeLogs([
+      { message: JSON.stringify({ attributes: { "some.other.metric": 1 } }) },
+      { message: "" },
+      { message: undefined },
+    ]),
+    "lg",
+    "ls",
+    createSilentLogger(),
+  );
+  expect(results).toEqual([]);
+});
+
+test("parseEvaluationLogEvent warns on and skips an unparseable line", () => {
+  const warnings: string[] = [];
+  const logger = createSilentLogger();
+  logger.warn = (...msgs: string[]) => warnings.push(msgs.join(" "));
+
+  expect(parseEvaluationLogEvent("AWS log control line, not JSON", logger)).toBeNull();
+  expect(warnings).toHaveLength(1);
+  expect(warnings[0]).toContain("unparseable");
+});

src/core/batchEvaluationResults.tsx

@@ -0,0 +1,117 @@
+import { GetLogEventsCommand, type CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs";
+import type { BatchEvaluationResultEntry } from "../handlers/eval/types";
+import type { Logger } from "../logging";
+
+// Per-session batch-evaluation result retrieval, mirroring
+// core/onlineEvalExecutionRole.tsx's pattern: a self-contained module that takes
+// an injected AWS client (here CloudWatchLogsClient) and owns one slice of Core's
+// behavior. A completed batch evaluation writes each score as an OTel-shaped log
+// record to a per-job CloudWatch stream; this module reads that stream and parses
+// the records. EvalClient calls readEvaluationResults with the client from
+// `this.clients.logs(...)` and the log group + stream from the job's outputConfig.
+
+// Terminal batch-evaluation statuses — after these, results are final and worth
+// retrieving. Mirrors the AgentCore BatchEvaluationStatus enum's terminal arm.
+const TERMINAL_STATUSES = new Set(["COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "STOPPED"]);
+
+export function isTerminalStatus(status?: string): boolean {
+  return !!status && TERMINAL_STATUSES.has(status);
+}
+
+// GetLogEvents returns at most 1 MB / 10,000 events per call, so a job with many
+// results spans multiple pages. This caps the page loop as a safety valve against
+// a non-advancing token (see below); at 10k events/page it allows ~1M results,
+// far beyond the 500-session job limit.
+const MAX_RESULT_PAGES = 100;
+
+// readEvaluationResults reads and parses the per-session/-trace/-tool scores from
+// a completed batch evaluation's CloudWatch result stream, following pagination to
+// completion. The caller supplies the log group and stream name from the job's
+// GetBatchEvaluation outputConfig (the service-selected values — we do not derive
+// the stream name, since its format is not part of the SDK contract).
+export async function readEvaluationResults(
+  logs: CloudWatchLogsClient,
+  logGroupName: string,
+  logStreamName: string,
+  logger: Logger,
+): Promise<BatchEvaluationResultEntry[]> {
+  const results: BatchEvaluationResultEntry[] = [];
+
+  // Page forward from the head. GetLogEvents echoes the input token back as
+  // nextForwardToken once the stream is exhausted, so the loop ends when the
+  // token stops advancing. startFromHead is only honored on the first call (no
+  // token); subsequent calls are positioned by the token.
+  let token: string | undefined;
+  for (let page = 0; page < MAX_RESULT_PAGES; page++) {
+    const response = await logs.send(
+      new GetLogEventsCommand({
+        logGroupName,
+        logStreamName,
+        startFromHead: true,
+        nextToken: token,
+      }),
+    );
+
+    for (const event of response.events ?? []) {
+      if (!event.message) continue;
+      const entry = parseEvaluationLogEvent(event.message, logger);
+      if (entry) results.push(entry);
+    }
+
+    const next = response.nextForwardToken;
+    if (!next || next === token) return results; // exhausted: token stopped advancing
+    token = next;
+  }
+
+  logger.warn(
+    `stopped reading batch-evaluation results after ${MAX_RESULT_PAGES} pages; results may be truncated`,
+  );
+  return results;
+}
+
+// parseEvaluationLogEvent turns one CloudWatch result-log message into a result
+// entry, or null for non-JSON / non-evaluation lines (log control lines, blank
+// messages). AgentCore emits each score as an OTel-shaped log record named
+// `gen_ai.evaluation.result`: the `gen_ai.evaluation.*` scores and the
+// `session.id` live under `attributes`, the trace id is the top-level `traceId`,
+// and the level is `attributes["aws.bedrock_agentcore.evaluation_level"]` (e.g.
+// "Trace" / "Session"). Field names verified against a recorded result stream
+// (src/core/__fixtures__/batch-eval-result-log-events.json). We keep `level` and
+// the ids — the old CLI dropped them, flattening every level into one list.
+export function parseEvaluationLogEvent(
+  message: string,
+  logger: Logger,
+): BatchEvaluationResultEntry | null {
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = JSON.parse(message) as Record<string, unknown>;
+  } catch {
+    // Swallow rather than throw: CloudWatch result streams interleave non-JSON
+    // control lines with the evaluation records, so one unparseable line is
+    // expected noise — failing here would drop every result for the job over it.
+    // Warn (not silent) so a systematic format change is still visible in logs.
+    logger.warn("skipping unparseable batch-evaluation result log line");
+    return null;
+  }
+  const attrs = (parsed["attributes"] ?? {}) as Record<string, unknown>;
+  const evaluatorId = attrs["gen_ai.evaluation.name"] as string | undefined;
+  if (!evaluatorId) return null;
+
+  const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined);
+  // Keys are read verbatim as the recorded stream emits them: scores, level,
+  // session id, and tool name under `attributes`; traceId/spanId at the top
+  // level. If AgentCore renames one, the corresponding field goes undefined and
+  // the fixture-replay test fails — the signal to update the key here.
+  return {
+    evaluatorId,
+    level: str(attrs["aws.bedrock_agentcore.evaluation_level"]),
+    sessionId: str(attrs["session.id"]),
+    traceId: str(parsed["traceId"]),
+    spanId: str(parsed["spanId"]),
+    toolName: str(attrs["gen_ai.tool.name"]),
+    score: attrs["gen_ai.evaluation.score.value"] as number | undefined,
+    label: str(attrs["gen_ai.evaluation.score.label"]),
+    explanation: str(attrs["gen_ai.evaluation.explanation"]),
+    error: str(attrs["gen_ai.evaluation.error"]),
+  };
+}

src/core/core.test.ts

@@ -4,6 +4,7 @@ import {
   type BedrockAgentCoreControlClient,
 } from "@aws-sdk/client-bedrock-agentcore-control";
 import type { IAMClient } from "@aws-sdk/client-iam";
+import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs";
 import {
   GetEventCommand,
   GetMemoryRecordCommand,
@@ -76,6 +77,9 @@ function fakeData(config: ClientConfig): BedrockAgentCoreClient {
 function fakeIam(config: ClientConfig): IAMClient {
   return { config, kind: "iam" } as unknown as IAMClient;
 }
+function fakeLogs(config: ClientConfig): CloudWatchLogsClient {
+  return { config, kind: "logs" } as unknown as CloudWatchLogsClient;
+}
 
 function coreWithDataSend(
   send: (command: unknown, options: unknown) => Promise<unknown>,
@@ -86,6 +90,7 @@ function coreWithDataSend(
     createDataClient: (config) =>
       ({ config, kind: "data", send }) as unknown as BedrockAgentCoreClient,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger,
   });
 }
@@ -121,6 +126,7 @@ function customJwtCore(
         },
       }) as unknown as BedrockAgentCoreClient,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     fetch,
     logger,
   });
@@ -135,6 +141,7 @@ test("control() constructs a client once per config and caches it", () => {
     },
     createDataClient: fakeData,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
 
@@ -154,6 +161,7 @@ test("control() builds a distinct client per distinct config", () => {
     },
     createDataClient: fakeData,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
 
@@ -177,6 +185,7 @@ test("data() caches independently of control()", () => {
       return fakeData(config);
     },
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
 
@@ -194,6 +203,7 @@ test("exposes feature sub-clients", () => {
     createControlClient: fakeControl,
     createDataClient: fakeData,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
   expect(core.harness).toBeDefined();
@@ -346,6 +356,7 @@ test("getRuntime sends the abort signal to the control client", async () => {
       }) as unknown as BedrockAgentCoreControlClient,
     createDataClient: fakeData,
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
   const controller = new AbortController();
@@ -380,6 +391,7 @@ test("invokeHarness sends an InvokeHarnessCommand on the data client with the ab
       } as unknown as BedrockAgentCoreClient;
     },
     createIamClient: fakeIam,
+    createLogsClient: fakeLogs,
     logger: createSilentLogger(),
   });
 

src/core/datasetDownload.test.ts

@@ -32,7 +32,7 @@ function stubClients(dataset: Record<string, unknown>): AwsClients {
     throw new Error(`unexpected command: ${(command as object).constructor.name}`);
   };
   const client = { send } as never;
-  return { control: () => client, data: () => client, iam: () => client };
+  return { control: () => client, data: () => client, iam: () => client, logs: () => client };
 }
 
 describe("EvalClient.downloadDataset", () => {