diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index e5b05974c..81cef85c7 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -2,7 +2,7 @@ name: Deploy to GitHub Pages on: push: - branches: [ "main" ] + branches: [master] workflow_dispatch: # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages @@ -37,4 +37,4 @@ jobs: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 577a4f199..d49ff0c71 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ node_modules/ .eslintcache # build output -dist/ \ No newline at end of file +dist/ +# macOS +.DS_Store diff --git a/AGENTS.md b/AGENTS.md index d509d5b6b..ad2e36816 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ ## Build, Lint, and Test Commands - **Build:** - `bun run build` (uses tsup) + `bun run build` (uses tsdown) - **Dev:** `bun run dev` - **Lint:** @@ -13,7 +13,7 @@ - **Test all:** `bun test` - **Test single file:** - `bun test tests/claude-request.test.ts` + `bun test tests/anthropic-request.test.ts` - **Start (prod):** `bun run start` diff --git a/Dockerfile b/Dockerfile index 1265220ef..1cbcabe8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,13 +13,14 @@ WORKDIR /app COPY ./package.json ./bun.lock ./ RUN bun install --frozen-lockfile --production --ignore-scripts --no-cache -COPY --from=builder /app/dist ./dist +COPY --from=builder --chown=bun:bun /app/dist ./dist EXPOSE 4141 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD wget --spider -q http://localhost:4141/ || exit 1 -COPY entrypoint.sh /entrypoint.sh +COPY --chown=bun:bun entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh +USER bun ENTRYPOINT ["/entrypoint.sh"] diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 000000000..324024707 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,83 @@ +# Add `models` command, formalize `/responses` support, harden defaults + +This branch is **9 commits ahead of `master`**. It adds a non-interactive +`models` command, formalizes and hardens the `/responses` adapter the chat +handler already depended on, makes the server local-only by default, and fixes +documentation, Docker, and CI drift. + +## Summary of changes + +### Features + +- **`models` CLI command** (`e273b3f`) — inspect the current Copilot models and + their supported endpoints without starting the server. Useful for + non-interactive deployments. Supports `--json`, `--github-token`, + `--account-type`, `--show-token`, and `--proxy-env`. +- **`/responses` endpoint routing** (`fb77680`) — models that only advertise + `/responses` (and not `/chat/completions`) are now transparently routed + through a Responses API adapter that converts both non-streaming and streaming + responses back into Chat Completions shape. +- **Tool calls over `/responses`** (`a695374`) — chat-completions tools and + `tool_choice` are mapped into the Responses API shape, assistant `tool_calls` + and tool results become `function_call`/`function_call_output` input items, and + `function_call` outputs and `function_call_arguments` deltas are translated + back into `tool_calls` (non-streaming and streaming). Replaces the earlier + drop-with-warning behavior. +- **Local-only by default** (`44c74c7`) — the server now binds to `127.0.0.1` + unless `--host` is passed, so it is not reachable from other machines out of + the box. A warning prints when binding to a non-local host. The Docker + entrypoint passes `--host 0.0.0.0` so published ports keep working. + +### Fixes + +- **Lazy VS Code version fetch** (`3d24eef`) — the VS Code version is now fetched + only when called, not as an import-time network side effect. +- **Tolerate invalid tool-call JSON** (`dec3694`) — malformed tool arguments from + Copilot no longer crash `translateToAnthropic`; the tool input falls back to + an empty object. +- **Harden `/responses` stream adapter** (`e8f316c`) — malformed or non-JSON SSE + events (e.g. keepalives) are skipped instead of killing the stream, and a + warning is logged when tool definitions/calls are dropped for + `/responses`-only models. + +### Tests + +- **Anthropic edge cases** (`dec3694`) — images, `tool_result` ordering + (including multiple results per message), mixed text/tool streaming block + transitions, invalid tool JSON, and cache-token accounting. +- **`/responses` adapter** (`fb77680`, `e8f316c`) — endpoint selection, + non-streaming conversion, streaming event conversion, malformed-event + skipping, and unknown-event handling. +- **`count_tokens` handler** (`ddede24`) — drives the real Hono route and locks + the claude (1.15) and grok (1.03) multipliers, the 346/480-token tool + overhead, the `mcp__`/`claude-code` beta exemption, and the invalid-JSON + fallback. + +### Docs / chore + +- **Non-root Docker** (`58249ba`) — `Dockerfile` runs as `USER bun`; README + volume path corrected to `/home/bun/.local/share/copilot-api`. +- **Docs drift** — `AGENTS.md` updated (`tsup` → `tsdown`, corrected test + command); README documents the `models` command, the `--host` flag, and + single-account/local-use assumptions. +- **GitHub Pages workflow** — now deploys from `master` (the branch this fork and + upstream actually expose). + +## Verification + +- `bun run lint:all` — passing +- `bun run typecheck` — passing +- `bun test` — **60 passing** (was 29) +- `bun run build` — passing + +## Notes for the maintainer + +- The `/responses` adapter now forwards function tool calls, but image content + is still stringified (`[image: ]`) rather than sent as a structured + input-image part. The tool mapping was implemented against the documented + OpenAI Responses API shapes; it should be validated against a live + `/responses`-only Copilot model, since Copilot's variant may differ in field + names (e.g. `call_id` vs `id`). +- Docker bind-mount permissions: under `USER bun`, the host directory mounted at + `/home/bun/.local/share/copilot-api` must be writable by uid 1000. If you hit + an `EACCES` on first run, `chown 1000:1000 ./copilot-data` on the host. diff --git a/README.md b/README.md index 0d36c13c9..6cce502cc 100644 --- a/README.md +++ b/README.md @@ -74,25 +74,25 @@ mkdir -p ./copilot-data # Run the container with a bind mount to persist the token # This ensures your authentication survives container restarts -docker run -p 4141:4141 -v $(pwd)/copilot-data:/root/.local/share/copilot-api copilot-api +docker run -p 4141:4141 -v $(pwd)/copilot-data:/home/bun/.local/share/copilot-api copilot-api ``` > **Note:** -> The GitHub token and related data will be stored in `copilot-data` on your host. This is mapped to `/root/.local/share/copilot-api` inside the container, ensuring persistence across restarts. +> The GitHub token and related data will be stored in `copilot-data` on your host. This is mapped to `/home/bun/.local/share/copilot-api` inside the container, ensuring persistence across restarts. ### Docker with Environment Variables You can pass the GitHub token directly to the container using environment variables: ```sh -# Build with GitHub token -docker build --build-arg GH_TOKEN=your_github_token_here -t copilot-api . +# Build the image +docker build -t copilot-api . # Run with GitHub token docker run -p 4141:4141 -e GH_TOKEN=your_github_token_here copilot-api # Run with additional options -docker run -p 4141:4141 -e GH_TOKEN=your_token copilot-api start --verbose --port 4141 +docker run -p 4141:4141 -e GH_TOKEN=your_token copilot-api --verbose --port 4141 ``` ### Docker Compose Example @@ -142,6 +142,7 @@ Copilot API now uses a subcommand structure with these main commands: - `start`: Start the Copilot API server. This command will also handle authentication if needed. - `auth`: Run GitHub authentication flow without starting the server. This is typically used if you need to generate a token for use with the `--github-token` option, especially in non-interactive environments. +- `models`: List the current GitHub Copilot models available to the API. This is useful for non-interactive deployments where you want to inspect model support without starting the server. - `check-usage`: Show your current GitHub Copilot usage and quota information directly in the terminal (no server required). - `debug`: Display diagnostic information including version, runtime details, file paths, and authentication status. Useful for troubleshooting and support. @@ -154,6 +155,7 @@ The following command line options are available for the `start` command: | Option | Description | Default | Alias | | -------------- | ----------------------------------------------------------------------------- | ---------- | ----- | | --port | Port to listen on | 4141 | -p | +| --host | Host to bind to. Defaults to local-only. Use 0.0.0.0 to expose to the network (e.g. inside Docker) | 127.0.0.1 | none | | --verbose | Enable verbose logging | false | -v | | --account-type | Account type to use (individual, business, enterprise) | individual | -a | | --manual | Enable manual request approval | false | none | @@ -171,6 +173,17 @@ The following command line options are available for the `start` command: | --verbose | Enable verbose logging | false | -v | | --show-token | Show GitHub token on auth | false | none | +### Models Command Options + +| Option | Description | Default | Alias | +| -------------- | ----------------------------------------------------------------------------- | ---------- | ----- | +| --verbose | Enable verbose logging | false | -v | +| --account-type | Account type to use (individual, business, enterprise) | individual | -a | +| --github-token | Provide GitHub token directly (must be generated using the `auth` subcommand) | none | -g | +| --show-token | Show GitHub and Copilot tokens on fetch | false | none | +| --proxy-env | Initialize proxy from environment variables | false | none | +| --json | Output raw model response as JSON | false | none | + ### Debug Command Options | Option | Description | Default | Alias | @@ -307,10 +320,11 @@ Here is an example `.claude/settings.json` file: "env": { "ANTHROPIC_BASE_URL": "http://localhost:4141", "ANTHROPIC_AUTH_TOKEN": "dummy", - "ANTHROPIC_MODEL": "gpt-4.1", - "ANTHROPIC_DEFAULT_SONNET_MODEL": "gpt-4.1", - "ANTHROPIC_SMALL_FAST_MODEL": "gpt-4.1", - "ANTHROPIC_DEFAULT_HAIKU_MODEL": "gpt-4.1", + "ANTHROPIC_MODEL": "claude-sonnet-4.6", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-sonnet-4.6", + "ANTHROPIC_DEFAULT_OPUS_MODEL": "claude-opus-5", + "ANTHROPIC_SMALL_FAST_MODEL": "claude-haiku-4.5", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "claude-haiku-4.5", "DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1" }, @@ -322,6 +336,8 @@ Here is an example `.claude/settings.json` file: } ``` +> **Note:** Available models depend on your GitHub Copilot subscription tier. Run `bun run models` to see the models currently exposed by your account (for example, Copilot Pro+ adds access to newer Claude Opus/Sonnet 5 and GPT‑5.5/5.6‑series models). + You can find more options here: [Claude Code settings](https://docs.anthropic.com/en/docs/claude-code/settings#environment-variables) You can also read more about IDE integration here: [Add Claude Code to your IDE](https://docs.anthropic.com/en/docs/claude-code/ide-integrations) @@ -342,6 +358,12 @@ bun run dev bun run start ``` +### List Available Models + +```sh +bun run models +``` + ## Usage Tips - To avoid hitting GitHub Copilot's rate limits, you can use the following flags: @@ -349,3 +371,5 @@ bun run start - `--rate-limit `: Enforces a minimum time interval between requests. For example, `copilot-api start --rate-limit 30` will ensure there's at least a 30-second gap between requests. - `--wait`: Use this with `--rate-limit`. It makes the server wait for the cooldown period to end instead of rejecting the request with an error. This is useful for clients that don't automatically retry on rate limit errors. - If you have a GitHub business or enterprise plan account with Copilot, use the `--account-type` flag (e.g., `--account-type business`). See the [official documentation](https://docs.github.com/en/enterprise-cloud@latest/copilot/managing-copilot/managing-github-copilot-in-your-organization/managing-access-to-github-copilot-in-your-organization/managing-github-copilot-access-to-your-organizations-network#configuring-copilot-subscription-based-network-routing-for-your-enterprise-or-organization) for more details. +- This server is designed for localhost or trusted local-network use. It is intentionally single-account: all clients share the same GitHub/Copilot token and model cache. +- The server binds to `127.0.0.1` by default, so it is not reachable from other machines unless you explicitly pass `--host` (for example `--host 0.0.0.0`). The Docker entrypoint passes `--host 0.0.0.0` so published ports work; restrict exposure on the host side with `-p 127.0.0.1:4141:4141` if you want it local-only. diff --git a/entrypoint.sh b/entrypoint.sh index dfe63c902..7d5cc4fdc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,6 +4,8 @@ if [ "$1" = "--auth" ]; then exec bun run dist/main.js auth else # Default command - exec bun run dist/main.js start -g "$GH_TOKEN" "$@" + # Bind to 0.0.0.0 inside the container so published ports work; + # restrict exposure on the host side (e.g. -p 127.0.0.1:4141:4141). + exec bun run dist/main.js start --host 0.0.0.0 -g "$GH_TOKEN" "$@" fi diff --git a/package.json b/package.json index a5adbb8e7..3b2744fa0 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "knip": "knip-bun", "lint": "eslint --cache", "lint:all": "eslint --cache .", + "models": "bun run ./src/main.ts models", "prepack": "bun run build", "prepare": "simple-git-hooks", "release": "bumpp && bun publish --access public", diff --git a/scripts/validate-responses-tools.sh b/scripts/validate-responses-tools.sh new file mode 100755 index 000000000..5156a7f4f --- /dev/null +++ b/scripts/validate-responses-tools.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# Validate /responses tool forwarding against a LIVE /responses-only Copilot model. +# +# This is the one thing the unit tests cannot prove: that copilot-api's +# chat-completions -> Responses API translation matches Copilot's actual wire +# format (e.g. call_id vs id) for a model that ONLY supports /responses. +# +# Prereqs: +# - copilot-api server running locally (default http://localhost:4141) +# - jq installed +# +# Usage: +# ./scripts/validate-responses-tools.sh +# BASE_URL=http://localhost:4141 MODEL=gpt-5.4-mini ./scripts/validate-responses-tools.sh +# +# Pick a model whose supported_endpoints is /responses-only. From `bun run models`, +# good candidates: gpt-5.4-mini, gpt-5.3-codex, gpt-5.5, gpt-5.6-sol, +# mai-code-1-flash-picker. + +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:4141}" +MODEL="${MODEL:-gpt-5.4-mini}" + +pass() { printf '\033[32m✓ %s\033[0m\n' "$1"; } +fail() { printf '\033[31m✗ %s\033[0m\n' "$1"; exit 1; } +info() { printf '\033[36m• %s\033[0m\n' "$1"; } + +info "Server: $BASE_URL" +info "Model: $MODEL" + +# 0. Confirm the model is actually /responses-only (otherwise this proves nothing). +endpoints="$(curl -sS --max-time 15 "$BASE_URL/v1/models" \ + | jq -r --arg m "$MODEL" '.data[] | select(.id==$m) | (.supported_endpoints // []) | join(",")' || true)" +if [ -z "$endpoints" ]; then + info "Note: /v1/models did not list supported_endpoints for $MODEL (the OpenAI-shaped models route omits them). Proceeding anyway." +else + info "supported_endpoints: $endpoints" + case "$endpoints" in + *"/chat/completions"*) info "WARNING: $MODEL also supports /chat/completions, so this will NOT exercise the /responses path. Pick a /responses-only model." ;; + esac +fi + +weather_tool='{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { "city": { "type": "string", "description": "City name" } }, + "required": ["city"] + } + } +}' + +# 1. Non-streaming: model should emit a tool_call we can parse. +info "Test 1: non-streaming tool call" +req1="$(jq -n --arg model "$MODEL" --argjson tool "$weather_tool" '{ + model: $model, + max_tokens: 256, + temperature: 0, + tools: [$tool], + tool_choice: "auto", + messages: [{ role: "user", content: "What is the weather in Boston? Use the get_weather tool." }] +}')" + +resp1="$(curl -sS --max-time 60 "$BASE_URL/v1/chat/completions" \ + -H 'content-type: application/json' -d "$req1")" + +echo "$resp1" | jq '.' >/dev/null 2>&1 || fail "Response 1 was not valid JSON: $resp1" + +tool_name="$(echo "$resp1" | jq -r '.choices[0].message.tool_calls[0].function.name // empty')" +tool_id="$(echo "$resp1" | jq -r '.choices[0].message.tool_calls[0].id // empty')" +tool_args="$(echo "$resp1" | jq -r '.choices[0].message.tool_calls[0].function.arguments // empty')" +finish="$(echo "$resp1" | jq -r '.choices[0].finish_reason // empty')" + +if [ -z "$tool_name" ]; then + fail "No tool_call returned. The model may have answered directly, or the /responses function_call mapping is wrong. Raw: $(echo "$resp1" | jq -c '.choices[0]')" +fi +[ "$tool_name" = "get_weather" ] || fail "Tool name mismatch: got '$tool_name'" +[ -n "$tool_id" ] || fail "tool_calls[0].id is empty (call_id mapping likely wrong)" +echo "$tool_args" | jq '.' >/dev/null 2>&1 || fail "tool arguments are not valid JSON: '$tool_args'" +pass "non-streaming tool call: name=$tool_name id=$tool_id finish=$finish args=$tool_args" + +# 2. Round-trip: feed the tool result back as a tool message; expect a normal answer. +info "Test 2: tool-result round-trip (function_call_output mapping)" +req2="$(jq -n --arg model "$MODEL" --argjson tool "$weather_tool" \ + --arg id "$tool_id" --arg args "$tool_args" '{ + model: $model, + max_tokens: 256, + temperature: 0, + tools: [$tool], + messages: [ + { role: "user", content: "What is the weather in Boston? Use the get_weather tool." }, + { role: "assistant", content: null, tool_calls: [ + { id: $id, type: "function", function: { name: "get_weather", arguments: $args } } + ]}, + { role: "tool", tool_call_id: $id, content: "Sunny, 72F" } + ] +}')" + +resp2="$(curl -sS --max-time 60 "$BASE_URL/v1/chat/completions" \ + -H 'content-type: application/json' -d "$req2")" + +echo "$resp2" | jq '.' >/dev/null 2>&1 || fail "Response 2 was not valid JSON: $resp2" +content2="$(echo "$resp2" | jq -r '.choices[0].message.content // empty')" +[ -n "$content2" ] || fail "No content after feeding tool result back (function_call_output mapping likely wrong). Raw: $(echo "$resp2" | jq -c '.choices[0]')" +pass "tool-result round-trip produced an answer: ${content2:0:120}" + +# 3. Streaming: confirm tool_call deltas reconstruct. +info "Test 3: streaming tool call" +req3="$(echo "$req1" | jq '. + {stream: true}')" +stream="$(curl -sS --max-time 60 -N "$BASE_URL/v1/chat/completions" \ + -H 'content-type: application/json' -d "$req3")" + +# Reassemble tool name + arguments from SSE data lines. +sname="$(echo "$stream" | sed -n 's/^data: //p' | grep -v '^\[DONE\]$' \ + | jq -rs 'map(.choices[0].delta.tool_calls[0].function.name // empty) | map(select(length>0)) | first // empty' 2>/dev/null || true)" +sargs="$(echo "$stream" | sed -n 's/^data: //p' | grep -v '^\[DONE\]$' \ + | jq -rs 'map(.choices[0].delta.tool_calls[0].function.arguments // empty) | join("")' 2>/dev/null || true)" + +if [ -z "$sname" ]; then + fail "Streaming produced no tool_call name. The output_item.added -> tool_calls mapping may be wrong. First lines: $(echo "$stream" | head -3)" +fi +[ "$sname" = "get_weather" ] || fail "Streaming tool name mismatch: '$sname'" +echo "$sargs" | jq '.' >/dev/null 2>&1 || fail "Streaming reassembled args are not valid JSON: '$sargs'" +pass "streaming tool call: name=$sname args=$sargs" + +printf '\n\033[32mAll /responses tool-forwarding checks passed for %s.\033[0m\n' "$MODEL" diff --git a/src/main.ts b/src/main.ts index 4f6ca784b..1aa5a5e9f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,6 +5,7 @@ import { defineCommand, runMain } from "citty" import { auth } from "./auth" import { checkUsage } from "./check-usage" import { debug } from "./debug" +import { models } from "./models" import { start } from "./start" const main = defineCommand({ @@ -13,7 +14,7 @@ const main = defineCommand({ description: "A wrapper around GitHub Copilot API to make it OpenAI compatible, making it usable for other tools.", }, - subCommands: { auth, start, "check-usage": checkUsage, debug }, + subCommands: { auth, start, models, "check-usage": checkUsage, debug }, }) await runMain(main) diff --git a/src/models.ts b/src/models.ts new file mode 100644 index 000000000..52b0b3e4d --- /dev/null +++ b/src/models.ts @@ -0,0 +1,120 @@ +#!/usr/bin/env node + +import { defineCommand } from "citty" +import consola from "consola" + +import { ensurePaths } from "./lib/paths" +import { initProxyFromEnv } from "./lib/proxy" +import { state } from "./lib/state" +import { setupGitHubToken } from "./lib/token" +import { cacheVSCodeVersion } from "./lib/utils" +import { getModels } from "./services/copilot/get-models" +import { getCopilotToken } from "./services/github/get-copilot-token" + +interface RunModelsOptions { + verbose: boolean + accountType: string + githubToken?: string + showToken: boolean + proxyEnv: boolean + json: boolean +} + +export async function runModels(options: RunModelsOptions): Promise { + if (options.proxyEnv) { + initProxyFromEnv() + } + + if (options.verbose) { + consola.level = 5 + consola.info("Verbose logging enabled") + } + + state.accountType = options.accountType + state.showToken = options.showToken + + await ensurePaths() + await cacheVSCodeVersion() + + if (options.githubToken) { + state.githubToken = options.githubToken + consola.info("Using provided GitHub token") + } else { + await setupGitHubToken() + } + + const { token } = await getCopilotToken() + state.copilotToken = token + + if (state.showToken) { + consola.info("Copilot token:", token) + } + + const models = await getModels() + + if (options.json) { + console.log(JSON.stringify(models, null, 2)) + return + } + + consola.box( + models.data + .map((model) => { + const endpoints = model.supported_endpoints?.join(", ") ?? "default" + return `${model.id} (${model.vendor}; ${endpoints})` + }) + .join("\n"), + ) +} + +export const models = defineCommand({ + meta: { + name: "models", + description: "List the current GitHub Copilot models available to the API", + }, + args: { + verbose: { + alias: "v", + type: "boolean", + default: false, + description: "Enable verbose logging", + }, + "account-type": { + alias: "a", + type: "string", + default: "individual", + description: "Account type to use (individual, business, enterprise)", + }, + "github-token": { + alias: "g", + type: "string", + description: + "Provide GitHub token directly (must be generated using the `auth` subcommand)", + }, + "show-token": { + type: "boolean", + default: false, + description: "Show GitHub and Copilot tokens on fetch", + }, + "proxy-env": { + type: "boolean", + default: false, + description: "Initialize proxy from environment variables", + }, + json: { + type: "boolean", + default: false, + description: "Output raw model response as JSON", + }, + }, + run({ args }) { + return runModels({ + verbose: args.verbose, + accountType: args["account-type"], + githubToken: args["github-token"], + showToken: args["show-token"], + proxyEnv: args["proxy-env"], + json: args.json, + }) + }, +}) diff --git a/src/routes/chat-completions/handler.ts b/src/routes/chat-completions/handler.ts index 04a5ae9ed..c17cccf3b 100644 --- a/src/routes/chat-completions/handler.ts +++ b/src/routes/chat-completions/handler.ts @@ -13,6 +13,14 @@ import { type ChatCompletionResponse, type ChatCompletionsPayload, } from "~/services/copilot/create-chat-completions" +import { + createResponsesFromChatCompletions, + responseEventToChatChunks, + responseToChatCompletion, + shouldUseResponsesEndpoint, + type ResponseApiResponse, + type ResponsesStreamState, +} from "~/services/copilot/create-responses" export async function handleCompletion(c: Context) { await checkRateLimit(state) @@ -47,6 +55,36 @@ export async function handleCompletion(c: Context) { consola.debug("Set max_tokens to:", JSON.stringify(payload.max_tokens)) } + if (shouldUseResponsesEndpoint(selectedModel?.supported_endpoints)) { + const response = await createResponsesFromChatCompletions(payload) + + if (isNonStreamingResponse(response)) { + consola.debug("Non-streaming response:", JSON.stringify(response)) + return c.json(responseToChatCompletion(response)) + } + + consola.debug("Streaming response from responses endpoint") + return streamSSE(c, async (stream) => { + const streamState: ResponsesStreamState = { + id: "", + model: payload.model, + created: Math.floor(Date.now() / 1000), + roleSent: false, + } + + for await (const event of response) { + if (!event.data) continue + + const chunks = responseEventToChatChunks(event.data, streamState) + for (const chunk of chunks) { + await stream.writeSSE({ + data: chunk === "[DONE]" ? chunk : JSON.stringify(chunk), + }) + } + } + }) + } + const response = await createChatCompletions(payload) if (isNonStreaming(response)) { @@ -66,3 +104,7 @@ export async function handleCompletion(c: Context) { const isNonStreaming = ( response: Awaited>, ): response is ChatCompletionResponse => Object.hasOwn(response, "choices") + +const isNonStreamingResponse = ( + response: Awaited>, +): response is ResponseApiResponse => !(Symbol.asyncIterator in response) diff --git a/src/routes/messages/non-stream-translation.ts b/src/routes/messages/non-stream-translation.ts index dc41e6382..45dbf4faf 100644 --- a/src/routes/messages/non-stream-translation.ts +++ b/src/routes/messages/non-stream-translation.ts @@ -47,11 +47,25 @@ export function translateToOpenAI( } function translateModelName(model: string): string { - // Subagent requests use a specific model number which Copilot doesn't support - if (model.startsWith("claude-sonnet-4-")) { - return model.replace(/^claude-sonnet-4-.*/, "claude-sonnet-4") - } else if (model.startsWith("claude-opus-")) { - return model.replace(/^claude-opus-4-.*/, "claude-opus-4") + // Claude clients may send dated IDs that are not exposed by Copilot. + // Keep already-supported dotted IDs unchanged and map only dated aliases. + if (/^claude-sonnet-4-5-\d+$/.test(model)) { + return "claude-sonnet-4.5" + } + if (/^claude-sonnet-4-6-\d+$/.test(model)) { + return "claude-sonnet-4.6" + } + if (/^claude-sonnet-4-\d+$/.test(model)) { + return "claude-sonnet-4.6" + } + if (/^claude-opus-4-5-\d+$/.test(model)) { + return "claude-opus-4.5" + } + if (/^claude-opus-4-\d+$/.test(model)) { + return "claude-opus-4.5" + } + if (/^claude-opus-5-\d+$/.test(model)) { + return "claude-opus-5" } return model } @@ -352,6 +366,20 @@ function getAnthropicToolUseBlocks( type: "tool_use", id: toolCall.id, name: toolCall.function.name, - input: JSON.parse(toolCall.function.arguments) as Record, + input: safeParseToolInput(toolCall.function.arguments), })) } + +function safeParseToolInput(args: string): Record { + try { + const parsed: unknown = JSON.parse(args) + if (parsed !== null && typeof parsed === "object") { + return parsed as Record + } + return {} + } catch { + // Models occasionally emit malformed JSON for tool arguments. + // Return an empty input instead of crashing the whole response. + return {} + } +} diff --git a/src/routes/models/route.ts b/src/routes/models/route.ts index 5254e2af7..1ae07399d 100644 --- a/src/routes/models/route.ts +++ b/src/routes/models/route.ts @@ -21,6 +21,11 @@ modelRoutes.get("/", async (c) => { created_at: new Date(0).toISOString(), // No date available from source owned_by: model.vendor, display_name: model.name, + model_picker_enabled: model.model_picker_enabled, + preview: model.preview, + policy: model.policy, + capabilities: model.capabilities, + supported_endpoints: model.supported_endpoints, })) return c.json({ diff --git a/src/services/copilot/create-responses.ts b/src/services/copilot/create-responses.ts new file mode 100644 index 000000000..d42be3df9 --- /dev/null +++ b/src/services/copilot/create-responses.ts @@ -0,0 +1,463 @@ +import consola from "consola" +import { events } from "fetch-event-stream" + +import { copilotBaseUrl, copilotHeaders } from "~/lib/api-config" +import { HTTPError } from "~/lib/error" +import { state } from "~/lib/state" + +import type { + ChatCompletionChunk, + ChatCompletionResponse, + ChatCompletionsPayload, + ContentPart, + Message, + Tool, + ToolCall, +} from "./create-chat-completions" + +export const createResponsesFromChatCompletions = async ( + payload: ChatCompletionsPayload, +) => { + if (!state.copilotToken) throw new Error("Copilot token not found") + + const response = await fetch(`${copilotBaseUrl(state)}/responses`, { + method: "POST", + headers: copilotHeaders(state), + body: JSON.stringify(toResponsesPayload(payload)), + }) + + if (!response.ok) { + throw new HTTPError("Failed to create response", response) + } + + if (payload.stream) { + return events(response) + } + + return (await response.json()) as ResponseApiResponse +} + +export const shouldUseResponsesEndpoint = (endpoints?: Array) => + endpoints ? + endpoints.includes("/responses") && !endpoints.includes("/chat/completions") + : false + +export function responseToChatCompletion( + response: ResponseApiResponse, +): ChatCompletionResponse { + const toolCalls = getResponseToolCalls(response) + const text = getResponseText(response) + + return { + id: response.id, + object: "chat.completion", + created: response.created_at, + model: response.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + // Chat Completions convention: content is null when the turn is + // purely tool calls. + content: text || (toolCalls.length > 0 ? null : ""), + ...(toolCalls.length > 0 && { tool_calls: toolCalls }), + }, + logprobs: null, + finish_reason: nonStreamFinishReason(response, toolCalls.length > 0), + }, + ], + usage: toChatUsage(response.usage), + } +} + +function nonStreamFinishReason( + response: ResponseApiResponse, + hasToolCalls: boolean, +): "stop" | "length" | "tool_calls" { + if (hasToolCalls) return "tool_calls" + return response.status === "completed" ? "stop" : "length" +} + +function getResponseToolCalls(response: ResponseApiResponse): Array { + return ( + response.output + ?.filter((item) => item.type === "function_call") + .map((item) => ({ + id: item.call_id ?? item.id ?? "", + type: "function" as const, + function: { + name: item.name ?? "", + arguments: item.arguments ?? "", + }, + })) ?? [] + ) +} + +type ChatChunkDelta = ChatCompletionChunk["choices"][number]["delta"] +type ChatChunkFinish = ChatCompletionChunk["choices"][number]["finish_reason"] + +interface ChunkParts { + delta: ChatChunkDelta + finishReason: ChatChunkFinish + usage?: ChatCompletionChunk["usage"] +} + +function buildChunk( + state: ResponsesStreamState, + { delta, finishReason, usage }: ChunkParts, +): ChatCompletionChunk { + return { + id: state.id, + object: "chat.completion.chunk", + created: state.created, + model: state.model, + choices: [{ index: 0, delta, finish_reason: finishReason, logprobs: null }], + ...(usage && { usage }), + } +} + +function consumeRole(state: ResponsesStreamState): "assistant" | undefined { + const role = state.roleSent ? undefined : "assistant" + state.roleSent = true + return role +} + +function handleTextDelta( + event: ResponseStreamEvent, + state: ResponsesStreamState, +): Array { + if (!event.delta) return [] + return [ + buildChunk(state, { + delta: { role: consumeRole(state), content: event.delta }, + finishReason: null, + }), + ] +} + +function handleFunctionCallAdded( + event: ResponseStreamEvent, + state: ResponsesStreamState, +): Array { + if (event.item?.type !== "function_call") return [] + + state.toolCalls ??= {} + const index = state.toolCallCount ?? 0 + state.toolCallCount = index + 1 + state.hasToolCalls = true + state.toolCalls[event.item.id ?? event.item_id ?? String(index)] = { index } + + return [ + buildChunk(state, { + delta: { + role: consumeRole(state), + tool_calls: [ + { + index, + id: event.item.call_id ?? event.item.id ?? "", + type: "function", + function: { name: event.item.name ?? "", arguments: "" }, + }, + ], + }, + finishReason: null, + }), + ] +} + +function handleFunctionCallArgsDelta( + event: ResponseStreamEvent, + state: ResponsesStreamState, +): Array { + if (!event.item_id || !event.delta) return [] + const toolCall = state.toolCalls?.[event.item_id] + if (!toolCall) return [] + + return [ + buildChunk(state, { + delta: { + tool_calls: [ + { index: toolCall.index, function: { arguments: event.delta } }, + ], + }, + finishReason: null, + }), + ] +} + +function handleCompleted( + event: ResponseStreamEvent, + state: ResponsesStreamState, +): Array { + if (!event.response) return [] + state.id = event.response.id + state.model = event.response.model + state.created = event.response.created_at + state.roleSent = true + + return [ + buildChunk(state, { + delta: { content: null }, + finishReason: state.hasToolCalls ? "tool_calls" : "stop", + usage: toChatUsage(event.response.usage), + }), + "[DONE]", + ] +} + +export function responseEventToChatChunks( + eventData: string, + state: ResponsesStreamState, +): Array { + let event: ResponseStreamEvent + try { + event = JSON.parse(eventData) as ResponseStreamEvent + } catch { + // Skip malformed or non-JSON SSE events (e.g. keepalives) + // instead of killing the whole stream. + consola.debug("Skipping non-JSON responses stream event:", eventData) + return [] + } + + if (event.type === "response.created" && event.response) { + state.id = event.response.id + state.model = event.response.model + state.created = event.response.created_at + return [] + } + + switch (event.type) { + case "response.output_text.delta": { + return handleTextDelta(event, state) + } + case "response.output_item.added": { + return handleFunctionCallAdded(event, state) + } + case "response.function_call_arguments.delta": { + return handleFunctionCallArgsDelta(event, state) + } + case "response.completed": { + return handleCompleted(event, state) + } + default: { + return [] + } + } +} + +export interface ResponsesStreamState { + created: number + id: string + model: string + roleSent: boolean + // Maps a streamed function_call item id to its chat-completions tool_calls + // array index. Lazily initialized so existing construction sites still work. + toolCalls?: Record + toolCallCount?: number + hasToolCalls?: boolean +} + +interface ResponsesPayload { + input: Array + max_output_tokens?: number | null + model: string + stream?: boolean | null + temperature?: number | null + top_p?: number | null + tools?: Array + tool_choice?: ResponsesToolChoice +} + +interface ResponseInputMessage { + type?: "message" + content: string + role: "assistant" | "developer" | "system" | "user" +} + +interface ResponseFunctionCallInput { + type: "function_call" + call_id: string + name: string + arguments: string +} + +interface ResponseFunctionCallOutputInput { + type: "function_call_output" + call_id: string + output: string +} + +type ResponseInputItem = + | ResponseInputMessage + | ResponseFunctionCallInput + | ResponseFunctionCallOutputInput + +interface ResponsesFunctionTool { + type: "function" + name: string + description?: string + parameters: Record +} + +type ResponsesToolChoice = + | "auto" + | "none" + | "required" + | { type: "function"; name: string } + +export interface ResponseApiResponse { + created_at: number + error?: unknown + id: string + model: string + object: "response" + output?: Array + status: string + usage?: ResponseUsage | null +} + +interface ResponseOutputItem { + type?: string + content?: Array + // Present when type === "function_call" + id?: string + call_id?: string + name?: string + arguments?: string +} + +interface ResponseContentPart { + text?: string + type: string +} + +interface ResponseUsage { + input_tokens?: number + output_tokens?: number + total_tokens?: number +} + +interface ResponseStreamEvent { + delta?: string + response?: ResponseApiResponse + type: string + // Function-call streaming fields + item?: ResponseOutputItem + item_id?: string + output_index?: number +} + +export function toResponsesPayload( + payload: ChatCompletionsPayload, +): ResponsesPayload { + return { + model: payload.model, + input: payload.messages.flatMap((message) => toResponseInputItems(message)), + max_output_tokens: payload.max_tokens, + stream: payload.stream, + temperature: payload.temperature, + top_p: payload.top_p, + tools: toResponsesTools(payload.tools), + tool_choice: toResponsesToolChoice(payload.tool_choice), + } +} + +function toResponsesTools( + tools: ChatCompletionsPayload["tools"], +): Array | undefined { + if (!tools || tools.length === 0) return undefined + // Chat Completions nests the function under `function`; the Responses API + // expects the name/description/parameters flattened onto the tool itself. + return tools.map((tool: Tool) => ({ + type: "function", + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + })) +} + +function toResponsesToolChoice( + toolChoice: ChatCompletionsPayload["tool_choice"], +): ResponsesToolChoice | undefined { + if (toolChoice === null || toolChoice === undefined) return undefined + if (typeof toolChoice === "string") return toolChoice + return { type: "function", name: toolChoice.function.name } +} + +function toResponseInputItems(message: Message): Array { + // A tool result becomes a function_call_output keyed by its call id. + if (message.role === "tool") { + return [ + { + type: "function_call_output", + call_id: message.tool_call_id ?? "", + output: contentToText(message.content), + }, + ] + } + + // An assistant turn that issued tool calls becomes optional text followed by + // one function_call item per call. + if ( + message.role === "assistant" + && message.tool_calls + && message.tool_calls.length > 0 + ) { + const items: Array = [] + const text = contentToText(message.content) + if (text) { + items.push({ type: "message", role: "assistant", content: text }) + } + for (const toolCall of message.tool_calls) { + items.push({ + type: "function_call", + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments, + }) + } + return items + } + + return [ + { + type: "message", + role: message.role, + content: contentToText(message.content), + }, + ] +} + +function contentToText(content: Message["content"]): string { + if (typeof content === "string") return content + if (!content) return "" + + return content.map((part) => contentPartToText(part)).join("\n") +} + +function contentPartToText(part: ContentPart): string { + if (part.type === "text") return part.text + + return `[image: ${part.image_url.url}]` +} + +function getResponseText(response: ResponseApiResponse): string { + return ( + response.output + ?.flatMap((item) => item.content ?? []) + .filter((part) => part.type === "output_text") + .map((part) => part.text ?? "") + .join("") ?? "" + ) +} + +function toChatUsage(usage?: ResponseUsage | null) { + if (!usage) return undefined + + return { + prompt_tokens: usage.input_tokens ?? 0, + completion_tokens: usage.output_tokens ?? 0, + total_tokens: usage.total_tokens ?? 0, + } +} diff --git a/src/services/copilot/get-models.ts b/src/services/copilot/get-models.ts index 3cfa30af0..efd61f276 100644 --- a/src/services/copilot/get-models.ts +++ b/src/services/copilot/get-models.ts @@ -46,6 +46,7 @@ export interface Model { name: string object: string preview: boolean + supported_endpoints?: Array vendor: string version: string policy?: { diff --git a/src/services/get-vscode-version.ts b/src/services/get-vscode-version.ts index 6078f09b5..51ea5e24f 100644 --- a/src/services/get-vscode-version.ts +++ b/src/services/get-vscode-version.ts @@ -29,5 +29,3 @@ export async function getVSCodeVersion() { clearTimeout(timeout) } } - -await getVSCodeVersion() diff --git a/src/start.ts b/src/start.ts index 14abbbdff..1e8a48320 100644 --- a/src/start.ts +++ b/src/start.ts @@ -16,6 +16,7 @@ import { server } from "./server" interface RunServerOptions { port: number + host: string verbose: boolean accountType: string manual: boolean @@ -64,7 +65,14 @@ export async function runServer(options: RunServerOptions): Promise { `Available models: \n${state.models?.data.map((model) => `- ${model.id}`).join("\n")}`, ) - const serverUrl = `http://localhost:${options.port}` + const displayHost = options.host === "0.0.0.0" ? "localhost" : options.host + const serverUrl = `http://${displayHost}:${options.port}` + + if (options.host !== "127.0.0.1" && options.host !== "localhost") { + consola.warn( + `Server will listen on ${options.host} and may be reachable from other machines. Use the default host (127.0.0.1) for local-only access.`, + ) + } if (options.claudeCode) { invariant(state.models, "Models should be loaded by now") @@ -116,6 +124,7 @@ export async function runServer(options: RunServerOptions): Promise { serve({ fetch: server.fetch as ServerHandler, + hostname: options.host, port: options.port, }) } @@ -132,6 +141,12 @@ export const start = defineCommand({ default: "4141", description: "Port to listen on", }, + host: { + type: "string", + default: "127.0.0.1", + description: + "Host to bind to. Defaults to 127.0.0.1 (local-only). Use 0.0.0.0 to expose to the network (e.g. inside Docker)", + }, verbose: { alias: "v", type: "boolean", @@ -193,6 +208,7 @@ export const start = defineCommand({ return runServer({ port: Number.parseInt(args.port, 10), + host: args.host, verbose: args.verbose, accountType: args["account-type"], manual: args.manual, diff --git a/tests/anthropic-edge-cases.test.ts b/tests/anthropic-edge-cases.test.ts new file mode 100644 index 000000000..aa4f35dc3 --- /dev/null +++ b/tests/anthropic-edge-cases.test.ts @@ -0,0 +1,494 @@ +import { describe, test, expect } from "bun:test" + +import type { + AnthropicMessagesPayload, + AnthropicStreamState, + AnthropicTextBlock, + AnthropicToolUseBlock, +} from "~/routes/messages/anthropic-types" +import type { + ChatCompletionChunk, + ChatCompletionResponse, + ContentPart, +} from "~/services/copilot/create-chat-completions" + +import { + translateToAnthropic, + translateToOpenAI, +} from "../src/routes/messages/non-stream-translation" +import { translateChunkToAnthropicEvents } from "../src/routes/messages/stream-translation" + +function freshStreamState(): AnthropicStreamState { + return { + messageStartSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + toolCalls: {}, + } +} + +function makeChunk(partial: Partial): ChatCompletionChunk { + return { + id: "chunk-1", + object: "chat.completion.chunk", + created: 0, + model: "gpt-4o", + choices: [], + ...partial, + } +} + +describe("Anthropic request edge cases", () => { + test("translates image blocks into image_url content parts", () => { + const payload: AnthropicMessagesPayload = { + model: "gpt-4o", + max_tokens: 100, + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is in this image?" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + const userMessage = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMessage?.content)).toBe(true) + + const parts = userMessage?.content as Array + const imagePart = parts.find((p) => p.type === "image_url") + expect(imagePart).toBeDefined() + if (imagePart?.type === "image_url") { + expect(imagePart.image_url.url).toBe("data:image/png;base64,iVBORw0KGgo=") + } + const textPart = parts.find((p) => p.type === "text") + expect(textPart).toBeDefined() + }) + + test("translates tool_result blocks into tool role messages before user content", () => { + const payload: AnthropicMessagesPayload = { + model: "gpt-4o", + max_tokens: 100, + messages: [ + { role: "user", content: "What's the weather?" }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_1", + name: "get_weather", + input: { location: "Boston" }, + }, + ], + }, + { + role: "user", + content: [ + { type: "text", text: "Thanks, summarize that." }, + { + type: "tool_result", + tool_use_id: "toolu_1", + content: "Sunny, 75F", + }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + const roles = result.messages.map((m) => m.role) + + // tool_result must directly follow the assistant tool_call message + const assistantIndex = roles.indexOf("assistant") + expect(roles[assistantIndex + 1]).toBe("tool") + + const toolMessage = result.messages[assistantIndex + 1] + expect(toolMessage.tool_call_id).toBe("toolu_1") + expect(toolMessage.content).toBe("Sunny, 75F") + + // remaining user text still arrives after the tool message + expect(roles[assistantIndex + 2]).toBe("user") + }) + + test("handles multiple tool_result blocks in a single user message", () => { + const payload: AnthropicMessagesPayload = { + model: "gpt-4o", + max_tokens: 100, + messages: [ + { + role: "assistant", + content: [ + { type: "tool_use", id: "toolu_a", name: "a", input: {} }, + { type: "tool_use", id: "toolu_b", name: "b", input: {} }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_a", content: "A" }, + { type: "tool_result", tool_use_id: "toolu_b", content: "B" }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + const toolMessages = result.messages.filter((m) => m.role === "tool") + expect(toolMessages).toHaveLength(2) + expect(toolMessages.map((m) => m.tool_call_id)).toEqual([ + "toolu_a", + "toolu_b", + ]) + }) +}) + +function makeResponse( + overrides: Partial, +): ChatCompletionResponse { + return { + id: "resp-1", + object: "chat.completion", + created: 0, + model: "gpt-4o", + choices: [], + ...overrides, + } as ChatCompletionResponse +} + +describe("Anthropic response edge cases", () => { + test("does not crash on invalid tool call JSON and falls back to empty input", () => { + const response = makeResponse({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_bad", + type: "function", + function: { name: "broken_tool", arguments: "{not json" }, + }, + ], + }, + }, + ], + }) + + const result = translateToAnthropic(response) + const toolUse = result.content.find( + (b): b is AnthropicToolUseBlock => b.type === "tool_use", + ) + expect(toolUse).toBeDefined() + expect(toolUse?.input).toEqual({}) + expect(result.stop_reason).toBe("tool_use") + }) + + test("falls back to empty input when tool arguments parse to a non-object", () => { + const response = makeResponse({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_scalar", + type: "function", + function: { name: "scalar_tool", arguments: "42" }, + }, + ], + }, + }, + ], + }) + + const result = translateToAnthropic(response) + const toolUse = result.content.find( + (b): b is AnthropicToolUseBlock => b.type === "tool_use", + ) + expect(toolUse?.input).toEqual({}) + }) + + test("subtracts cached tokens from input_tokens and reports cache_read_input_tokens", () => { + const response = makeResponse({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "stop", + message: { role: "assistant", content: "Hi" }, + }, + ], + usage: { + prompt_tokens: 1000, + completion_tokens: 5, + total_tokens: 1005, + prompt_tokens_details: { cached_tokens: 800 }, + }, + }) + + const result = translateToAnthropic(response) + expect(result.usage.input_tokens).toBe(200) + expect(result.usage.cache_read_input_tokens).toBe(800) + expect(result.usage.output_tokens).toBe(5) + }) + + test("omits cache_read_input_tokens when no cached token details exist", () => { + const response = makeResponse({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "stop", + message: { role: "assistant", content: "Hi" }, + }, + ], + usage: { prompt_tokens: 100, completion_tokens: 5, total_tokens: 105 }, + }) + + const result = translateToAnthropic(response) + expect(result.usage.input_tokens).toBe(100) + expect("cache_read_input_tokens" in result.usage).toBe(false) + }) +}) + +describe("Anthropic stream translation edge cases", () => { + test("closes a text block before opening a tool block, and vice versa", () => { + const state = freshStreamState() + + // 1. text chunk + const textEvents = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { role: "assistant", content: "Let me check." }, + }, + ], + }), + state, + ) + expect(textEvents.map((e) => e.type)).toEqual([ + "message_start", + "content_block_start", + "content_block_delta", + ]) + + // 2. tool call chunk: must close the text block first + const toolEvents = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: "" }, + }, + ], + }, + }, + ], + }), + state, + ) + expect(toolEvents.map((e) => e.type)).toEqual([ + "content_block_stop", + "content_block_start", + ]) + const blockStart = toolEvents.find((e) => e.type === "content_block_start") + expect( + blockStart?.type === "content_block_start" + && blockStart.content_block.type === "tool_use" + && blockStart.content_block.name, + ).toBe("get_weather") + + // 3. text again after the tool block: closes tool block, opens new text block + const backToTextEvents = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { content: "Done." }, + }, + ], + }), + state, + ) + expect(backToTextEvents.map((e) => e.type)).toEqual([ + "content_block_stop", + "content_block_start", + "content_block_delta", + ]) + + // Block indices must be strictly increasing across the three blocks + expect(state.contentBlockIndex).toBe(2) + }) + + test("streams partial tool JSON as input_json_delta tied to the right block", () => { + const state = freshStreamState() + + translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { + role: "assistant", + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: "" }, + }, + ], + }, + }, + ], + }), + state, + ) + + const argEvents = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { + tool_calls: [ + { index: 0, function: { arguments: '{"location":"Bo' } }, + ], + }, + }, + ], + }), + state, + ) + + expect(argEvents).toHaveLength(1) + const deltaEvent = argEvents[0] + expect( + deltaEvent.type === "content_block_delta" + && deltaEvent.delta.type === "input_json_delta" + && deltaEvent.delta.partial_json, + ).toBe('{"location":"Bo') + }) + + test("ignores argument deltas for unknown tool call indices", () => { + const state = freshStreamState() + state.messageStartSent = true + + const events = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: null, + delta: { + tool_calls: [{ index: 7, function: { arguments: '{"x":1}' } }], + }, + }, + ], + }), + state, + ) + + expect(events).toHaveLength(0) + }) + + test("reports cache-aware usage in message_delta on finish", () => { + const state = freshStreamState() + state.messageStartSent = true + state.contentBlockOpen = true + + const events = translateChunkToAnthropicEvents( + makeChunk({ + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "stop", + delta: {}, + }, + ], + usage: { + prompt_tokens: 500, + completion_tokens: 20, + total_tokens: 520, + prompt_tokens_details: { cached_tokens: 300 }, + }, + }), + state, + ) + + const messageDelta = events.find((e) => e.type === "message_delta") + expect(messageDelta?.type).toBe("message_delta") + if (messageDelta?.type === "message_delta") { + expect(messageDelta.usage?.input_tokens).toBe(200) + expect(messageDelta.usage?.cache_read_input_tokens).toBe(300) + expect(messageDelta.usage?.output_tokens).toBe(20) + } + expect(events.at(-1)?.type).toBe("message_stop") + }) + + test("handles empty choices chunks without emitting events", () => { + const state = freshStreamState() + const events = translateChunkToAnthropicEvents( + makeChunk({ choices: [] }), + state, + ) + expect(events).toHaveLength(0) + expect(state.messageStartSent).toBe(false) + }) +}) + +describe("system prompt translation", () => { + test("joins array-form system prompts into a single system message", () => { + const payload: AnthropicMessagesPayload = { + model: "gpt-4o", + max_tokens: 10, + system: [ + { type: "text", text: "You are helpful." }, + { type: "text", text: "Be concise." }, + ] as Array, + messages: [{ role: "user", content: "Hi" }], + } + + const result = translateToOpenAI(payload) + expect(result.messages[0].role).toBe("system") + expect(result.messages[0].content).toBe("You are helpful.\n\nBe concise.") + }) +}) diff --git a/tests/anthropic-request.test.ts b/tests/anthropic-request.test.ts index 06c663778..2042aee80 100644 --- a/tests/anthropic-request.test.ts +++ b/tests/anthropic-request.test.ts @@ -63,6 +63,34 @@ function isValidChatCompletionRequest(payload: unknown): boolean { } describe("Anthropic to OpenAI translation logic", () => { + test("maps dated Claude IDs to currently supported Copilot models", () => { + const sonnet = translateToOpenAI({ + model: "claude-sonnet-4-20250514", + messages: [{ role: "user", content: "Hello!" }], + max_tokens: 100, + }) + const opus = translateToOpenAI({ + model: "claude-opus-4-20250514", + messages: [{ role: "user", content: "Hello!" }], + max_tokens: 100, + }) + const opusNext = translateToOpenAI({ + model: "claude-opus-5-20251001", + messages: [{ role: "user", content: "Hello!" }], + max_tokens: 100, + }) + const current = translateToOpenAI({ + model: "claude-sonnet-4.6", + messages: [{ role: "user", content: "Hello!" }], + max_tokens: 100, + }) + + expect(sonnet.model).toBe("claude-sonnet-4.6") + expect(opus.model).toBe("claude-opus-4.5") + expect(opusNext.model).toBe("claude-opus-5") + expect(current.model).toBe("claude-sonnet-4.6") + }) + test("should translate minimal Anthropic payload to valid OpenAI payload", () => { const anthropicPayload: AnthropicMessagesPayload = { model: "gpt-4o", diff --git a/tests/count-tokens-handler.test.ts b/tests/count-tokens-handler.test.ts new file mode 100644 index 000000000..08088ac1f --- /dev/null +++ b/tests/count-tokens-handler.test.ts @@ -0,0 +1,185 @@ +import { afterEach, describe, expect, test } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" +import type { Model } from "~/services/copilot/get-models" + +import { state } from "~/lib/state" +import { getTokenCount } from "~/lib/tokenizer" +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" +import { messageRoutes } from "~/routes/messages/route" + +function makeModel(id: string): Model { + return { + id, + name: id, + object: "model", + vendor: "test", + version: "1", + preview: false, + model_picker_enabled: true, + capabilities: { + family: id, + object: "model_capabilities", + type: "chat", + tokenizer: "o200k_base", + limits: { max_output_tokens: 4096 }, + supports: { tool_calls: true }, + }, + } +} + +function setModels(...ids: Array): void { + state.models = { object: "list", data: ids.map((id) => makeModel(id)) } +} + +async function countTokens( + payload: AnthropicMessagesPayload, + headers: Record = {}, +): Promise { + const res = await messageRoutes.request("/count_tokens", { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(payload), + }) + const json = (await res.json()) as { input_tokens: number } + return json.input_tokens +} + +/** Re-derive the base count the handler starts from, before overhead/multiplier. */ +async function baseCount(payload: AnthropicMessagesPayload): Promise { + const model = state.models?.data.find((m) => m.id === payload.model) + if (!model) throw new Error("model not seeded for baseCount") + const openAIPayload = translateToOpenAI(payload) + const { input, output } = await getTokenCount(openAIPayload, model) + return input + output +} + +afterEach(() => { + state.models = undefined +}) + +describe("count_tokens handler", () => { + test("returns default count of 1 when the model is unknown", async () => { + setModels("some-other-model") + const result = await countTokens({ + model: "nonexistent-model", + max_tokens: 10, + messages: [{ role: "user", content: "Hello there" }], + }) + expect(result).toBe(1) + }) + + test("applies the 1.15 multiplier for claude models", async () => { + setModels("claude-sonnet-4") + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + max_tokens: 10, + messages: [{ role: "user", content: "Hello there, how are you?" }], + } + const base = await baseCount(payload) + const result = await countTokens(payload) + expect(result).toBe(Math.round(base * 1.15)) + }) + + test("applies the 1.03 multiplier for grok models", async () => { + setModels("grok-code") + const payload: AnthropicMessagesPayload = { + model: "grok-code", + max_tokens: 10, + messages: [{ role: "user", content: "Hello there, how are you?" }], + } + const base = await baseCount(payload) + const result = await countTokens(payload) + expect(result).toBe(Math.round(base * 1.03)) + }) + + test("adds 346-token tool overhead for claude before the multiplier", async () => { + setModels("claude-sonnet-4") + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + max_tokens: 10, + messages: [{ role: "user", content: "What's the weather?" }], + tools: [ + { + name: "get_weather", + description: "Get weather", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const base = await baseCount(payload) + const result = await countTokens(payload) + expect(result).toBe(Math.round((base + 346) * 1.15)) + }) + + test("adds 480-token tool overhead for grok before the multiplier", async () => { + setModels("grok-code") + const payload: AnthropicMessagesPayload = { + model: "grok-code", + max_tokens: 10, + messages: [{ role: "user", content: "What's the weather?" }], + tools: [ + { + name: "get_weather", + description: "Get weather", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const base = await baseCount(payload) + const result = await countTokens(payload) + expect(result).toBe(Math.round((base + 480) * 1.03)) + }) + + test("skips tool overhead when an mcp__ tool is present under the claude-code beta", async () => { + setModels("claude-sonnet-4") + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + max_tokens: 10, + messages: [{ role: "user", content: "Use a tool" }], + tools: [ + { + name: "mcp__server__do_thing", + description: "An MCP tool", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const base = await baseCount(payload) + const result = await countTokens(payload, { + "anthropic-beta": "claude-code-20250101", + }) + // No 346 overhead because an mcp__ tool exists under the claude-code beta + expect(result).toBe(Math.round(base * 1.15)) + }) + + test("still adds overhead for an mcp__ tool when the claude-code beta is absent", async () => { + setModels("claude-sonnet-4") + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + max_tokens: 10, + messages: [{ role: "user", content: "Use a tool" }], + tools: [ + { + name: "mcp__server__do_thing", + description: "An MCP tool", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const base = await baseCount(payload) + const result = await countTokens(payload) + expect(result).toBe(Math.round((base + 346) * 1.15)) + }) + + test("falls back to a count of 1 when the body is not valid JSON", async () => { + setModels("claude-sonnet-4") + const res = await messageRoutes.request("/count_tokens", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ not valid json", + }) + const json = (await res.json()) as { input_tokens: number } + expect(json.input_tokens).toBe(1) + }) +}) diff --git a/tests/create-responses.test.ts b/tests/create-responses.test.ts new file mode 100644 index 000000000..468b36216 --- /dev/null +++ b/tests/create-responses.test.ts @@ -0,0 +1,457 @@ +import { describe, expect, test } from "bun:test" + +import { + responseEventToChatChunks, + responseToChatCompletion, + shouldUseResponsesEndpoint, + toResponsesPayload, + type ResponseApiResponse, + type ResponsesStreamState, +} from "~/services/copilot/create-responses" + +describe("Responses API adapter", () => { + test("uses responses only for models that do not support chat completions", () => { + expect(shouldUseResponsesEndpoint(["/responses"])).toBe(true) + expect( + shouldUseResponsesEndpoint(["/responses", "/chat/completions"]), + ).toBe(false) + expect(shouldUseResponsesEndpoint(["/chat/completions"])).toBe(false) + expect(shouldUseResponsesEndpoint()).toBe(false) + }) + + test("converts a non-streaming response to a chat completion", () => { + const response: ResponseApiResponse = { + id: "resp_123", + object: "response", + created_at: 1700000000, + model: "gpt-test", + status: "completed", + output: [ + { + content: [ + { type: "output_text", text: "Hello" }, + { type: "output_text", text: " there" }, + ], + }, + ], + usage: { + input_tokens: 2, + output_tokens: 3, + total_tokens: 5, + }, + } + + const chatCompletion = responseToChatCompletion(response) + + expect(chatCompletion.id).toBe("resp_123") + expect(chatCompletion.object).toBe("chat.completion") + expect(chatCompletion.choices[0].message.content).toBe("Hello there") + expect(chatCompletion.choices[0].finish_reason).toBe("stop") + expect(chatCompletion.usage).toEqual({ + prompt_tokens: 2, + completion_tokens: 3, + total_tokens: 5, + }) + }) + + test("converts response stream events to chat completion chunks", () => { + const streamState: ResponsesStreamState = { + id: "", + model: "gpt-test", + created: 0, + roleSent: false, + } + + expect( + responseEventToChatChunks( + JSON.stringify({ + type: "response.created", + response: { + id: "resp_123", + object: "response", + created_at: 1700000000, + model: "gpt-test", + status: "in_progress", + }, + }), + streamState, + ), + ).toEqual([]) + + const deltaChunks = responseEventToChatChunks( + JSON.stringify({ + type: "response.output_text.delta", + delta: "Hello", + }), + streamState, + ) + + expect(deltaChunks).toHaveLength(1) + expect(deltaChunks[0]).toMatchObject({ + id: "resp_123", + object: "chat.completion.chunk", + choices: [ + { + delta: { + role: "assistant", + content: "Hello", + }, + finish_reason: null, + }, + ], + }) + + const completedChunks = responseEventToChatChunks( + JSON.stringify({ + type: "response.completed", + response: { + id: "resp_123", + object: "response", + created_at: 1700000000, + model: "gpt-test", + status: "completed", + usage: { + input_tokens: 2, + output_tokens: 3, + total_tokens: 5, + }, + }, + }), + streamState, + ) + + expect(completedChunks.at(-1)).toBe("[DONE]") + expect(completedChunks[0]).toMatchObject({ + choices: [ + { + delta: { + content: null, + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 2, + completion_tokens: 3, + total_tokens: 5, + }, + }) + }) + + test("skips malformed stream events instead of throwing", () => { + const streamState: ResponsesStreamState = { + id: "resp_123", + model: "gpt-test", + created: 1700000000, + roleSent: true, + } + + expect(responseEventToChatChunks("not json {", streamState)).toEqual([]) + expect(responseEventToChatChunks("[DONE]", streamState)).toEqual([]) + expect(responseEventToChatChunks("", streamState)).toEqual([]) + + // Stream state must be untouched so later valid events still work + const deltaChunks = responseEventToChatChunks( + JSON.stringify({ type: "response.output_text.delta", delta: "Hi" }), + streamState, + ) + expect(deltaChunks).toHaveLength(1) + }) + + test("ignores unknown event types", () => { + const streamState: ResponsesStreamState = { + id: "resp_123", + model: "gpt-test", + created: 1700000000, + roleSent: false, + } + + expect( + responseEventToChatChunks( + JSON.stringify({ type: "response.output_item.added" }), + streamState, + ), + ).toEqual([]) + expect(streamState.roleSent).toBe(false) + }) +}) + +describe("Responses API tool forwarding", () => { + test("flattens chat-completions tools into responses function tools", () => { + const payload = toResponsesPayload({ + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { type: "object", properties: { city: {} } }, + }, + }, + ], + }) as unknown as { + tools?: Array> + } + + expect(payload.tools).toEqual([ + { + type: "function", + name: "get_weather", + description: "Get weather", + parameters: { type: "object", properties: { city: {} } }, + }, + ]) + }) + + test("maps tool_choice variants", () => { + const auto = toResponsesPayload({ + model: "m", + messages: [], + tool_choice: "auto", + }) as unknown as { tool_choice?: unknown } + expect(auto.tool_choice).toBe("auto") + + const forced = toResponsesPayload({ + model: "m", + messages: [], + tool_choice: { type: "function", function: { name: "get_weather" } }, + }) as unknown as { tool_choice?: unknown } + expect(forced.tool_choice).toEqual({ + type: "function", + name: "get_weather", + }) + + const none = toResponsesPayload({ + model: "m", + messages: [], + }) as unknown as { tool_choice?: unknown } + expect(none.tool_choice).toBeUndefined() + }) + + test("converts assistant tool_calls and tool results into input items", () => { + const payload = toResponsesPayload({ + model: "m", + messages: [ + { role: "user", content: "weather?" }, + { + role: "assistant", + content: "Let me check.", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Boston"}', + }, + }, + ], + }, + { role: "tool", tool_call_id: "call_1", content: "Sunny" }, + ], + }) as unknown as { input: Array> } + + expect(payload.input).toEqual([ + { type: "message", role: "user", content: "weather?" }, + { type: "message", role: "assistant", content: "Let me check." }, + { + type: "function_call", + call_id: "call_1", + name: "get_weather", + arguments: '{"city":"Boston"}', + }, + { type: "function_call_output", call_id: "call_1", output: "Sunny" }, + ]) + }) + + test("omits assistant text when the turn is purely a tool call", () => { + const payload = toResponsesPayload({ + model: "m", + messages: [ + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_9", + type: "function", + function: { name: "noop", arguments: "{}" }, + }, + ], + }, + ], + }) as unknown as { input: Array> } + + expect(payload.input).toEqual([ + { + type: "function_call", + call_id: "call_9", + name: "noop", + arguments: "{}", + }, + ]) + }) +}) + +describe("Responses API tool forwarding (responses)", () => { + test("converts a non-streaming function_call output into tool_calls", () => { + const response: ResponseApiResponse = { + id: "resp_1", + object: "response", + created_at: 1700000000, + model: "gpt-test", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_1", + call_id: "call_1", + name: "get_weather", + arguments: '{"city":"Boston"}', + }, + ], + } + + const chat = responseToChatCompletion(response) + expect(chat.choices[0].finish_reason).toBe("tool_calls") + expect(chat.choices[0].message.content).toBeNull() + expect(chat.choices[0].message.tool_calls).toEqual([ + { + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Boston"}' }, + }, + ]) + }) + + test("streams a function call as opening + argument chunks", () => { + const streamState: ResponsesStreamState = { + id: "resp_1", + model: "gpt-test", + created: 1700000000, + roleSent: false, + } + + const openChunks = responseEventToChatChunks( + JSON.stringify({ + type: "response.output_item.added", + output_index: 0, + item: { + type: "function_call", + id: "fc_1", + call_id: "call_1", + name: "get_weather", + arguments: "", + }, + }), + streamState, + ) + + expect(openChunks).toHaveLength(1) + expect(openChunks[0]).toMatchObject({ + choices: [ + { + delta: { + role: "assistant", + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: "" }, + }, + ], + }, + finish_reason: null, + }, + ], + }) + + const argChunks = responseEventToChatChunks( + JSON.stringify({ + type: "response.function_call_arguments.delta", + item_id: "fc_1", + delta: '{"city":', + }), + streamState, + ) + expect(argChunks).toHaveLength(1) + expect(argChunks[0]).toMatchObject({ + choices: [ + { + delta: { + tool_calls: [{ index: 0, function: { arguments: '{"city":' } }], + }, + }, + ], + }) + + const completed = responseEventToChatChunks( + JSON.stringify({ + type: "response.completed", + response: { + id: "resp_1", + object: "response", + created_at: 1700000000, + model: "gpt-test", + status: "completed", + }, + }), + streamState, + ) + expect(completed[0]).toMatchObject({ + choices: [{ finish_reason: "tool_calls" }], + }) + expect(completed.at(-1)).toBe("[DONE]") + }) + + test("ignores argument deltas for unknown function-call item ids", () => { + const streamState: ResponsesStreamState = { + id: "resp_1", + model: "gpt-test", + created: 1700000000, + roleSent: true, + } + + expect( + responseEventToChatChunks( + JSON.stringify({ + type: "response.function_call_arguments.delta", + item_id: "unknown", + delta: "{}", + }), + streamState, + ), + ).toEqual([]) + }) + + test("assigns increasing indices to parallel function calls", () => { + const streamState: ResponsesStreamState = { + id: "resp_1", + model: "gpt-test", + created: 1700000000, + roleSent: false, + } + + responseEventToChatChunks( + JSON.stringify({ + type: "response.output_item.added", + item: { type: "function_call", id: "fc_1", call_id: "c1", name: "a" }, + }), + streamState, + ) + const second = responseEventToChatChunks( + JSON.stringify({ + type: "response.output_item.added", + item: { type: "function_call", id: "fc_2", call_id: "c2", name: "b" }, + }), + streamState, + ) + + expect(second[0]).toMatchObject({ + choices: [{ delta: { tool_calls: [{ index: 1, id: "c2" }] } }], + }) + }) +})