From 1388925d67fef6d13e6f0b638f631a7bae5ace4a Mon Sep 17 00:00:00 2001 From: Talha Date: Thu, 6 Aug 2026 12:01:53 +0500 Subject: [PATCH 1/6] feat(mcp): bundle OpenAI plugin and interactive app --- MANIFEST.in | 4 + README.md | 9 +- docs/integrations/openai-plugin.md | 66 +++ pyproject.toml | 5 + release-please-config.json | 5 + release-please-config.stable.json | 5 + src/vidxp/assets/mcp_app/index.html | 469 ++++++++++++++++++ .../vidxp/.codex-plugin/plugin.json | 28 ++ src/vidxp/bundled_plugins/vidxp/.mcp.json | 8 + .../skills/vidxp-find-video-evidence/SKILL.md | 56 +++ .../agents/openai.yaml | 13 + .../vidxp/skills/vidxp-ingest-video/SKILL.md | 35 ++ .../vidxp-ingest-video/agents/openai.yaml | 13 + src/vidxp/mcp.py | 195 +++++++- src/vidxp/mcp_app.py | 19 + tests/test_mcp.py | 71 ++- tests/test_packaging.py | 72 +++ 17 files changed, 1053 insertions(+), 20 deletions(-) create mode 100644 docs/integrations/openai-plugin.md create mode 100644 src/vidxp/assets/mcp_app/index.html create mode 100644 src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json create mode 100644 src/vidxp/bundled_plugins/vidxp/.mcp.json create mode 100644 src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md create mode 100644 src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml create mode 100644 src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md create mode 100644 src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml create mode 100644 src/vidxp/mcp_app.py diff --git a/MANIFEST.in b/MANIFEST.in index cdda21d..bdb0400 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,10 @@ include LICENSE include docs/images/logo.png recursive-include src/vidxp/assets/upload_page * recursive-include src/vidxp/assets/artifact_download * +recursive-include src/vidxp/assets/mcp_app * +include src/vidxp/bundled_plugins/vidxp/.mcp.json +include src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json +recursive-include src/vidxp/bundled_plugins/vidxp/skills * include web/upload-page/package.json include web/upload-page/package-lock.json include web/upload-page/scripts/build.mjs diff --git a/README.md b/README.md index 7dcc62f..48913e8 100644 --- a/README.md +++ b/README.md @@ -166,11 +166,14 @@ VidXP includes reusable skill source folders for the two common agent workflows: - [Ingest and index videos](skills/vidxp-ingest-video/SKILL.md) - [Find moments and return inspectable evidence](skills/vidxp-find-video-evidence/SKILL.md) -Download a skill folder and add it through a supported ChatGPT desktop or Codex -Skills surface. The skills require a connected VidXP MCP server; installable -plugin packaging for additional ChatGPT surfaces will follow separately. +Download a skill folder directly, or install the versioned VidXP plugin bundle +shipped inside the Python package. The plugin keeps both skills and the local +`vidxp-mcp` server definition together. Its MCP App resource also gives +compatible hosts an interactive upload and evidence-review view; every workflow +continues to work through ordinary MCP tool results when a host has no UI. - [Python, HTTP, and MCP installation](INSTALLATION_GUIDE.md) +- [ChatGPT and Codex plugin integration](docs/integrations/openai-plugin.md) - [Optional capability packages](INSTALLATION_GUIDE.md#optional-dependency-extras) - [Coolify server setup](docs/deployment/coolify.md) diff --git a/docs/integrations/openai-plugin.md b/docs/integrations/openai-plugin.md new file mode 100644 index 0000000..4ad70ae --- /dev/null +++ b/docs/integrations/openai-plugin.md @@ -0,0 +1,66 @@ +# ChatGPT and Codex plugin integration + +VidXP ships one versioned plugin bundle with its Python distribution. The +bundle combines the local MCP server definition with VidXP's canonical ingest +and evidence-search skills, while the MCP server exposes an optional interactive +view for hosts that implement MCP Apps. + +The packaged bundle lives at +`src/vidxp/bundled_plugins/vidxp/` and contains: + +- `.codex-plugin/plugin.json`, the plugin manifest; +- `.mcp.json`, which starts the installed `vidxp-mcp` command; and +- `skills/`, a release snapshot of the canonical root `skills/` folders. + +The root skill folders remain the authoring source. Packaging tests require the +bundled snapshot to match their file inventory and text exactly, and Release +Please keeps the plugin manifest version aligned with the Python package. + +## Interactive MCP App + +`create_media_upload` and `get_job_evidence` advertise the same +`ui://vidxp/evidence-review-v1.html` resource. A compatible host can render it +inline to: + +- open VidXP's short-lived HTTPS upload page and refresh session progress; +- review answer claims and annotated evidence-board pages; +- select up to ten ranked evidence IDs; and +- request exact keyframes or clips with `materialize_job_evidence` without + rerunning retrieval. + +The component completes the MCP Apps `ui/initialize` handshake, then uses the +standard `tools/call`, `ui/open-link`, `ui/request-display-mode`, +`ui/update-model-context`, and resize messages. ChatGPT-specific `window.openai` +helpers are feature-detected only as compatibility fallbacks and for ephemeral +widget state. VidXP remains authoritative for upload, job, search, and artifact +state, and non-UI clients receive the existing text, image, and resource-link +content. + +The resource is self-contained, uses system fonts, has no remote script or +style dependencies, and publishes an explicit empty resource/connect CSP. Add +only exact HTTPS origins if future component assets or requests require them. + +## Connect ChatGPT + +The bundled `.mcp.json` is for local plugin hosts. A ChatGPT connection still +requires a publicly reachable Streamable HTTP endpoint (VidXP serves `/mcp`), +an HTTPS deployment or secure development tunnel, and registration in ChatGPT +Developer Mode. + +Do not add a placeholder `.app.json`. That file can reference only the real app +identifier issued after the remote MCP connection is registered. Once that ID +exists, add the descriptor to the plugin bundle and validate the deployed app +through ChatGPT Developer Mode. + +Before public submission, also complete the current OpenAI review requirements, +including organization verification, public endpoint availability, privacy and +support URLs, accurate tool metadata, and CSP validation. + +## Official references + +- [Plugin architecture](https://developers.openai.com/plugins/concepts/plugins) +- [Package a plugin](https://developers.openai.com/plugins/build/plugins) +- [Add a ChatGPT UI](https://developers.openai.com/plugins/build/chatgpt-ui) +- [UI guidelines](https://developers.openai.com/plugins/concepts/ui-guidelines) +- [Connect from ChatGPT](https://developers.openai.com/plugins/deploy/connect-chatgpt) +- [App review](https://developers.openai.com/plugins/deploy/app-review) diff --git a/pyproject.toml b/pyproject.toml index e042bca..e290795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,12 +71,17 @@ include = ["vidxp*"] [tool.setuptools.package-data] vidxp = [ "assets/artifact_download/*", + "assets/mcp_app/*", "assets/upload_page/*", "benchmarks/requirements.txt", "capabilities/*/requirements.txt", "requirements/*.txt", "migrations/*.py", "migrations/versions/*.py", + "bundled_plugins/vidxp/.mcp.json", + "bundled_plugins/vidxp/.codex-plugin/plugin.json", + "bundled_plugins/vidxp/skills/*/SKILL.md", + "bundled_plugins/vidxp/skills/*/agents/*.yaml", ] [tool.setuptools.dynamic.optional-dependencies] diff --git a/release-please-config.json b/release-please-config.json index 0eba41b..5ecd2f6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -16,6 +16,11 @@ "path": "uv.lock", "type": "generic" }, + { + "jsonpath": "$.version", + "path": "src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json", + "type": "json" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/release-please-config.stable.json b/release-please-config.stable.json index 3c95045..45d5255 100644 --- a/release-please-config.stable.json +++ b/release-please-config.stable.json @@ -19,6 +19,11 @@ "path": "uv.lock", "type": "generic" }, + { + "jsonpath": "$.version", + "path": "src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json", + "type": "json" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/src/vidxp/assets/mcp_app/index.html b/src/vidxp/assets/mcp_app/index.html new file mode 100644 index 0000000..3a46add --- /dev/null +++ b/src/vidxp/assets/mcp_app/index.html @@ -0,0 +1,469 @@ + + + + + + VidXP evidence review + + + +
+
+
+

VidXP

+

Preparing video workspace…

+

Waiting for the tool result.

+
+ +
+
+

+
+ + + diff --git a/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json b/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json new file mode 100644 index 0000000..4232648 --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json @@ -0,0 +1,28 @@ +{ + "name": "vidxp", + "version": "0.4.0-b.3", + "description": "Ingest, index, search, and inspect video evidence with VidXP.", + "author": { + "name": "Grayhat" + }, + "homepage": "https://github.com/grayhatdevelopers/vidxp", + "repository": "https://github.com/grayhatdevelopers/vidxp", + "license": "MIT", + "keywords": ["video", "search", "evidence", "mcp"], + "skills": "./skills/", + "interface": { + "displayName": "VidXP", + "shortDescription": "Search video and inspect grounded evidence.", + "longDescription": "Use VidXP to ingest and index videos, search dialogue and scenes, answer grounded questions, and review evidence boards, frames, and clips.", + "developerName": "Grayhat", + "category": "Productivity", + "capabilities": ["Read", "Write", "Interactive"], + "websiteURL": "https://github.com/grayhatdevelopers/vidxp", + "defaultPrompt": [ + "Find and verify moments in my indexed videos.", + "Ingest and index a video with VidXP." + ], + "brandColor": "#6D5EF7" + }, + "mcpServers": "./.mcp.json" +} diff --git a/src/vidxp/bundled_plugins/vidxp/.mcp.json b/src/vidxp/bundled_plugins/vidxp/.mcp.json new file mode 100644 index 0000000..bed572e --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "vidxp": { + "command": "vidxp-mcp", + "args": ["--repository", "default"] + } + } +} diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md new file mode 100644 index 0000000..0f8027b --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md @@ -0,0 +1,56 @@ +--- +name: vidxp-find-video-evidence +description: Use VidXP to search indexed videos, answer grounded questions about video content, locate when people, actions, dialogue, or scenes occur, and return inspectable evidence boards, keyframes, or clips. Trigger for requests such as "find where X appears," "when does Y happen," "what is said," "what happens," or "show me the matching clip," even when the user does not name VidXP. Do not trigger for ingesting new media or ordinary video editing. +--- + +# Find video evidence with VidXP + +## Workflow + +1. Resolve the `vidxp` MCP tools, then call `get_workspace`. If the requested + video is not indexed, explain that it must be indexed first. +2. Use `search_moments` to locate moments or `query_video` for a synthesized, + grounded answer. Use `command.query` with `search_moments` and + `command.question` with `query_video`. Set `command.media_id` when the user + means one video. +3. Omit `command.evidence_delivery` for the normal path. The completed job + includes an annotated board covering the ranked results. +4. Call `wait_job` for bounded waits. Pass its `observation_token` as + `after_observation_token` on the next wait. When terminal, call + `get_job_evidence` once. It returns the concise evidence index and visual + content without the full structured job dump. Use `get_job` only when exact + machine fields not present in that index are actually needed. Search and + query may take time; update the user + when the stage changes or about once per minute, never after every wait and + never with an invented ETA. +5. Inspect and show the returned board before making visual claims. Use its tile + evidence IDs for follow-up: + - `materialize_job_evidence` accepts up to ten selected IDs and returns + model-visible standalone keyframes or clip links without rerunning retrieval. + - `create_evidence_board` is only for a custom selection or the + `next_start_rank` continuation; wait on its returned job ID the same way. +6. When standalone artifacts are required in the initial job, put exactly this + inside `command`: `"evidence_delivery": {"mode": + "keyframes_and_clips", "max_items": 3}`. Never send + `command.materialize`. + +## Actor scope + +- Actor data is available through `query_video`, not name search. It represents + anonymous, video-scoped face clusters—not a named or cross-video identity. +- Treat its image as a representative full frame, not an exact face crop or + proof of continuous presence. Do not claim exhaustive named appearances. +- Use scene evidence plus board inspection for named-person requests, and label + uncertain matches as candidates. + +## Output + +- The final response must visibly embed a returned board or frame, or include a + working downloadable resource link—not timestamps alone. Use the returned + `local_path` or `download_url`; never write an unlinked label such as “View + evidence board.” Use `get_artifact_download` only if neither is returned. +- Preserve the source job and evidence IDs. Describe scores as retrieval scores, + and distinguish a visible appearance from a dialogue or caption mention. +- Stop waiting on success, failure, or cancellation. An empty result means no + matching indexed evidence was found, not that the event is absent from the + original video. diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml new file mode 100644 index 0000000..8324467 --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml @@ -0,0 +1,13 @@ +interface: + display_name: "Find Video Evidence with VidXP" + short_description: "Search indexed videos and return verifiable evidence" + default_prompt: "Use $vidxp-find-video-evidence to find and verify moments in my indexed videos." + +policy: + allow_implicit_invocation: true + +dependencies: + tools: + - type: "mcp" + value: "vidxp" + description: "VidXP video search and evidence tools" diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md new file mode 100644 index 0000000..198023f --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md @@ -0,0 +1,35 @@ +--- +name: vidxp-ingest-video +description: Use VidXP to upload, import, register, and automatically index video files through its MCP tools. Trigger for requests to add, upload, ingest, import, register, or index one or more videos, including attached videos and accessible local paths, even when the user does not name VidXP. Do not trigger for editing, transcoding, or searching a video that is already indexed. +--- + +# Ingest video with VidXP + +## Workflow + +1. Resolve the `vidxp` MCP tools and call `get_workspace`. Do not import a video + that is already registered or indexed. +2. Choose indexable modalities from the workspace. Use `dialogue` and `scene` + for ordinary content retrieval. Add `actor` only when anonymous recurring-face + clusters are wanted; it does not identify people by name. +3. Call `get_runtime_readiness`. If selected models are missing, submit + `prepare_models`, use `wait_job` with its observation token for subsequent + bounded waits, then fetch `get_job` once when terminal. +4. Use `ingest_local_media` for one to ten paths accessible to VidXP; otherwise + use `create_media_upload` and give the returned link to the user. Keep + `index_after_import` enabled unless registration-only behavior was requested. +5. Poll the returned ingestion or upload ID with `get_media_ingestion` or + `get_media_upload`. Honor its poll interval, reuse the same identifiers, and + do not resubmit unchanged work. +6. Stop at a terminal state and report each file's state, media ID, index job, + and searchable snapshot or generation. If indexing fails after registration, + retry with `start_indexing`; do not upload the file again. +7. If the request also asks about the video, continue directly into the VidXP + evidence workflow once it is searchable. + +## Long operations + +- Tell the user that model preparation and indexing can take several minutes. +- Update when the stage changes or about once per minute; do not narrate every + status check or invent an ETA. +- Treat files independently so one failure does not hide successful siblings. diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml new file mode 100644 index 0000000..f502ebe --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml @@ -0,0 +1,13 @@ +interface: + display_name: "Ingest Video with VidXP" + short_description: "Upload or ingest videos and index them with VidXP" + default_prompt: "Use $vidxp-ingest-video to ingest and index my video with VidXP." + +policy: + allow_implicit_invocation: true + +dependencies: + tools: + - type: "mcp" + value: "vidxp" + description: "VidXP video ingestion and indexing tools" diff --git a/src/vidxp/mcp.py b/src/vidxp/mcp.py index 991afb7..c2f25ed 100644 --- a/src/vidxp/mcp.py +++ b/src/vidxp/mcp.py @@ -103,6 +103,11 @@ scoped_job_id, scoped_request_key, ) +from vidxp.mcp_app import ( + MCP_APP_MIME_TYPE, + MCP_APP_RESOURCE_URI, + load_mcp_app_html, +) from vidxp.core.identifiers import ArtifactId from vidxp.evidence_delivery import ( EvidenceDeliveryService, @@ -149,6 +154,15 @@ ) +def _mcp_app_tool_meta(invoking: str, invoked: str) -> dict[str, object]: + return { + "ui": {"resourceUri": MCP_APP_RESOURCE_URI}, + "openai/outputTemplate": MCP_APP_RESOURCE_URI, + "openai/toolInvocation/invoking": invoking, + "openai/toolInvocation/invoked": invoked, + } + + class PrincipalBridge: """Carry the principal validated by the outer ASGI boundary into tools.""" @@ -547,6 +561,27 @@ async def lifecycle(_server): lifespan=lifecycle, ) + @server.resource( + MCP_APP_RESOURCE_URI, + name="vidxp_mcp_app", + title="VidXP video workspace", + description=( + "Interactive upload progress and evidence review for MCP Apps hosts." + ), + mime_type=MCP_APP_MIME_TYPE, + meta={ + "ui": { + "prefersBorder": True, + "csp": { + "connectDomains": [], + "resourceDomains": [], + }, + } + }, + ) + async def read_mcp_app() -> str: + return load_mcp_app_html() + async def artifact_bytes( artifact_id: ArtifactId, *, @@ -1061,9 +1096,125 @@ def evidence_index( ) return "\n".join(lines) - async def evidence_content( + def evidence_app_payload( + *, job: Job, - ) -> list[ImageContent | ResourceLink | TextContent]: + source_job_id: JobId, + delivery: EvidenceDeliveryResult, + query_result: QueryAnswer | None, + ) -> dict[str, object]: + board = delivery.board + pages: list[dict[str, object]] = [] + tiles: list[dict[str, object]] = [] + if board is not None: + for page in board.pages: + artifact = page.artifact + delivery_info = artifact.delivery + pages.append( + { + "page_number": page.page_number, + "media_id": page.media_id, + "width": page.width, + "height": page.height, + "tile_ids": list(page.tile_ids), + "resource_uri": artifact.resource_uri, + "download_url": ( + delivery_info.download_url + if delivery_info is not None + else None + ), + } + ) + for tile in board.tiles: + tiles.append( + { + "evidence_id": tile.evidence_id, + "rank": tile.rank, + "page_number": tile.page_number, + "position": tile.position, + "media_id": tile.media_id, + "modalities": list(tile.modalities), + "start": tile.start, + "end": tile.end, + "display_text": concise_text(tile.display_text), + "state": tile.state.value, + } + ) + requested_count = board.requested_count + rendered_count = board.rendered_count + failed_count = board.failed_count + next_start_rank = board.next_start_rank + else: + for item in delivery.items: + resolved = item.range + tiles.append( + { + "evidence_id": item.evidence_id, + "rank": item.rank, + "page_number": None, + "position": item.rank, + "media_id": item.media_id, + "modalities": list(item.modalities), + "start": ( + resolved.source_start_seconds + if resolved is not None + else 0.0 + ), + "end": ( + resolved.source_end_seconds + if resolved is not None + else 0.0 + ), + "display_text": None, + "state": item.state.value, + } + ) + requested_count = len(delivery.items) + rendered_count = sum( + item.state.value == "ready" for item in delivery.items + ) + failed_count = requested_count - rendered_count + next_start_rank = None + + answer: dict[str, object] | None = None + if query_result is not None: + answer = { + "mode": query_result.mode.value, + "claims": [ + { + "text": concise_text(claim.text, limit=512) or "", + "evidence_ids": list(claim.evidence_ids), + } + for claim in query_result.claims + ], + "fallback_reason": concise_text( + query_result.fallback_reason, + limit=512, + ), + } + + return { + "view": "evidence", + "job_id": job.job_id, + "source_job_id": source_job_id, + "job_kind": job.kind.value, + "answer": answer, + "board": { + "requested_count": requested_count, + "rendered_count": rendered_count, + "failed_count": failed_count, + "next_start_rank": next_start_rank, + "pages": pages, + "tiles": tiles, + }, + } + + async def evidence_presentation( + job: Job, + ) -> tuple[ + dict[str, object], + list[ImageContent | ResourceLink | TextContent], + ]: query_result = None if job.kind in {JobKind.search, JobKind.query}: result = job.result.result @@ -1082,18 +1233,27 @@ async def evidence_content( delivery=projected_delivery, query_result=query_result, ) + source_job_id = job.job_id else: board = job.result.result projected_board, blocks = await project_evidence_board(board) + projected_delivery = EvidenceDeliveryResult( + policy=EvidenceDeliveryPolicy(mode=EvidenceDeliveryMode.none), + items=(), + board=projected_board, + ) index = evidence_index( source_job_id=board.source_job_id, - delivery=EvidenceDeliveryResult( - policy=EvidenceDeliveryPolicy(mode=EvidenceDeliveryMode.none), - items=(), - board=projected_board, - ), + delivery=projected_delivery, ) - return [TextContent(type="text", text=index), *blocks] + source_job_id = board.source_job_id + payload = evidence_app_payload( + job=job, + source_job_id=source_job_id, + delivery=projected_delivery, + query_result=query_result, + ) + return payload, [TextContent(type="text", text=index), *blocks] def completed_evidence_result( source_job_id: JobId, @@ -1227,6 +1387,10 @@ async def get_media(media_id: MediaId) -> MediaAsset: "Automatic indexing defaults on. Poll only get_media_upload." ), annotations=_SUBMIT, + meta=_mcp_app_tool_meta( + "Creating a VidXP upload session…", + "VidXP upload session ready.", + ), structured_output=True, ) async def create_media_upload( @@ -1826,10 +1990,14 @@ async def get_job(job_id: JobId) -> Job: description=( "Present a completed search, query, or evidence-board job as a " "concise evidence index plus model-visible board images and resource " - "links. This intentionally omits structuredContent; use get_job only " - "when the full machine record is actually needed." + "links. The compact structured result drives the optional VidXP " + "evidence-review UI without exposing the full machine record." ), annotations=_READ_ONLY, + meta=_mcp_app_tool_meta( + "Opening VidXP evidence…", + "VidXP evidence ready.", + ), ) async def get_job_evidence(job_id: JobId) -> CallToolResult: def completed_evidence_job(_actor: Principal) -> Job: @@ -1855,10 +2023,13 @@ def completed_evidence_job(_actor: Principal) -> Job: operation=completed_evidence_job, ) try: - blocks = await evidence_content(job) + structured_content, blocks = await evidence_presentation(job) except ApplicationError as exc: raise _application_error(exc) from exc - return CallToolResult(content=blocks) + return CallToolResult( + content=blocks, + structured_content=structured_content, + ) @server.tool( title="Get compact job status", diff --git a/src/vidxp/mcp_app.py b/src/vidxp/mcp_app.py new file mode 100644 index 0000000..8794824 --- /dev/null +++ b/src/vidxp/mcp_app.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from functools import lru_cache +from importlib.resources import files + + +MCP_APP_RESOURCE_URI = "ui://vidxp/evidence-review-v1.html" +MCP_APP_MIME_TYPE = "text/html;profile=mcp-app" + + +@lru_cache(maxsize=1) +def load_mcp_app_html() -> str: + """Load the self-contained MCP App resource shipped with VidXP.""" + + return ( + files("vidxp") + .joinpath("assets", "mcp_app", "index.html") + .read_text(encoding="utf-8") + ) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index c6f1834..e7f1782 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -72,6 +72,7 @@ SearchMomentsPlanStep, WorkspaceOverview, ) +from vidxp.mcp_app import MCP_APP_MIME_TYPE, MCP_APP_RESOURCE_URI from vidxp.authentication import ( AuthenticatedBearer, OIDCBearerAuthenticator, @@ -439,12 +440,38 @@ async def test_curated_tools_publish_their_intended_output_contracts(self): async with Client(server) as client: discovered = await client.list_tools() result = await client.call_tool("list_capabilities", {}) + app_resource = await client.read_resource(MCP_APP_RESOURCE_URI) self.assertEqual( [tool.name for tool in discovered.tools], MCP_TOOL_NAMES, ) tools = {tool.name: tool for tool in discovered.tools} + for name in ("create_media_upload", "get_job_evidence"): + self.assertEqual( + tools[name].meta["ui"]["resourceUri"], + MCP_APP_RESOURCE_URI, + ) + self.assertEqual( + tools[name].meta["openai/outputTemplate"], + MCP_APP_RESOURCE_URI, + ) + app_contents = app_resource.contents[0] + self.assertEqual(app_contents.mime_type, MCP_APP_MIME_TYPE) + self.assertEqual( + app_contents.meta["ui"]["csp"], + {"connectDomains": [], "resourceDomains": []}, + ) + self.assertIn("ui/notifications/tool-result", app_contents.text) + self.assertIn('request("ui/initialize"', app_contents.text) + self.assertIn('notify("ui/notifications/initialized"', app_contents.text) + self.assertIn('request("tools/call"', app_contents.text) + self.assertIn('request("ui/open-link"', app_contents.text) + self.assertIn('request("ui/request-display-mode"', app_contents.text) + self.assertIn('request("ui/update-model-context"', app_contents.text) + self.assertIn("window.openai?.requestDisplayMode", app_contents.text) + self.assertIn("materialize_job_evidence", app_contents.text) + self.assertNotIn("