diff --git a/MANIFEST.in b/MANIFEST.in index cdda21d..bdb0400 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,10 @@ include LICENSE include docs/images/logo.png recursive-include src/vidxp/assets/upload_page * recursive-include src/vidxp/assets/artifact_download * +recursive-include src/vidxp/assets/mcp_app * +include src/vidxp/bundled_plugins/vidxp/.mcp.json +include src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json +recursive-include src/vidxp/bundled_plugins/vidxp/skills * include web/upload-page/package.json include web/upload-page/package-lock.json include web/upload-page/scripts/build.mjs diff --git a/README.md b/README.md index 7dcc62f..2926077 100644 --- a/README.md +++ b/README.md @@ -166,11 +166,16 @@ VidXP includes reusable skill source folders for the two common agent workflows: - [Ingest and index videos](skills/vidxp-ingest-video/SKILL.md) - [Find moments and return inspectable evidence](skills/vidxp-find-video-evidence/SKILL.md) -Download a skill folder and add it through a supported ChatGPT desktop or Codex -Skills surface. The skills require a connected VidXP MCP server; installable -plugin packaging for additional ChatGPT surfaces will follow separately. +Download a skill folder directly, or install the versioned VidXP plugin bundle +shipped inside the Python package. The plugin keeps both skills and the local +`vidxp-mcp` server definition together. Its MCP App resource also gives +compatible hosts an interactive upload and evidence-review view; every workflow +continues to work through ordinary MCP tool results when a host has no UI. +When the MCP feature is installed, VidXP Desktop can configure another MCP +client or install the complete local plugin directly into Codex. - [Python, HTTP, and MCP installation](INSTALLATION_GUIDE.md) +- [ChatGPT and Codex plugin integration](docs/integrations/openai-plugin.md) - [Optional capability packages](INSTALLATION_GUIDE.md#optional-dependency-extras) - [Coolify server setup](docs/deployment/coolify.md) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 1ab1967..1dcc5c8 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -486,6 +486,17 @@ struct LocalWorkerStatus { detail: String, } +#[derive(Clone, Debug, Deserialize, Serialize)] +struct CodexPluginInstallResult { + plugin_name: String, + plugin_id: Option, + plugin_version: String, + marketplace_name: String, + marketplace_path: String, + installed_path: Option, + detail: String, +} + #[derive(Clone)] struct TrayMenuItems { installation: MenuItem, @@ -3641,6 +3652,49 @@ async fn mcp_client_config( .map_err(|error| format!("MCP configuration stopped unexpectedly: {error}"))? } +#[tauri::command] +async fn install_codex_plugin( + app: AppHandle, + state: tauri::State<'_, DesktopState>, +) -> Result { + let _active = state.active_operations.register()?; + tauri::async_runtime::spawn_blocking(move || { + let (profile, paths) = selected_target_context(&app)?; + if !profile + .surfaces + .iter() + .any(|surface| surface == "mcp" || surface == "server") + { + return Err( + "The selected VidXP installation does not expose an installed MCP surface.".into(), + ); + } + let installer_path = target_companion_executable(&profile, "vidxp-codex-plugin"); + if !installer_path.is_file() { + return Err(format!( + "The selected installation did not provide {}. Update VidXP and try again.", + installer_path.display() + )); + } + let marketplace_root = paths.private_data.join("codex-marketplace"); + let mut command = target_command(&profile, &paths, &installer_path); + command + .arg("--marketplace-root") + .arg(&marketplace_root) + .arg("--repository") + .arg("default") + .arg("--index-directory") + .arg(&profile.repository_root) + .arg("--data-dir") + .arg(&profile.data_root); + let output = checked_output(command, "VidXP Codex plugin setup")?; + serde_json::from_slice(&output.stdout) + .map_err(|error| format!("VidXP returned invalid Codex setup details: {error}")) + }) + .await + .map_err(|error| format!("Codex plugin setup stopped unexpectedly: {error}"))? +} + fn execute_worker_action(app: &AppHandle, action: &str) -> Result { let (profile, paths) = selected_target_context(app)?; if !profile.surfaces.iter().any(|surface| surface == "worker") { @@ -4601,6 +4655,7 @@ pub fn run() { target_doctor, configure_external_installation, mcp_client_config, + install_codex_plugin, local_worker_status, start_local_worker, stop_local_worker, diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 80d7276..7020171 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -12,7 +12,7 @@ const mocks = vi.hoisted(() => ({ prepareManagedModels: vi.fn(), onManagedSetupProgress: vi.fn(), runtimeManifest: vi.fn(), runtimeStatus: vi.fn(), launchUi: vi.fn(), chooseModelDirectory: vi.fn(), modelDirectoryInventory: vi.fn(), - targetDoctor: vi.fn(), mcpClientConfig: vi.fn(), localServerStatus: vi.fn(), localWorkerStatus: vi.fn(), browserServiceStatus: vi.fn(), + targetDoctor: vi.fn(), mcpClientConfig: vi.fn(), installCodexPlugin: vi.fn(), localServerStatus: vi.fn(), localWorkerStatus: vi.fn(), browserServiceStatus: vi.fn(), startLocalServer: vi.fn(), startSharedServer: vi.fn(), stopLocalServer: vi.fn(), startSharedBrowser: vi.fn(), stopBrowserService: vi.fn(), startLocalWorker: vi.fn(), stopLocalWorker: vi.fn(), configureExternalInstallation: vi.fn(), })); @@ -107,6 +107,12 @@ describe('desktop target lifecycle', () => { mocks.launchUi.mockResolvedValue(undefined); mocks.targetDoctor.mockResolvedValue({ ok: true, modalities: ['scene'], checks: [{ capability: 'media', kind: 'distribution', name: 'ffmpeg', ok: true }] }); mocks.mcpClientConfig.mockResolvedValue('{"mcpServers":{"vidxp":{"command":"vidxp-mcp"}}}'); + mocks.installCodexPlugin.mockResolvedValue({ + plugin_name: 'vidxp', plugin_id: 'vidxp@vidxp-local', plugin_version: '0.4.0+codex.1234', + marketplace_name: 'vidxp-local', marketplace_path: 'C:\\Data\\codex-marketplace\\.agents\\plugins\\marketplace.json', + installed_path: 'C:\\Users\\test\\.codex\\plugins\\vidxp', + detail: 'VidXP is installed in Codex with its MCP server and skills. Start a new Codex chat to use the updated plugin.', + }); mocks.browserServiceStatus.mockResolvedValue({ state: 'stopped', running: false, shared: false, port: null, local_url: null, network_url: null, detail: 'Stopped.' }); mocks.startSharedBrowser.mockResolvedValue({ state: 'ready', running: true, shared: true, port: 8501, local_url: 'http://127.0.0.1:8501', network_url: 'http://192.168.1.20:8501', detail: 'Shared.' }); mocks.stopBrowserService.mockResolvedValue({ state: 'stopped', running: false, shared: false, port: null, local_url: null, network_url: null, detail: 'Stopped.' }); @@ -176,7 +182,11 @@ describe('desktop target lifecycle', () => { expect(await screen.findByText('VidXP is ready')).toBeVisible(); expect(mocks.targetDoctor).toHaveBeenCalledTimes(1); - await user.click(screen.getByRole('button', { name: 'Set up connection' })); + await user.click(screen.getByRole('button', { name: 'Set up in Codex' })); + expect(await screen.findByText('VidXP is installed in Codex with its MCP server and skills. Start a new Codex chat to use the updated plugin.')).toBeVisible(); + expect(mocks.installCodexPlugin).toHaveBeenCalledTimes(1); + + await user.click(screen.getByRole('button', { name: 'Copy MCP setup' })); expect(await screen.findByRole('heading', { name: 'Connect an AI assistant' })).toBeVisible(); expect(mocks.mcpClientConfig).toHaveBeenCalledTimes(1); @@ -211,7 +221,8 @@ describe('desktop target lifecycle', () => { await user.click(screen.getByRole('button', { name: 'Apply changes' })); await waitFor(() => expect(mocks.configureExternalInstallation).toHaveBeenCalledWith([], ['worker', 'browser', 'mcp'])); - expect(await screen.findByRole('button', { name: 'Set up connection' })).toBeVisible(); + expect(await screen.findByRole('button', { name: 'Set up in Codex' })).toBeVisible(); + expect(screen.getByRole('button', { name: 'Copy MCP setup' })).toBeVisible(); }); it('offers an in-place manifest update when the selected runtime contract is too old', async () => { diff --git a/desktop/src/components/TargetSummary.tsx b/desktop/src/components/TargetSummary.tsx index cd5d9bd..cc0381c 100644 --- a/desktop/src/components/TargetSummary.tsx +++ b/desktop/src/components/TargetSummary.tsx @@ -1,11 +1,12 @@ import { Alert, Badge, Button, Checkbox, Code, Group, Loader, Modal, Stack, Text, Title } from '@mantine/core'; -import { IconActivityHeartbeat, IconCopy, IconExternalLink, IconPlayerPlay, IconPlayerStop, IconRefresh, IconSettings, IconShare, IconTerminal2 } from '@tabler/icons-react'; +import { IconActivityHeartbeat, IconCopy, IconExternalLink, IconPlugConnected, IconPlayerPlay, IconPlayerStop, IconRefresh, IconSettings, IconShare, IconTerminal2 } from '@tabler/icons-react'; import { useEffect, useState } from 'react'; import { errorMessage, browserServiceStatus, configureExternalInstallation, + installCodexPlugin, localServerStatus, localWorkerStatus, mcpClientConfig, @@ -20,6 +21,7 @@ import { targetDoctor, type DoctorReport, type BrowserServiceStatus, + type CodexPluginInstallResult, type LocalServerStatus, type LocalWorkerStatus, type RuntimeManifest, @@ -55,7 +57,8 @@ export function TargetSummary({ profile, validationError, checking, operationPen const [browser, setBrowser] = useState(null); const [worker, setWorker] = useState(null); const [mcpConfig, setMcpConfig] = useState(null); - const [busy, setBusy] = useState<'doctor' | 'config' | 'features' | 'worker-start' | 'worker-stop' | 'browser-share' | 'browser-stop' | 'server-start' | 'server-share' | 'server-stop' | null>(null); + const [codexSetup, setCodexSetup] = useState(null); + const [busy, setBusy] = useState<'doctor' | 'config' | 'codex' | 'features' | 'worker-start' | 'worker-stop' | 'browser-share' | 'browser-stop' | 'server-start' | 'server-share' | 'server-stop' | null>(null); const [runtimeFailure, setRuntimeFailure] = useState(null); const [copied, setCopied] = useState(false); const [shareCopied, setShareCopied] = useState(false); @@ -81,6 +84,7 @@ export function TargetSummary({ profile, validationError, checking, operationPen useEffect(() => { setDoctor(null); setMcpConfig(null); + setCodexSetup(null); setRuntimeFailure(null); if (!serverAvailable) { setServer(null); @@ -178,6 +182,19 @@ export function TargetSummary({ profile, validationError, checking, operationPen } } + async function setupCodex() { + setBusy('codex'); + setRuntimeFailure(null); + setCodexSetup(null); + try { + setCodexSetup(await installCodexPlugin()); + } catch (error) { + setRuntimeFailure(errorMessage(error, 'VidXP could not install the Codex plugin.')); + } finally { + setBusy(null); + } + } + async function openExternalSetup() { setBusy('features'); setRuntimeFailure(null); @@ -379,10 +396,18 @@ export function TargetSummary({ profile, validationError, checking, operationPen } {mcpAvailable && -
AI assistant integrationCreate the MCP setup needed to use VidXP from a compatible AI assistant.
- +
AI assistant integrationInstall VidXP's MCP server and skills in Codex, or copy the MCP setup for another compatible assistant.
+ + + + } + {codexSetup && + {codexSetup.detail} +
Installation detailsPlugin {codexSetup.plugin_version} from {codexSetup.marketplace_name}{codexSetup.installed_path && {codexSetup.installed_path}}
+ } + {serverAvailable &&
App integration service diff --git a/desktop/src/tauri.test.ts b/desktop/src/tauri.test.ts index 8ce6466..658dbd6 100644 --- a/desktop/src/tauri.test.ts +++ b/desktop/src/tauri.test.ts @@ -9,6 +9,7 @@ import { beginManagedSetup, displayPath, installRuntime, + installCodexPlugin, configureExternalInstallation, browserServiceStatus, localServerStatus, @@ -108,6 +109,7 @@ describe('desktop IPC adapter', () => { await targetDoctor(); await mcpClientConfig(); + await installCodexPlugin(); await localWorkerStatus(); await startLocalWorker(); await stopLocalWorker(); @@ -121,16 +123,17 @@ describe('desktop IPC adapter', () => { expect(invoke).toHaveBeenNthCalledWith(1, 'target_doctor'); expect(invoke).toHaveBeenNthCalledWith(2, 'mcp_client_config'); - expect(invoke).toHaveBeenNthCalledWith(3, 'local_worker_status'); - expect(invoke).toHaveBeenNthCalledWith(4, 'start_local_worker'); - expect(invoke).toHaveBeenNthCalledWith(5, 'stop_local_worker'); - expect(invoke).toHaveBeenNthCalledWith(6, 'browser_service_status'); - expect(invoke).toHaveBeenNthCalledWith(7, 'start_shared_browser'); - expect(invoke).toHaveBeenNthCalledWith(8, 'stop_browser_service'); - expect(invoke).toHaveBeenNthCalledWith(9, 'local_server_status'); - expect(invoke).toHaveBeenNthCalledWith(10, 'start_local_server'); - expect(invoke).toHaveBeenNthCalledWith(11, 'start_shared_server'); - expect(invoke).toHaveBeenNthCalledWith(12, 'stop_local_server'); + expect(invoke).toHaveBeenNthCalledWith(3, 'install_codex_plugin'); + expect(invoke).toHaveBeenNthCalledWith(4, 'local_worker_status'); + expect(invoke).toHaveBeenNthCalledWith(5, 'start_local_worker'); + expect(invoke).toHaveBeenNthCalledWith(6, 'stop_local_worker'); + expect(invoke).toHaveBeenNthCalledWith(7, 'browser_service_status'); + expect(invoke).toHaveBeenNthCalledWith(8, 'start_shared_browser'); + expect(invoke).toHaveBeenNthCalledWith(9, 'stop_browser_service'); + expect(invoke).toHaveBeenNthCalledWith(10, 'local_server_status'); + expect(invoke).toHaveBeenNthCalledWith(11, 'start_local_server'); + expect(invoke).toHaveBeenNthCalledWith(12, 'start_shared_server'); + expect(invoke).toHaveBeenNthCalledWith(13, 'stop_local_server'); }); it('adds optional surfaces to the selected existing installation', async () => { diff --git a/desktop/src/tauri.ts b/desktop/src/tauri.ts index 123ac90..78df41e 100644 --- a/desktop/src/tauri.ts +++ b/desktop/src/tauri.ts @@ -254,6 +254,16 @@ export interface LocalWorkerStatus { detail: string; } +export interface CodexPluginInstallResult { + plugin_name: string; + plugin_id: string | null; + plugin_version: string; + marketplace_name: string; + marketplace_path: string; + installed_path: string | null; + detail: string; +} + interface WireInstallTransitionResult { install: InstallRuntimeResult; setup: WireTargetState; @@ -374,6 +384,7 @@ export function configureExternalInstallation(capabilities: string[], surfaces: return invoke('configure_external_installation', { capabilities, surfaces }).then(normalizeState); } export function mcpClientConfig(): Promise { return invoke('mcp_client_config'); } +export function installCodexPlugin(): Promise { return invoke('install_codex_plugin'); } export function localWorkerStatus(): Promise { return invoke('local_worker_status'); } export function startLocalWorker(): Promise { return invoke('start_local_worker'); } export function stopLocalWorker(): Promise { return invoke('stop_local_worker'); } diff --git a/docs/desktop.md b/docs/desktop.md index a755a3a..a899996 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -187,8 +187,18 @@ one browser tab. Closing a configured control panel hides it to the tray. action, and **Quit VidXP** runs supervised shutdown for the interface and any Desktop-owned repository worker. The active-target panel also reports and controls local video processing through the existing `JobService` worker -lifecycle, reports the app integration service, creates AI-assistant MCP config, -and presents the existing doctor result as product health rather than raw output. +lifecycle, reports the app integration service, creates copyable AI-assistant +MCP config, installs the bundled MCP-and-skills plugin directly into Codex, and +presents the existing doctor result as product health rather than raw output. +Codex setup exports a target-specific copy of the plugin from the selected +VidXP runtime into a dedicated Desktop-private `vidxp-local` marketplace. The +copy keeps the versioned skills but replaces its generic MCP command with the +selected runtime's absolute executable, repository, and data paths. Desktop +then uses `codex plugin marketplace add --json` and +`codex plugin add vidxp@vidxp-local --json`; it never edits Codex configuration +or the user's personal marketplace by hand. Re-running setup refreshes the +managed bundle using a deterministic plugin-version cache key. A new Codex chat +is required before the refreshed skills and tools are available. The browser and app integration service remain loopback-only by default. Explicit sharing controls compose their existing `--share` modes: browser sharing reports its LAN URL and warns that it is unauthenticated, while API/MCP diff --git a/docs/integrations/openai-plugin.md b/docs/integrations/openai-plugin.md new file mode 100644 index 0000000..df86622 --- /dev/null +++ b/docs/integrations/openai-plugin.md @@ -0,0 +1,90 @@ +# ChatGPT and Codex plugin integration + +VidXP ships one versioned plugin bundle with its Python distribution. The +bundle combines the local MCP server definition with VidXP's canonical ingest +and evidence-search skills, while the MCP server exposes an optional interactive +view for hosts that implement MCP Apps. + +The packaged bundle lives at +`src/vidxp/bundled_plugins/vidxp/` and contains: + +- `.codex-plugin/plugin.json`, the plugin manifest; +- `.mcp.json`, which starts the installed `vidxp-mcp` command; and +- `skills/`, a release snapshot of the canonical root `skills/` folders. + +The root skill folders remain the authoring source. Packaging tests require the +bundled snapshot to match their file inventory and text exactly, and Release +Please keeps the plugin manifest version aligned with the Python package. + +## Install in Codex + +With **AI assistant integration** enabled, VidXP Desktop shows two distinct +actions: + +- **Set up in Codex** exports a target-specific copy of this bundle to a + Desktop-managed `vidxp-local` marketplace, registers that marketplace through + the Codex CLI, and installs the plugin. The generated `.mcp.json` pins the + selected installation's absolute `vidxp-mcp` command, repository, and data + paths, so Codex does not depend on its process `PATH`. +- **Copy MCP setup** retains the transport-only JSON flow for other compatible + local MCP clients. + +The Codex action installs the MCP server and both skills as one unit. It uses +the documented JSON forms of `codex plugin marketplace add` and +`codex plugin add`, leaves personal marketplace files and `config.toml` +untouched, and asks the user to start a new Codex chat after installation. The +exported marketplace lives in VidXP Desktop's private application-data +directory. Its catalog is written to the Codex marketplace contract at +`.agents/plugins/marketplace.json`, with the plugin at `plugins/vidxp/`. The +exporter also removes the obsolete root-level `marketplace.json` from earlier +VidXP-managed exports. The canonical distributable bundle remains checked into +this repository and packaged in every VidXP wheel. + +## Interactive MCP App + +`create_media_upload` and `get_job_evidence` advertise the same +`ui://vidxp/evidence-review-v1.html` resource. A compatible host can render it +inline to: + +- open VidXP's short-lived HTTPS upload page and refresh session progress; +- review answer claims and annotated evidence-board pages; +- select up to ten ranked evidence IDs; and +- request exact keyframes or clips with `materialize_job_evidence` without + rerunning retrieval. + +The component completes the MCP Apps `ui/initialize` handshake, then uses the +standard `tools/call`, `ui/open-link`, `ui/request-display-mode`, +`ui/update-model-context`, and resize messages. ChatGPT-specific `window.openai` +helpers are feature-detected only as compatibility fallbacks and for ephemeral +widget state. VidXP remains authoritative for upload, job, search, and artifact +state, and non-UI clients receive the existing text, image, and resource-link +content. + +The resource is self-contained, uses system fonts, has no remote script or +style dependencies, and publishes an explicit empty resource/connect CSP. Add +only exact HTTPS origins if future component assets or requests require them. + +## Connect ChatGPT + +The bundled `.mcp.json` is for local plugin hosts. A ChatGPT connection still +requires a publicly reachable Streamable HTTP endpoint (VidXP serves `/mcp`), +an HTTPS deployment or secure development tunnel, and registration in ChatGPT +Developer Mode. + +Do not add a placeholder `.app.json`. That file can reference only the real app +identifier issued after the remote MCP connection is registered. Once that ID +exists, add the descriptor to the plugin bundle and validate the deployed app +through ChatGPT Developer Mode. + +Before public submission, also complete the current OpenAI review requirements, +including organization verification, public endpoint availability, privacy and +support URLs, accurate tool metadata, and CSP validation. + +## Official references + +- [Plugin architecture](https://developers.openai.com/plugins/concepts/plugins) +- [Package a plugin](https://developers.openai.com/plugins/build/plugins) +- [Add a ChatGPT UI](https://developers.openai.com/plugins/build/chatgpt-ui) +- [UI guidelines](https://developers.openai.com/plugins/concepts/ui-guidelines) +- [Connect from ChatGPT](https://developers.openai.com/plugins/deploy/connect-chatgpt) +- [App review](https://developers.openai.com/plugins/deploy/app-review) diff --git a/pyproject.toml b/pyproject.toml index e042bca..0fa6398 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ vidxp-worker = "vidxp.workflow_worker:main" vidxp-database = "vidxp.database_cli:main" vidxp-hooks = "vidxp.hook_cli:main" vidxp-mcp = "vidxp.mcp_cli:main" +vidxp-codex-plugin = "vidxp.codex_plugin_cli:main" [project.urls] Homepage = "https://github.com/grayhatdevelopers/vidxp" @@ -71,12 +72,17 @@ include = ["vidxp*"] [tool.setuptools.package-data] vidxp = [ "assets/artifact_download/*", + "assets/mcp_app/*", "assets/upload_page/*", "benchmarks/requirements.txt", "capabilities/*/requirements.txt", "requirements/*.txt", "migrations/*.py", "migrations/versions/*.py", + "bundled_plugins/vidxp/.mcp.json", + "bundled_plugins/vidxp/.codex-plugin/plugin.json", + "bundled_plugins/vidxp/skills/*/SKILL.md", + "bundled_plugins/vidxp/skills/*/agents/*.yaml", ] [tool.setuptools.dynamic.optional-dependencies] diff --git a/release-please-config.json b/release-please-config.json index 0eba41b..5ecd2f6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -16,6 +16,11 @@ "path": "uv.lock", "type": "generic" }, + { + "jsonpath": "$.version", + "path": "src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json", + "type": "json" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/release-please-config.stable.json b/release-please-config.stable.json index 3c95045..45d5255 100644 --- a/release-please-config.stable.json +++ b/release-please-config.stable.json @@ -19,6 +19,11 @@ "path": "uv.lock", "type": "generic" }, + { + "jsonpath": "$.version", + "path": "src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json", + "type": "json" + }, { "path": "desktop/src-tauri/Cargo.toml", "type": "generic" diff --git a/skills/vidxp-find-video-evidence/SKILL.md b/skills/vidxp-find-video-evidence/SKILL.md index 0f8027b..d78fe75 100644 --- a/skills/vidxp-find-video-evidence/SKILL.md +++ b/skills/vidxp-find-video-evidence/SKILL.md @@ -1,6 +1,6 @@ --- name: vidxp-find-video-evidence -description: Use VidXP to search indexed videos, answer grounded questions about video content, locate when people, actions, dialogue, or scenes occur, and return inspectable evidence boards, keyframes, or clips. Trigger for requests such as "find where X appears," "when does Y happen," "what is said," "what happens," or "show me the matching clip," even when the user does not name VidXP. Do not trigger for ingesting new media or ordinary video editing. +description: Use VidXP to search indexed videos and surface inspectable evidence boards, keyframes, and clips before analysis. Trigger for requests such as "find where X appears," "when does Y happen," "what is said," "what happens," or "show me the matching clip," even when the user does not name VidXP. Favor one-pass evidence delivery and only add brief accuracy feedback; do not trigger for ingesting new media or ordinary video editing. --- # Find video evidence with VidXP @@ -9,30 +9,28 @@ description: Use VidXP to search indexed videos, answer grounded questions about 1. Resolve the `vidxp` MCP tools, then call `get_workspace`. If the requested video is not indexed, explain that it must be indexed first. -2. Use `search_moments` to locate moments or `query_video` for a synthesized, - grounded answer. Use `command.query` with `search_moments` and - `command.question` with `query_video`. Set `command.media_id` when the user - means one video. -3. Omit `command.evidence_delivery` for the normal path. The completed job - includes an annotated board covering the ranked results. +2. Submit one retrieval job. Use `search_moments` to locate moments; use + `query_video` only when the user asks for a synthesized answer. Use + `command.query` with `search_moments` and `command.question` with + `query_video`. Set `command.media_id` when the user means one video. +3. In that initial job, put exactly this inside `command`: + `"evidence_delivery": {"mode": "keyframes_and_clips", "max_items": 3}`. + This prepares the ranked board, standalone keyframes, and clips without a + second retrieval pass. Never send `command.materialize`. 4. Call `wait_job` for bounded waits. Pass its `observation_token` as `after_observation_token` on the next wait. When terminal, call `get_job_evidence` once. It returns the concise evidence index and visual - content without the full structured job dump. Use `get_job` only when exact - machine fields not present in that index are actually needed. Search and - query may take time; update the user - when the stage changes or about once per minute, never after every wait and - never with an invented ETA. -5. Inspect and show the returned board before making visual claims. Use its tile - evidence IDs for follow-up: - - `materialize_job_evidence` accepts up to ten selected IDs and returns - model-visible standalone keyframes or clip links without rerunning retrieval. - - `create_evidence_board` is only for a custom selection or the - `next_start_rank` continuation; wait on its returned job ID the same way. -6. When standalone artifacts are required in the initial job, put exactly this - inside `command`: `"evidence_delivery": {"mode": - "keyframes_and_clips", "max_items": 3}`. Never send - `command.materialize`. + content without the full structured job dump. Search and query may take + time; update the user when the stage changes or about once per minute, never + after every wait and never with an invented ETA. +5. Surface the returned board, keyframes, and clips immediately. Do not call + `get_job`, repeat the search, materialize more evidence, create another + board, or perform a self-directed verification loop before showing the + initial evidence. +6. Stop after the first evidence delivery. Only when the user explicitly asks + for another selection or format, use tile evidence IDs with + `materialize_job_evidence`, or use `create_evidence_board` for a custom + selection or `next_start_rank` continuation. ## Actor scope @@ -40,15 +38,20 @@ description: Use VidXP to search indexed videos, answer grounded questions about anonymous, video-scoped face clusters—not a named or cross-video identity. - Treat its image as a representative full frame, not an exact face crop or proof of continuous presence. Do not claim exhaustive named appearances. -- Use scene evidence plus board inspection for named-person requests, and label - uncertain matches as candidates. +- For named-person requests, surface the best scene candidates immediately and + label uncertain matches as candidates. Do not delay delivery while trying to + prove identity through additional searches. ## Output -- The final response must visibly embed a returned board or frame, or include a - working downloadable resource link—not timestamps alone. Use the returned - `local_path` or `download_url`; never write an unlinked label such as “View - evidence board.” Use `get_artifact_download` only if neither is returned. +- Lead with evidence, not a search narrative: first embed the returned board or + frame or provide its working resource link, then list the ready clips and + keyframes. Use the returned `local_path` or `download_url`; never write an + unlinked label such as “View evidence board.” Use `get_artifact_download` + only if neither is returned. +- After the evidence, add at most a brief accuracy note. State uncertainty or + visible mismatches without launching another search. Accuracy feedback must + not replace or precede the evidence. - Preserve the source job and evidence IDs. Describe scores as retrieval scores, and distinguish a visible appearance from a dialogue or caption mention. - Stop waiting on success, failure, or cancellation. An empty result means no diff --git a/skills/vidxp-find-video-evidence/agents/openai.yaml b/skills/vidxp-find-video-evidence/agents/openai.yaml index 8324467..4d2a39f 100644 --- a/skills/vidxp-find-video-evidence/agents/openai.yaml +++ b/skills/vidxp-find-video-evidence/agents/openai.yaml @@ -1,7 +1,7 @@ interface: display_name: "Find Video Evidence with VidXP" - short_description: "Search indexed videos and return verifiable evidence" - default_prompt: "Use $vidxp-find-video-evidence to find and verify moments in my indexed videos." + short_description: "Surface video boards, frames, and clips first" + default_prompt: "Use $vidxp-find-video-evidence to surface the best board, keyframes, and clips before brief accuracy feedback." policy: allow_implicit_invocation: true diff --git a/src/vidxp/assets/mcp_app/index.html b/src/vidxp/assets/mcp_app/index.html new file mode 100644 index 0000000..3a46add --- /dev/null +++ b/src/vidxp/assets/mcp_app/index.html @@ -0,0 +1,469 @@ + + + + + + VidXP evidence review + + + +
+
+
+

VidXP

+

Preparing video workspace…

+

Waiting for the tool result.

+
+ +
+
+

+
+ + + diff --git a/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json b/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json new file mode 100644 index 0000000..4232648 --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/.codex-plugin/plugin.json @@ -0,0 +1,28 @@ +{ + "name": "vidxp", + "version": "0.4.0-b.3", + "description": "Ingest, index, search, and inspect video evidence with VidXP.", + "author": { + "name": "Grayhat" + }, + "homepage": "https://github.com/grayhatdevelopers/vidxp", + "repository": "https://github.com/grayhatdevelopers/vidxp", + "license": "MIT", + "keywords": ["video", "search", "evidence", "mcp"], + "skills": "./skills/", + "interface": { + "displayName": "VidXP", + "shortDescription": "Search video and inspect grounded evidence.", + "longDescription": "Use VidXP to ingest and index videos, search dialogue and scenes, answer grounded questions, and review evidence boards, frames, and clips.", + "developerName": "Grayhat", + "category": "Productivity", + "capabilities": ["Read", "Write", "Interactive"], + "websiteURL": "https://github.com/grayhatdevelopers/vidxp", + "defaultPrompt": [ + "Find and verify moments in my indexed videos.", + "Ingest and index a video with VidXP." + ], + "brandColor": "#6D5EF7" + }, + "mcpServers": "./.mcp.json" +} diff --git a/src/vidxp/bundled_plugins/vidxp/.mcp.json b/src/vidxp/bundled_plugins/vidxp/.mcp.json new file mode 100644 index 0000000..bed572e --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "vidxp": { + "command": "vidxp-mcp", + "args": ["--repository", "default"] + } + } +} diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md new file mode 100644 index 0000000..d78fe75 --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/SKILL.md @@ -0,0 +1,59 @@ +--- +name: vidxp-find-video-evidence +description: Use VidXP to search indexed videos and surface inspectable evidence boards, keyframes, and clips before analysis. Trigger for requests such as "find where X appears," "when does Y happen," "what is said," "what happens," or "show me the matching clip," even when the user does not name VidXP. Favor one-pass evidence delivery and only add brief accuracy feedback; do not trigger for ingesting new media or ordinary video editing. +--- + +# Find video evidence with VidXP + +## Workflow + +1. Resolve the `vidxp` MCP tools, then call `get_workspace`. If the requested + video is not indexed, explain that it must be indexed first. +2. Submit one retrieval job. Use `search_moments` to locate moments; use + `query_video` only when the user asks for a synthesized answer. Use + `command.query` with `search_moments` and `command.question` with + `query_video`. Set `command.media_id` when the user means one video. +3. In that initial job, put exactly this inside `command`: + `"evidence_delivery": {"mode": "keyframes_and_clips", "max_items": 3}`. + This prepares the ranked board, standalone keyframes, and clips without a + second retrieval pass. Never send `command.materialize`. +4. Call `wait_job` for bounded waits. Pass its `observation_token` as + `after_observation_token` on the next wait. When terminal, call + `get_job_evidence` once. It returns the concise evidence index and visual + content without the full structured job dump. Search and query may take + time; update the user when the stage changes or about once per minute, never + after every wait and never with an invented ETA. +5. Surface the returned board, keyframes, and clips immediately. Do not call + `get_job`, repeat the search, materialize more evidence, create another + board, or perform a self-directed verification loop before showing the + initial evidence. +6. Stop after the first evidence delivery. Only when the user explicitly asks + for another selection or format, use tile evidence IDs with + `materialize_job_evidence`, or use `create_evidence_board` for a custom + selection or `next_start_rank` continuation. + +## Actor scope + +- Actor data is available through `query_video`, not name search. It represents + anonymous, video-scoped face clusters—not a named or cross-video identity. +- Treat its image as a representative full frame, not an exact face crop or + proof of continuous presence. Do not claim exhaustive named appearances. +- For named-person requests, surface the best scene candidates immediately and + label uncertain matches as candidates. Do not delay delivery while trying to + prove identity through additional searches. + +## Output + +- Lead with evidence, not a search narrative: first embed the returned board or + frame or provide its working resource link, then list the ready clips and + keyframes. Use the returned `local_path` or `download_url`; never write an + unlinked label such as “View evidence board.” Use `get_artifact_download` + only if neither is returned. +- After the evidence, add at most a brief accuracy note. State uncertainty or + visible mismatches without launching another search. Accuracy feedback must + not replace or precede the evidence. +- Preserve the source job and evidence IDs. Describe scores as retrieval scores, + and distinguish a visible appearance from a dialogue or caption mention. +- Stop waiting on success, failure, or cancellation. An empty result means no + matching indexed evidence was found, not that the event is absent from the + original video. diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml new file mode 100644 index 0000000..4d2a39f --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-find-video-evidence/agents/openai.yaml @@ -0,0 +1,13 @@ +interface: + display_name: "Find Video Evidence with VidXP" + short_description: "Surface video boards, frames, and clips first" + default_prompt: "Use $vidxp-find-video-evidence to surface the best board, keyframes, and clips before brief accuracy feedback." + +policy: + allow_implicit_invocation: true + +dependencies: + tools: + - type: "mcp" + value: "vidxp" + description: "VidXP video search and evidence tools" diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md new file mode 100644 index 0000000..198023f --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/SKILL.md @@ -0,0 +1,35 @@ +--- +name: vidxp-ingest-video +description: Use VidXP to upload, import, register, and automatically index video files through its MCP tools. Trigger for requests to add, upload, ingest, import, register, or index one or more videos, including attached videos and accessible local paths, even when the user does not name VidXP. Do not trigger for editing, transcoding, or searching a video that is already indexed. +--- + +# Ingest video with VidXP + +## Workflow + +1. Resolve the `vidxp` MCP tools and call `get_workspace`. Do not import a video + that is already registered or indexed. +2. Choose indexable modalities from the workspace. Use `dialogue` and `scene` + for ordinary content retrieval. Add `actor` only when anonymous recurring-face + clusters are wanted; it does not identify people by name. +3. Call `get_runtime_readiness`. If selected models are missing, submit + `prepare_models`, use `wait_job` with its observation token for subsequent + bounded waits, then fetch `get_job` once when terminal. +4. Use `ingest_local_media` for one to ten paths accessible to VidXP; otherwise + use `create_media_upload` and give the returned link to the user. Keep + `index_after_import` enabled unless registration-only behavior was requested. +5. Poll the returned ingestion or upload ID with `get_media_ingestion` or + `get_media_upload`. Honor its poll interval, reuse the same identifiers, and + do not resubmit unchanged work. +6. Stop at a terminal state and report each file's state, media ID, index job, + and searchable snapshot or generation. If indexing fails after registration, + retry with `start_indexing`; do not upload the file again. +7. If the request also asks about the video, continue directly into the VidXP + evidence workflow once it is searchable. + +## Long operations + +- Tell the user that model preparation and indexing can take several minutes. +- Update when the stage changes or about once per minute; do not narrate every + status check or invent an ETA. +- Treat files independently so one failure does not hide successful siblings. diff --git a/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml new file mode 100644 index 0000000..f502ebe --- /dev/null +++ b/src/vidxp/bundled_plugins/vidxp/skills/vidxp-ingest-video/agents/openai.yaml @@ -0,0 +1,13 @@ +interface: + display_name: "Ingest Video with VidXP" + short_description: "Upload or ingest videos and index them with VidXP" + default_prompt: "Use $vidxp-ingest-video to ingest and index my video with VidXP." + +policy: + allow_implicit_invocation: true + +dependencies: + tools: + - type: "mcp" + value: "vidxp" + description: "VidXP video ingestion and indexing tools" diff --git a/src/vidxp/codex_plugin.py b/src/vidxp/codex_plugin.py new file mode 100644 index 0000000..5ad3410 --- /dev/null +++ b/src/vidxp/codex_plugin.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import tomllib +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from vidxp import __version__ +from vidxp.mcp_cli import stdio_client_config + + +PLUGIN_NAME = "vidxp" +MARKETPLACE_NAME = "vidxp-local" +MANAGED_MARKER = ".vidxp-managed-marketplace" +MARKETPLACE_MANIFEST = Path(".agents") / "plugins" / "marketplace.json" + + +class CodexPluginInstallError(RuntimeError): + """Raised when the bundled Codex plugin cannot be installed safely.""" + + +@dataclass(frozen=True) +class CodexPluginExport: + marketplace_root: str + marketplace_path: str + marketplace_name: str + plugin_name: str + plugin_version: str + + def to_dict(self) -> dict[str, str]: + return asdict(self) + + +@dataclass(frozen=True) +class CodexPluginInstall: + plugin_name: str + plugin_id: str | None + plugin_version: str + marketplace_name: str + marketplace_path: str + installed_path: str | None + detail: str + + def to_dict(self) -> dict[str, str | None]: + return asdict(self) + + +def bundled_codex_plugin() -> Path: + return Path(__file__).resolve().parent / "bundled_plugins" / PLUGIN_NAME + + +def _json_bytes(payload: Any) -> bytes: + return ( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def _write_json(path: Path, payload: Any) -> None: + path.write_bytes(_json_bytes(payload)) + + +def _bundle_digest(source: Path, mcp_config: dict[str, Any]) -> str: + digest = hashlib.sha256(_json_bytes(mcp_config)) + for path in sorted(item for item in source.rglob("*") if item.is_file()): + digest.update(path.relative_to(source).as_posix().encode("utf-8")) + digest.update(path.read_bytes()) + return digest.hexdigest()[:12] + + +def _validated_marketplace_root(marketplace_root: Path) -> Path: + root = marketplace_root.expanduser().resolve() + if root == Path(root.anchor): + raise CodexPluginInstallError( + "The Codex marketplace cannot be written at a filesystem root." + ) + marker = root / MANAGED_MARKER + if root.exists() and any(root.iterdir()) and not marker.is_file(): + raise CodexPluginInstallError( + f"Refusing to replace the unmanaged marketplace directory at {root}." + ) + return root + + +def export_codex_plugin( + marketplace_root: Path, + *, + registry: str | None = None, + repository: str = "default", + index_directory: str | None = None, + data_directory: Path | None = None, + device: str | None = None, +) -> CodexPluginExport: + """Export a target-specific copy of VidXP's bundled Codex plugin.""" + + source = bundled_codex_plugin() + if not (source / ".codex-plugin" / "plugin.json").is_file(): + raise CodexPluginInstallError( + "This VidXP installation does not contain the bundled Codex plugin." + ) + root = _validated_marketplace_root(marketplace_root) + plugins_root = root / "plugins" + plugins_root.mkdir(parents=True, exist_ok=True) + plugin_root = plugins_root / PLUGIN_NAME + marker = root / MANAGED_MARKER + + mcp_config = stdio_client_config( + registry=registry, + repository=repository, + index_directory=index_directory, + data_directory=data_directory, + device=device, + ) + plugin_version = ( + f"{__version__.split('+', 1)[0]}+codex." + f"{_bundle_digest(source, mcp_config)}" + ) + + staging_parent = Path(tempfile.mkdtemp(prefix=".vidxp-plugin-", dir=plugins_root)) + staging_plugin = staging_parent / PLUGIN_NAME + backup = plugins_root / ".vidxp-plugin-backup" + try: + shutil.copytree(source, staging_plugin) + manifest_path = staging_plugin / ".codex-plugin" / "plugin.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["version"] = plugin_version + _write_json(manifest_path, manifest) + _write_json(staging_plugin / ".mcp.json", mcp_config) + + if backup.exists(): + shutil.rmtree(backup) + if plugin_root.exists(): + if not marker.is_file(): + raise CodexPluginInstallError( + f"Refusing to replace the unmanaged plugin at {plugin_root}." + ) + os.replace(plugin_root, backup) + try: + os.replace(staging_plugin, plugin_root) + except Exception: + if backup.exists() and not plugin_root.exists(): + os.replace(backup, plugin_root) + raise + if backup.exists(): + shutil.rmtree(backup) + finally: + if staging_parent.exists(): + shutil.rmtree(staging_parent) + + marketplace = { + "name": MARKETPLACE_NAME, + "interface": {"displayName": "VidXP Local"}, + "plugins": [ + { + "name": PLUGIN_NAME, + "source": { + "source": "local", + "path": f"./plugins/{PLUGIN_NAME}", + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL", + }, + "category": "Productivity", + } + ], + } + marketplace_path = root / MARKETPLACE_MANIFEST + marketplace_path.parent.mkdir(parents=True, exist_ok=True) + _write_json(marketplace_path, marketplace) + legacy_marketplace_path = root / "marketplace.json" + if marker.is_file() and legacy_marketplace_path.is_file(): + legacy_marketplace_path.unlink() + marker.write_text("Managed by VidXP Desktop.\n", encoding="utf-8") + return CodexPluginExport( + marketplace_root=str(root), + marketplace_path=str(marketplace_path), + marketplace_name=MARKETPLACE_NAME, + plugin_name=PLUGIN_NAME, + plugin_version=plugin_version, + ) + + +CommandRunner = Callable[..., subprocess.CompletedProcess[str]] + + +def _run_codex_json( + command: str, + arguments: Sequence[str], + *, + runner: CommandRunner, +) -> dict[str, Any]: + try: + completed = runner( + [command, *arguments], + capture_output=True, + text=True, + encoding="utf-8", + timeout=60, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise CodexPluginInstallError(f"Codex could not be started: {exc}") from exc + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout).strip() + raise CodexPluginInstallError( + detail or f"Codex exited with status {completed.returncode}." + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise CodexPluginInstallError( + "Codex did not return valid plugin installation details." + ) from exc + if not isinstance(payload, dict): + raise CodexPluginInstallError( + "Codex returned an unexpected plugin installation response." + ) + return payload + + +def _configured_codex_command( + environment: Mapping[str, str], +) -> str | None: + codex_home = Path( + environment.get("CODEX_HOME", str(Path.home() / ".codex")) + ).expanduser() + try: + config = tomllib.loads( + (codex_home / "config.toml").read_text(encoding="utf-8") + ) + except (OSError, tomllib.TOMLDecodeError): + return None + servers = config.get("mcp_servers") + node_repl = servers.get("node_repl") if isinstance(servers, dict) else None + node_repl_environment = ( + node_repl.get("env") if isinstance(node_repl, dict) else None + ) + policy = config.get("shell_environment_policy") + policy_environment = policy.get("set") if isinstance(policy, dict) else None + for configured in (node_repl_environment, policy_environment): + command = ( + configured.get("CODEX_CLI_PATH") + if isinstance(configured, dict) + else None + ) + if isinstance(command, str) and command.strip(): + return command + return None + + +def resolve_codex_command( + *, + environment: Mapping[str, str] | None = None, + which: Callable[[str], str | None] = shutil.which, +) -> str | None: + """Locate the current CLI bundled with Codex or available on PATH.""" + current_environment = os.environ if environment is None else environment + candidates: list[str] = [] + environment_command = current_environment.get("CODEX_CLI_PATH") + if environment_command: + candidates.append(environment_command) + configured_command = _configured_codex_command(current_environment) + if configured_command: + candidates.append(configured_command) + + local_app_data = current_environment.get("LOCALAPPDATA") + if local_app_data: + bin_directory = Path(local_app_data) / "OpenAI" / "Codex" / "bin" + if bin_directory.is_dir(): + versioned = sorted( + bin_directory.glob("*/codex.exe"), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + candidates.extend(str(path) for path in versioned) + candidates.append(str(bin_directory / "codex.exe")) + + path_command = which("codex") + if path_command: + candidates.append(path_command) + + seen: set[str] = set() + for candidate in candidates: + normalized = os.path.normcase(os.path.abspath(os.path.expanduser(candidate))) + if normalized in seen: + continue + seen.add(normalized) + if Path(candidate).expanduser().is_file(): + return str(Path(candidate).expanduser()) + return None + + +def install_codex_plugin( + marketplace_root: Path, + *, + registry: str | None = None, + repository: str = "default", + index_directory: str | None = None, + data_directory: Path | None = None, + device: str | None = None, + codex_command: str | None = None, + runner: CommandRunner = subprocess.run, +) -> CodexPluginInstall: + """Export, register, and install VidXP's local Codex plugin.""" + + exported = export_codex_plugin( + marketplace_root, + registry=registry, + repository=repository, + index_directory=index_directory, + data_directory=data_directory, + device=device, + ) + command = codex_command or resolve_codex_command() + if command is None: + raise CodexPluginInstallError( + "The Codex CLI was not found. Install or update the ChatGPT desktop " + "app, make the codex command available, and try again." + ) + + marketplace_result = _run_codex_json( + command, + [ + "plugin", + "marketplace", + "add", + exported.marketplace_root, + "--json", + ], + runner=runner, + ) + marketplace_name = str( + marketplace_result.get("marketplaceName") or exported.marketplace_name + ) + plugin_result = _run_codex_json( + command, + [ + "plugin", + "add", + f"{exported.plugin_name}@{marketplace_name}", + "--json", + ], + runner=runner, + ) + return CodexPluginInstall( + plugin_name=str(plugin_result.get("name") or exported.plugin_name), + plugin_id=( + None + if plugin_result.get("pluginId") is None + else str(plugin_result["pluginId"]) + ), + plugin_version=str( + plugin_result.get("version") or exported.plugin_version + ), + marketplace_name=str( + plugin_result.get("marketplaceName") or marketplace_name + ), + marketplace_path=exported.marketplace_path, + installed_path=( + None + if plugin_result.get("installedPath") is None + else str(plugin_result["installedPath"]) + ), + detail=( + "VidXP is installed in Codex with its MCP server and skills. " + "Start a new Codex chat to use the updated plugin." + ), + ) diff --git a/src/vidxp/codex_plugin_cli.py b/src/vidxp/codex_plugin_cli.py new file mode 100644 index 0000000..c93ebf0 --- /dev/null +++ b/src/vidxp/codex_plugin_cli.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Sequence + +from vidxp.codex_plugin import CodexPluginInstallError, install_codex_plugin + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Install VidXP's bundled MCP server and skills in Codex." + ) + parser.add_argument( + "--marketplace-root", + type=Path, + required=True, + help="Dedicated local marketplace directory managed by VidXP Desktop.", + ) + parser.add_argument("--registry") + parser.add_argument("--repository", default="default") + parser.add_argument("--index-directory") + parser.add_argument("--data-dir", type=Path) + parser.add_argument("--device") + return parser + + +def main(arguments: Sequence[str] | None = None) -> None: + parser = _parser() + options = parser.parse_args(arguments) + try: + result = install_codex_plugin( + options.marketplace_root, + registry=options.registry, + repository=options.repository, + index_directory=options.index_directory, + data_directory=options.data_dir, + device=options.device, + ) + except (CodexPluginInstallError, OSError, ValueError) as exc: + parser.exit(1, f"VidXP could not set up Codex: {exc}\n") + print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/vidxp/mcp.py b/src/vidxp/mcp.py index 991afb7..c2f25ed 100644 --- a/src/vidxp/mcp.py +++ b/src/vidxp/mcp.py @@ -103,6 +103,11 @@ scoped_job_id, scoped_request_key, ) +from vidxp.mcp_app import ( + MCP_APP_MIME_TYPE, + MCP_APP_RESOURCE_URI, + load_mcp_app_html, +) from vidxp.core.identifiers import ArtifactId from vidxp.evidence_delivery import ( EvidenceDeliveryService, @@ -149,6 +154,15 @@ ) +def _mcp_app_tool_meta(invoking: str, invoked: str) -> dict[str, object]: + return { + "ui": {"resourceUri": MCP_APP_RESOURCE_URI}, + "openai/outputTemplate": MCP_APP_RESOURCE_URI, + "openai/toolInvocation/invoking": invoking, + "openai/toolInvocation/invoked": invoked, + } + + class PrincipalBridge: """Carry the principal validated by the outer ASGI boundary into tools.""" @@ -547,6 +561,27 @@ async def lifecycle(_server): lifespan=lifecycle, ) + @server.resource( + MCP_APP_RESOURCE_URI, + name="vidxp_mcp_app", + title="VidXP video workspace", + description=( + "Interactive upload progress and evidence review for MCP Apps hosts." + ), + mime_type=MCP_APP_MIME_TYPE, + meta={ + "ui": { + "prefersBorder": True, + "csp": { + "connectDomains": [], + "resourceDomains": [], + }, + } + }, + ) + async def read_mcp_app() -> str: + return load_mcp_app_html() + async def artifact_bytes( artifact_id: ArtifactId, *, @@ -1061,9 +1096,125 @@ def evidence_index( ) return "\n".join(lines) - async def evidence_content( + def evidence_app_payload( + *, job: Job, - ) -> list[ImageContent | ResourceLink | TextContent]: + source_job_id: JobId, + delivery: EvidenceDeliveryResult, + query_result: QueryAnswer | None, + ) -> dict[str, object]: + board = delivery.board + pages: list[dict[str, object]] = [] + tiles: list[dict[str, object]] = [] + if board is not None: + for page in board.pages: + artifact = page.artifact + delivery_info = artifact.delivery + pages.append( + { + "page_number": page.page_number, + "media_id": page.media_id, + "width": page.width, + "height": page.height, + "tile_ids": list(page.tile_ids), + "resource_uri": artifact.resource_uri, + "download_url": ( + delivery_info.download_url + if delivery_info is not None + else None + ), + } + ) + for tile in board.tiles: + tiles.append( + { + "evidence_id": tile.evidence_id, + "rank": tile.rank, + "page_number": tile.page_number, + "position": tile.position, + "media_id": tile.media_id, + "modalities": list(tile.modalities), + "start": tile.start, + "end": tile.end, + "display_text": concise_text(tile.display_text), + "state": tile.state.value, + } + ) + requested_count = board.requested_count + rendered_count = board.rendered_count + failed_count = board.failed_count + next_start_rank = board.next_start_rank + else: + for item in delivery.items: + resolved = item.range + tiles.append( + { + "evidence_id": item.evidence_id, + "rank": item.rank, + "page_number": None, + "position": item.rank, + "media_id": item.media_id, + "modalities": list(item.modalities), + "start": ( + resolved.source_start_seconds + if resolved is not None + else 0.0 + ), + "end": ( + resolved.source_end_seconds + if resolved is not None + else 0.0 + ), + "display_text": None, + "state": item.state.value, + } + ) + requested_count = len(delivery.items) + rendered_count = sum( + item.state.value == "ready" for item in delivery.items + ) + failed_count = requested_count - rendered_count + next_start_rank = None + + answer: dict[str, object] | None = None + if query_result is not None: + answer = { + "mode": query_result.mode.value, + "claims": [ + { + "text": concise_text(claim.text, limit=512) or "", + "evidence_ids": list(claim.evidence_ids), + } + for claim in query_result.claims + ], + "fallback_reason": concise_text( + query_result.fallback_reason, + limit=512, + ), + } + + return { + "view": "evidence", + "job_id": job.job_id, + "source_job_id": source_job_id, + "job_kind": job.kind.value, + "answer": answer, + "board": { + "requested_count": requested_count, + "rendered_count": rendered_count, + "failed_count": failed_count, + "next_start_rank": next_start_rank, + "pages": pages, + "tiles": tiles, + }, + } + + async def evidence_presentation( + job: Job, + ) -> tuple[ + dict[str, object], + list[ImageContent | ResourceLink | TextContent], + ]: query_result = None if job.kind in {JobKind.search, JobKind.query}: result = job.result.result @@ -1082,18 +1233,27 @@ async def evidence_content( delivery=projected_delivery, query_result=query_result, ) + source_job_id = job.job_id else: board = job.result.result projected_board, blocks = await project_evidence_board(board) + projected_delivery = EvidenceDeliveryResult( + policy=EvidenceDeliveryPolicy(mode=EvidenceDeliveryMode.none), + items=(), + board=projected_board, + ) index = evidence_index( source_job_id=board.source_job_id, - delivery=EvidenceDeliveryResult( - policy=EvidenceDeliveryPolicy(mode=EvidenceDeliveryMode.none), - items=(), - board=projected_board, - ), + delivery=projected_delivery, ) - return [TextContent(type="text", text=index), *blocks] + source_job_id = board.source_job_id + payload = evidence_app_payload( + job=job, + source_job_id=source_job_id, + delivery=projected_delivery, + query_result=query_result, + ) + return payload, [TextContent(type="text", text=index), *blocks] def completed_evidence_result( source_job_id: JobId, @@ -1227,6 +1387,10 @@ async def get_media(media_id: MediaId) -> MediaAsset: "Automatic indexing defaults on. Poll only get_media_upload." ), annotations=_SUBMIT, + meta=_mcp_app_tool_meta( + "Creating a VidXP upload session…", + "VidXP upload session ready.", + ), structured_output=True, ) async def create_media_upload( @@ -1826,10 +1990,14 @@ async def get_job(job_id: JobId) -> Job: description=( "Present a completed search, query, or evidence-board job as a " "concise evidence index plus model-visible board images and resource " - "links. This intentionally omits structuredContent; use get_job only " - "when the full machine record is actually needed." + "links. The compact structured result drives the optional VidXP " + "evidence-review UI without exposing the full machine record." ), annotations=_READ_ONLY, + meta=_mcp_app_tool_meta( + "Opening VidXP evidence…", + "VidXP evidence ready.", + ), ) async def get_job_evidence(job_id: JobId) -> CallToolResult: def completed_evidence_job(_actor: Principal) -> Job: @@ -1855,10 +2023,13 @@ def completed_evidence_job(_actor: Principal) -> Job: operation=completed_evidence_job, ) try: - blocks = await evidence_content(job) + structured_content, blocks = await evidence_presentation(job) except ApplicationError as exc: raise _application_error(exc) from exc - return CallToolResult(content=blocks) + return CallToolResult( + content=blocks, + structured_content=structured_content, + ) @server.tool( title="Get compact job status", diff --git a/src/vidxp/mcp_app.py b/src/vidxp/mcp_app.py new file mode 100644 index 0000000..8794824 --- /dev/null +++ b/src/vidxp/mcp_app.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from functools import lru_cache +from importlib.resources import files + + +MCP_APP_RESOURCE_URI = "ui://vidxp/evidence-review-v1.html" +MCP_APP_MIME_TYPE = "text/html;profile=mcp-app" + + +@lru_cache(maxsize=1) +def load_mcp_app_html() -> str: + """Load the self-contained MCP App resource shipped with VidXP.""" + + return ( + files("vidxp") + .joinpath("assets", "mcp_app", "index.html") + .read_text(encoding="utf-8") + ) diff --git a/tests/test_codex_plugin.py b/tests/test_codex_plugin.py new file mode 100644 index 0000000..9aa8a73 --- /dev/null +++ b/tests/test_codex_plugin.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from vidxp.codex_plugin import ( + CodexPluginInstallError, + export_codex_plugin, + install_codex_plugin, + resolve_codex_command, +) + + +def test_export_codex_plugin_materializes_skills_and_target_mcp_config() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) / "codex-marketplace" + index_directory = Path("C:/VidXP/repositories/default") + data_directory = Path("C:/VidXP") + exported = export_codex_plugin( + root, + repository="default", + index_directory=str(index_directory), + data_directory=data_directory, + ) + + plugin_root = root / "plugins" / "vidxp" + manifest = json.loads( + (plugin_root / ".codex-plugin" / "plugin.json").read_text( + encoding="utf-8" + ) + ) + mcp = json.loads((plugin_root / ".mcp.json").read_text(encoding="utf-8")) + marketplace_path = root / ".agents" / "plugins" / "marketplace.json" + marketplace = json.loads(marketplace_path.read_text(encoding="utf-8")) + + assert manifest["name"] == "vidxp" + assert manifest["version"] == exported.plugin_version + assert "+codex." in exported.plugin_version + assert (plugin_root / "skills" / "vidxp-ingest-video" / "SKILL.md").is_file() + assert ( + plugin_root / "skills" / "vidxp-find-video-evidence" / "SKILL.md" + ).is_file() + assert mcp["mcpServers"]["vidxp"]["args"] == [ + "--repository", + "default", + "--index-directory", + str(index_directory), + "--data-dir", + str(data_directory), + ] + assert Path(mcp["mcpServers"]["vidxp"]["command"]).name.lower() in { + "vidxp-mcp", + "vidxp-mcp.exe", + } + assert marketplace["name"] == "vidxp-local" + assert marketplace["plugins"][0]["source"]["path"] == "./plugins/vidxp" + assert marketplace["plugins"][0]["policy"] == { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL", + } + assert exported.marketplace_path == str(marketplace_path) + assert not (root / "marketplace.json").exists() + + +def test_export_migrates_the_legacy_managed_marketplace_layout() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) / "marketplace" + root.mkdir() + (root / ".vidxp-managed-marketplace").write_text( + "Managed by VidXP Desktop.\n", + encoding="utf-8", + ) + legacy_path = root / "marketplace.json" + legacy_path.write_text("{}\n", encoding="utf-8") + + exported = export_codex_plugin(root) + + assert Path(exported.marketplace_path).is_file() + assert not legacy_path.exists() + + +def test_install_codex_plugin_registers_marketplace_then_installs_bundle() -> None: + calls: list[list[str]] = [] + + def runner(command: list[str], **_: object) -> subprocess.CompletedProcess[str]: + calls.append(command) + if command[1:4] == ["plugin", "marketplace", "add"]: + payload = {"marketplaceName": "vidxp-local", "alreadyAdded": False} + else: + payload = { + "pluginId": "vidxp@vidxp-local", + "name": "vidxp", + "marketplaceName": "vidxp-local", + "version": "0.4.0+codex.example", + "installedPath": "/codex/cache/vidxp", + } + return subprocess.CompletedProcess(command, 0, json.dumps(payload), "") + + with TemporaryDirectory() as directory: + result = install_codex_plugin( + Path(directory) / "marketplace", + codex_command="codex-test", + runner=runner, + ) + + assert calls[0][0:4] == ["codex-test", "plugin", "marketplace", "add"] + assert calls[0][-1] == "--json" + assert calls[1] == [ + "codex-test", + "plugin", + "add", + "vidxp@vidxp-local", + "--json", + ] + assert result.plugin_id == "vidxp@vidxp-local" + assert result.installed_path == "/codex/cache/vidxp" + assert "MCP server and skills" in result.detail + + +def test_export_refuses_to_replace_an_unmanaged_marketplace() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) / "marketplace" + root.mkdir() + (root / "keep.txt").write_text("user data", encoding="utf-8") + + with pytest.raises(CodexPluginInstallError, match="unmanaged marketplace"): + export_codex_plugin(root) + + assert (root / "keep.txt").read_text(encoding="utf-8") == "user data" + + +def test_resolve_codex_command_prefers_desktop_configured_cli() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + codex_home = root / ".codex" + configured = root / "current" / "codex.exe" + stale = root / "OpenAI" / "Codex" / "bin" / "codex.exe" + configured.parent.mkdir(parents=True) + stale.parent.mkdir(parents=True) + configured.touch() + stale.touch() + codex_home.mkdir() + escaped_command = str(configured).replace("\\", "\\\\") + (codex_home / "config.toml").write_text( + "[mcp_servers.node_repl.env]\n" + f'CODEX_CLI_PATH = "{escaped_command}"\n', + encoding="utf-8", + ) + + resolved = resolve_codex_command( + environment={ + "CODEX_HOME": str(codex_home), + "LOCALAPPDATA": str(root), + }, + which=lambda _: None, + ) + + assert resolved == str(configured) + + +def test_resolve_codex_command_uses_desktop_environment_path() -> None: + with TemporaryDirectory() as directory: + command = Path(directory) / "codex.exe" + command.touch() + + resolved = resolve_codex_command( + environment={"CODEX_CLI_PATH": str(command)}, + which=lambda _: None, + ) + + assert resolved == str(command) + + +def test_resolve_codex_command_falls_back_to_local_app_install() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + command = root / "OpenAI" / "Codex" / "bin" / "codex.exe" + command.parent.mkdir(parents=True) + command.touch() + + resolved = resolve_codex_command( + environment={ + "CODEX_HOME": str(root / "missing-codex-home"), + "LOCALAPPDATA": str(root), + }, + which=lambda _: None, + ) + + assert resolved == str(command) + + +def test_resolve_codex_command_prefers_newest_versioned_local_cli() -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + bin_directory = root / "OpenAI" / "Codex" / "bin" + stable = bin_directory / "codex.exe" + older = bin_directory / "old" / "codex.exe" + current = bin_directory / "current" / "codex.exe" + for command in (stable, older, current): + command.parent.mkdir(parents=True, exist_ok=True) + command.touch() + os.utime(older, (1, 1)) + os.utime(current, (2, 2)) + + resolved = resolve_codex_command( + environment={ + "CODEX_HOME": str(root / "missing-codex-home"), + "LOCALAPPDATA": str(root), + }, + which=lambda _: None, + ) + + assert resolved == str(current) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index c6f1834..e7f1782 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -72,6 +72,7 @@ SearchMomentsPlanStep, WorkspaceOverview, ) +from vidxp.mcp_app import MCP_APP_MIME_TYPE, MCP_APP_RESOURCE_URI from vidxp.authentication import ( AuthenticatedBearer, OIDCBearerAuthenticator, @@ -439,12 +440,38 @@ async def test_curated_tools_publish_their_intended_output_contracts(self): async with Client(server) as client: discovered = await client.list_tools() result = await client.call_tool("list_capabilities", {}) + app_resource = await client.read_resource(MCP_APP_RESOURCE_URI) self.assertEqual( [tool.name for tool in discovered.tools], MCP_TOOL_NAMES, ) tools = {tool.name: tool for tool in discovered.tools} + for name in ("create_media_upload", "get_job_evidence"): + self.assertEqual( + tools[name].meta["ui"]["resourceUri"], + MCP_APP_RESOURCE_URI, + ) + self.assertEqual( + tools[name].meta["openai/outputTemplate"], + MCP_APP_RESOURCE_URI, + ) + app_contents = app_resource.contents[0] + self.assertEqual(app_contents.mime_type, MCP_APP_MIME_TYPE) + self.assertEqual( + app_contents.meta["ui"]["csp"], + {"connectDomains": [], "resourceDomains": []}, + ) + self.assertIn("ui/notifications/tool-result", app_contents.text) + self.assertIn('request("ui/initialize"', app_contents.text) + self.assertIn('notify("ui/notifications/initialized"', app_contents.text) + self.assertIn('request("tools/call"', app_contents.text) + self.assertIn('request("ui/open-link"', app_contents.text) + self.assertIn('request("ui/request-display-mode"', app_contents.text) + self.assertIn('request("ui/update-model-context"', app_contents.text) + self.assertIn("window.openai?.requestDisplayMode", app_contents.text) + self.assertIn("materialize_job_evidence", app_contents.text) + self.assertNotIn("