diff --git a/.gitmodules b/.gitmodules index 6bd7ad0c..38cd0244 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "packages/python-server-sdk"] path = packages/python-server-sdk url = https://github.com/fishjam-cloud/python-server-sdk.git +[submodule "api/smelter-cloud"] + path = api/composition + url = git@github.com:fishjam-cloud/foundry.git diff --git a/api/composition b/api/composition new file mode 160000 index 00000000..7d59e176 --- /dev/null +++ b/api/composition @@ -0,0 +1 @@ +Subproject commit 7d59e1769e19dba177627ac1faa865c6bd2a16df diff --git a/docs/api/reference.md b/docs/api/reference.md index e10073e1..4ee025f4 100644 --- a/docs/api/reference.md +++ b/docs/api/reference.md @@ -6,7 +6,7 @@ type: reference Describes APIs for direct interaction with Fishjam. -Fishjam publishes documentation for the Sandbox API and Fishjam Server APIs. +Fishjam publishes documentation for the Sandbox API, Fishjam Server APIs, and the Composition API. ## Sandbox API @@ -51,3 +51,56 @@ the first message that must be sent is an `AuthRequest`, with a valid Management Token. Next, you can should subscribe to notifications by sending `SubscribeRequest` event with `SERVER_NOTIFICATION` event type. + +## Compositions + +[Compositions](../explanation/compositions) are managed through the Composition API: a REST API plus a WebSocket stream for engine events. All requests go to `https://rtc.fishjam.io`. + +### REST API + +[Composition REST API Reference](/api/compositions) + +The [OpenAPI document](https://github.com/fishjam-cloud/documentation/blob/main/static/api/composition-openapi.json) is generated from the service's source code and republished together with documentation updates. + +### WebSocket event stream + +Some engine events (for example, an output finishing) are delivered over a WebSocket rather than HTTP. Connect to: + +``` +GET wss://rtc.fishjam.io/api/composition/{composition_id}/ws +``` + +Because browsers cannot set an `Authorization` header on a WebSocket, authentication rides on the `Sec-WebSocket-Protocol` header, which must carry **two** subprotocols (order does not matter): + +- `json.fishjam.io`: selects the JSON wire format. +- `bearer.auth.fishjam.io.`: your token, appended to the literal prefix. + +```js +const ws = new WebSocket( + `wss://rtc.fishjam.io/api/composition/${compositionId}/ws`, + ["json.fishjam.io", `bearer.auth.fishjam.io.${token}`], +); +``` + +Messages are JSON text frames, each with a `type` field identifying the event. + +### Composition authentication + +| What you're calling | How it authenticates | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Control-plane calls (composition, inputs, outputs, renderers, events, room) | `Authorization: Bearer `: your Fishjam **Management Token**, the same token used across Fishjam. | +| Publishing to an input (`/whip/{input_id}`) | The **input's** own bearer token, returned when you register a `whip_server` input (or the one you supplied). This is distinct from your account token. | +| Playing back an output (`/whep/{output_id}`) | Optional. Public playback is allowed; add a Bearer token only if you want to require one. | +| The WebSocket event stream | The subprotocol scheme described above. | + +Get your Management Token from the [**Fishjam developer panel**](https://fishjam.io/app). + +### Errors + +Every non-2xx response of the Composition API is a JSON object: + +```json +{ "message": "Composition not found", "http_status_code": 404 } +``` + +Common statuses are `400` (bad request), `401` (unauthorized), `404` (not found), `500` (server error), and `503` (no capacity, returned by composition creation). diff --git a/docs/explanation/compositions.mdx b/docs/explanation/compositions.mdx new file mode 100644 index 00000000..664ff649 --- /dev/null +++ b/docs/explanation/compositions.mdx @@ -0,0 +1,123 @@ +--- +type: explanation +title: Compositions +sidebar_position: 4.5 +description: Understand compositions, the Fishjam feature that mixes multiple live media streams into a single composed output in real time. +--- + +# Compositions + +_Understanding real-time stream composition in Fishjam_ + +A **composition** mixes multiple live media streams into a single output stream in real time. You send inputs (WebRTC, RTMP, HLS, or MP4), describe how they should be laid out, and it produces one composed output that you can publish anywhere, all without running any rendering infrastructure yourself. + +Compositions are built on [Smelter](https://smelter.dev), the open-source real-time video compositing engine by [Software Mansion](https://swmansion.com), and are a native part of Fishjam: they authenticate with the same Fishjam credentials and can compose the peers of a [Fishjam room](./rooms) directly. + +## What you can build + +- **Multi-party layouts**: arrange the cameras of a conference or livestream into grids, side-by-sides, or picture-in-picture. +- **Branded streams**: overlay logos, captions, lower-thirds, and backgrounds on top of live video. +- **React templates**: describe your layout as a React component that reacts to who is speaking, who joined, and custom application events. +- **Cross-protocol bridging**: take WebRTC inputs and republish the composed result to an RTMP destination, or vice versa. + +## Core concepts + +A **composition** is a single running compositing session. Into it you register: + +- **Inputs**: the live media sources being composed (WebRTC via WHIP/WHEP, RTMP, HLS, or MP4). +- **Outputs**: where the composed result is sent (WebRTC via WHIP, or RTMP). Each output carries a **scene** that describes the layout. +- **Renderers**: shared assets such as images and fonts you can place in a scene. + +An output's scene can either be described directly in the API or rendered by a **template**: a React component, written with the layout components and the [`@fishjam-cloud/composition`](./../how-to/compositions/write-and-deploy-a-template) hooks, that updates the layout live as the room changes. + +## Scenes + +Every video output carries a **scene**: a tree of components that describes how inputs, text, and images are arranged into the composed frame. Audio outputs carry an **audio scene** that describes which inputs are mixed together. + +### The video scene tree + +A video scene has a single `root` component. Each component has a `type` that determines how it lays out its children. + +```json +{ + "root": { + "type": "tiles", + "children": [ + { "type": "input_stream", "input_id": "camera_1" }, + { "type": "input_stream", "input_id": "camera_2" } + ] + } +} +``` + +The available component types are: + +| `type` | Purpose | +| -------------- | -------------------------------------------------------------------------------- | +| `input_stream` | Renders one registered input. Identified by `input_id`. | +| `view` | A container you position and style; the basic building block for custom layouts. | +| `tiles` | Automatically arranges its children into a grid. | +| `rescaler` | Fits a single child into a target area, preserving aspect ratio. | +| `text` | Renders a text string. | +| `image` | Renders a registered image. Identified by `image_id`. | + +Components nest freely: a `tiles` of `rescaler`s wrapping `input_stream`s, a `view` with a `text` caption over an `input_stream`, and so on. The styling and full property set of each component come from Smelter itself; see the [Smelter component reference](https://smelter.dev) for every available property. + +### The audio scene + +An audio scene lists the inputs to mix and, optionally, their relative volume: + +```json +{ + "inputs": [ + { "input_id": "camera_1" }, + { "input_id": "camera_2", "volume": 0.5 } + ] +} +``` + +`volume` defaults to `1.0`. Only the inputs you list are audible in the output. + +### Setting a scene + +You provide the initial scene when you register an output, under `video.initial` (a video scene) and `audio.initial` (an audio scene). See [Choose inputs and outputs](./../how-to/compositions/inputs-and-outputs) for the full output shape. + +### Updating a scene live + +To change the layout while a composition is running, call the output's update endpoint with the new `video` and/or `audio` scene: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/update" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "video": { + "root": { + "type": "rescaler", + "child": { "type": "input_stream", "input_id": "camera_1" } + } + } + }' +``` + +Only the sides you include are changed; omitting `audio` leaves the audio scene untouched, and vice versa. + +An update can be scheduled instead of applied immediately by adding `schedule_time_ms`, the composition timeline offset in milliseconds at which the change should take effect: + +```json +{ + "video": { "root": { "type": "tiles", "children": [] } }, + "schedule_time_ms": 5000 +} +``` + +### What if I don't want to update the scene on every change? + +Pushing a scene update every time a participant joins, leaves, or mutes gets tedious fast. Instead, you can describe the layout once as a **template**: a React component that receives the live room state and re-renders automatically as it changes, cutting manual scene management down to a minimum. See [Write and deploy a template](./../how-to/compositions/write-and-deploy-a-template) to build one. + +## Where to go next + +- [Compositions tutorial](./../tutorials/compositions): create your first composition end to end. +- [Write and deploy a template](./../how-to/compositions/write-and-deploy-a-template): build a React layout with the composition SDK. +- [Compose a Fishjam room](./../how-to/compositions/compose-a-fishjam-room): turn a room's peers into one composed stream. +- [Composition API](./../api/reference#compositions): the full REST surface. diff --git a/docs/how-to/compositions/_category_.json b/docs/how-to/compositions/_category_.json new file mode 100644 index 00000000..f0331dc1 --- /dev/null +++ b/docs/how-to/compositions/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Compositions", + "position": 5, + "link": { + "type": "generated-index", + "description": "Compose rooms and live streams into new outputs with templates, events, and multiple transports." + } +} diff --git a/docs/how-to/compositions/compose-a-fishjam-room.mdx b/docs/how-to/compositions/compose-a-fishjam-room.mdx new file mode 100644 index 00000000..5445f037 --- /dev/null +++ b/docs/how-to/compositions/compose-a-fishjam-room.mdx @@ -0,0 +1,137 @@ +--- +type: how-to +sidebar_position: 2 +description: Forward a Fishjam room into a composition and render its peers with a live React template. +--- + +# Compose a Fishjam room + +A composition can include the peers of a [Fishjam room](./../../explanation/rooms). You forward the room's tracks to the composition with a single Fishjam API call: Fishjam then pushes each participant's media into the composition as inputs, and hooks from `@fishjam-cloud/composition` let a [template](./write-and-deploy-a-template) render one tile per participant and push the result to a [livestream](./../../explanation/livestreams). + +``` +Fishjam room (peers) ──forwarded──▶ composition (template) ──WHIP──▶ Fishjam livestream ──WHEP──▶ [viewers] +``` + +## Prerequisites + +- A Fishjam room created with `"videoCodec": "h264"` (compositions consume h264 video), with peers publishing into it. +- A [livestream](./../../explanation/livestreams) (or any other WHIP/RTMP destination) to publish the composed stream to. +- A template project scaffolded with the composition CLI (see [Write and deploy a template](./write-and-deploy-a-template)). + +## Step 1: Write a room-aware template + +Hooks from `@fishjam-cloud/composition` give the template live room state; it re-renders automatically as participants join, leave, mute, or speak. + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { InputStream, Rescaler, Text, Tiles, View } from "@swmansion/smelter"; +import { usePeers, useSpeakingState } from "@fishjam-cloud/composition"; +import type { PeerWithStreams } from "@fishjam-cloud/composition"; + +type PeerMetadata = { displayName?: string }; + +function PeerTile({ peer }: { peer: PeerWithStreams }) { + const camera = peer.cameraStream; + const cameraOn = camera?.video && !camera.video.paused; + const speaking = useSpeakingState(peer.id) === "speech"; + const name = peer.metadata.peer?.displayName ?? peer.id; + + return ( + + {cameraOn ? ( + + + + ) : ( + + {name} + + )} + + ); +} + +export default function App() { + const peers = usePeers(); + const connected = peers.filter((peer) => peer.streams.length > 0); + + return ( + + + {connected.map((peer) => ( + + ))} + + + ); +} +``` + +| Hook | Returns | +| -------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `usePeers()` | All peers in the forwarded room, each with its streams (`cameraStream`, `screenShareStream`, `customStreams`). | +| `usePeer(peerId)` | A single peer, or `undefined`. | +| `useRoom()` | The forwarded room `{ id }`, or `undefined` before a room is forwarded. | +| `useSpeakingState(peerId)` | `"speech"` or `"silence"` for active-speaker highlighting. | + +The key link is `stream.inputId`: you pass it to `` to render that participant's forwarded track. A peer's streams fill in once its media actually starts flowing into the composition. + +Build the bundle as usual with `npm run build`. + +## Step 2: Create the composition and register the templated output + +Create the composition with auto-start and auto-cleanup off, since the room's inputs only appear once forwarding starts: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "autostart": false, "cleanup_without_inputs": false }' +``` + +Register a templated `whip_client` output that pushes to your livestream's WHIP endpoint. The output configuration and the template bundle go together in one multipart request: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/template" \ + -H "Authorization: Bearer $TOKEN" \ + -F 'config={ + "type": "whip_client", + "endpoint_url": "", + "bearer_token": "", + "video": { "resolution": { "width": 1280, "height": 720 }, "initial": { "root": { "type": "view" } } }, + "audio": { "initial": { "inputs": [] } } + };type=application/json' \ + -F "template=@dist/App.js" +``` + +## Step 3: Forward the room into the composition + +One call to the [Fishjam Server API](/api/rest) wires everything up: + +```bash +curl -X POST "https://fishjam.io/api/v1/connect/$FISHJAM_ID/room/$ROOM_ID/track_forwardings" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ \"compositionURL\": \"$COMPOSITION_URL/api/composition/$COMPOSITION\", \"selector\": \"all\" }" +``` + +Everything else happens automatically: Fishjam links the room to the composition, registers an input for every forwarded track, and streams the media in. Your template's `usePeers()` fills with the room's peers as their media starts flowing. You never register room inputs by hand. + +## Step 4: Start the composition + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/start" \ + -H "Authorization: Bearer $TOKEN" +``` + +Viewers can now watch the composed grid through the livestream's WHEP endpoint. + +## Step 5: Clean up + +Delete the composition when you are done; deleting the room (or removing its track forwarding) stops the media on the Fishjam side. + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" +``` diff --git a/docs/how-to/compositions/drive-a-template-with-events.mdx b/docs/how-to/compositions/drive-a-template-with-events.mdx new file mode 100644 index 00000000..edee9886 --- /dev/null +++ b/docs/how-to/compositions/drive-a-template-with-events.mdx @@ -0,0 +1,68 @@ +--- +type: how-to +sidebar_position: 3 +description: Send custom events to a running composition to update a template's on-screen state at runtime. +--- + +# Drive a template with events + +Once a [template](./write-and-deploy-a-template) is running, you can push **custom events** to it from your backend to change what it shows, such as toggling a live badge or updating a caption, without re-uploading the template. + +## Handle events in the template + +Inside a template, subscribe to events with the `eventBus`. Each `on` call returns an unsubscribe function. Event names and payloads are entirely defined by your application. + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { useEffect, useState } from "react"; +import { Text, View } from "@swmansion/smelter"; +import { eventBus } from "@fishjam-cloud/composition"; + +export default function App() { + const [caption, setCaption] = useState(null); + const [live, setLive] = useState(false); + + useEffect(() => { + const unsubs = [ + eventBus.on<{ text: string }>("SET_CAPTION", ({ text }) => + setCaption(text), + ), + eventBus.on("CLEAR_CAPTION", () => setCaption(null)), + eventBus.on<{ live: boolean }>("SET_LIVE", ({ live }) => setLive(live)), + ]; + return () => unsubs.forEach((off) => off()); + }, []); + + return ( + + {live && ● LIVE} + {caption && {caption}} + + ); +} +``` + +## Send events from your backend + +Send an event with `POST /api/composition/{composition_id}/event`. The `event_name` is matched against your `eventBus.on(...)` subscriptions, and `data` is delivered as the handler's argument. + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "event_name": "SET_CAPTION", "data": { "text": "Welcome to the stream" } }' +``` + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "event_name": "SET_LIVE", "data": { "live": true } }' +``` + +The template re-renders as soon as the event arrives. + +:::note +Event names and payloads are an application-defined contract between your backend and your template. There is no canonical set of events; `SET_LIVE` and `SET_CAPTION` above are just examples. Pick names and payload shapes that fit your app, and keep them in sync on both sides. +::: diff --git a/docs/how-to/compositions/inputs-and-outputs.mdx b/docs/how-to/compositions/inputs-and-outputs.mdx new file mode 100644 index 00000000..1de74274 --- /dev/null +++ b/docs/how-to/compositions/inputs-and-outputs.mdx @@ -0,0 +1,96 @@ +--- +type: how-to +sidebar_position: 4 +description: Choose the right input and output protocols for a composition, from WebRTC to RTMP, HLS, and MP4. +--- + +# Choose inputs and outputs + +A composition pulls media in through **inputs** and pushes the composed result out through **outputs**. Each is a tagged object whose `type` selects the protocol. This guide summarizes the available types and when to use each. + +## Inputs + +Register an input with `POST /api/composition/{composition_id}/input/{input_id}/register` and a body whose `type` is one of: + +| `type` | Use it when | Key fields | +| ------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `whip_server` | A WebRTC client should publish **into** the composition. | `bearer_token` (optional; generated and returned if omitted), `video` (optional, default `true`). | +| `whep_client` | The composition should **pull** a WebRTC stream from a WHEP endpoint. | `endpoint_url` (required), `bearer_token` (optional), `video` (optional). | +| `rtmp_server` | An encoder (OBS, hardware) should push RTMP in. | `app` (required), `stream_key` (required). | +| `hls` | The composition should pull an HLS playlist. | `url` (required). | +| `mp4` | You want to compose a file, optionally looped. | `url` (optional), `loop` (optional). | + +For `whip_server`, the register response returns the `bearer_token` a publisher uses to authenticate against the input's WHIP endpoint: + +```json +{ "bearer_token": "" } +``` + +Unregister any input with `POST …/input/{input_id}/unregister`. + +## Outputs + +Register an output with `POST /api/composition/{composition_id}/output/{output_id}/register` and a body whose `type` is one of: + +| `type` | Use it when | Key fields | +| ------------- | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| `whip_client` | Publish the composed result over WebRTC (for example to a Fishjam livestream). | `endpoint_url` (required), `bearer_token` (optional), `video`, `audio`. | +| `rtmp_client` | Publish to an RTMP destination (for example a social platform). | `url` (required), `video`, `audio`. | + +An output's `video` carries the resolution and initial [scene](./../../explanation/compositions#scenes), and `audio` carries the initial audio scene: + +```json +{ + "type": "whip_client", + "endpoint_url": "https://example.com/whip", + "video": { + "resolution": { "width": 1280, "height": 720 }, + "initial": { "root": { "type": "tiles", "children": [] } } + }, + "audio": { "initial": { "inputs": [] } } +} +``` + +Other output operations: + +- **Register a templated output** with `POST …/output/{output_id}/template` instead of `…/register`: a multipart request carrying the same configuration plus a template bundle (see [Write and deploy a template](./write-and-deploy-a-template)). +- **Update the scene** live with `POST …/output/{output_id}/update` (see [Scenes](./../../explanation/compositions#scenes)). +- **Force a keyframe** with `POST …/output/{output_id}/request_keyframe`, useful when a new subscriber joins. +- **Unregister** with `POST …/output/{output_id}/unregister`. + +## Renderers + +Renderers are shared assets you register once and then place in any scene. + +### Images + +Register an image with `POST …/image/{image_id}/register`: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/image/logo/register" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "asset_type": "auto", "url": "https://example.com/logo.png" }' +``` + +`asset_type` selects the format: `png`, `jpeg`, `svg`, `gif`, or `auto` to detect it from the URL. Each takes a `url`; `svg` also accepts a `resolution`. Place the image in a scene with an `image` component: + +```json +{ "type": "image", "image_id": "logo" } +``` + +Unregister with `POST …/image/{image_id}/unregister`. + +### Fonts + +Register a font with a multipart request carrying the font file in a single `font` part: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/font/register" \ + -H "Authorization: Bearer $TOKEN" \ + -F "font=@BrandFont.ttf" +``` + +The font is then available to `text` components. + +See the [Composition API](./../../api/reference#compositions) reference for the complete request and response schemas. diff --git a/docs/how-to/compositions/write-and-deploy-a-template.mdx b/docs/how-to/compositions/write-and-deploy-a-template.mdx new file mode 100644 index 00000000..9ac611d0 --- /dev/null +++ b/docs/how-to/compositions/write-and-deploy-a-template.mdx @@ -0,0 +1,126 @@ +--- +type: how-to +sidebar_position: 1 +description: Write a React composition template, bundle it with the CLI, and deploy it as an output. +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Write and deploy a template + +A **template** is a React component that describes an output's layout. Instead of sending a static scene, you write a component, bundle it into a single file with the composition CLI, and upload it when registering an output. + +## Scaffold a project + +Create a new template project with the CLI: + +```bash npm2yarn +npx @fishjam-cloud/composition-cli init my-template +cd my-template +npm install +``` + +This generates a ready-to-build project: a `package.json` with `build` and `typecheck` scripts, a TypeScript config, and a starter `src/App.tsx`. + +## Write the template + +A template **default-exports a React component**. You lay out the composition with components from [`@swmansion/smelter`](https://smelter.dev). The example below renders one registered input full-frame (`camera_1` from the [Compositions tutorial](./../../tutorials/compositions)) with a caption on top: + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { InputStream, Rescaler, Text, View } from "@swmansion/smelter"; + +export default function App() { + return ( + + + + + + + Hello from a template + + + + ); +} +``` + +`` renders one of the composition's registered inputs. Layout, styling, and every visual component (`View`, `Tiles`, `InputStream`, `Rescaler`, `Text`, `Image`, …) come from `@swmansion/smelter`. Its style model is not CSS: see the [Smelter reference](https://smelter.dev) for the full component and style documentation. + +Templates really shine when composing a Fishjam room: hooks from `@fishjam-cloud/composition` give the template live room state, so it re-renders as participants join, leave, or speak. See [Compose a Fishjam room](./compose-a-fishjam-room). + +## Build the bundle + +```bash npm2yarn +npm run build +``` + +This bundles and validates your template into a single file (default `dist/App.js`) against the template contract: + +- **A single JS bundle**: no CSS or extra emitted files. +- **Allowed imports only**: `react`, `react/jsx-runtime`, `@swmansion/smelter`, and `@fishjam-cloud/composition`. These are provided by the runtime and stay out of your bundle; anything else fails the build. +- **A default-exported React component.** +- **No `import.meta` and no dynamic `import()`** with a non-literal specifier. +- **Under ~950 KB**: the bundle ceiling after reserving room for the upload. +- **Loadable**: a smoke test imports the bundle and checks that the default export is a valid React component within a few seconds. + +## Deploy the template + +A templated output is registered in a **single multipart request** to `POST …/output/{output_id}/template`. Do not call the plain `…/register` endpoint first. The request carries two parts: + +- `config`: the same JSON body a regular output registration takes (see [Choose inputs and outputs](./inputs-and-outputs)). Its `video.initial` scene is only a placeholder; the template takes over rendering as soon as it loads. +- `template`: the built bundle. + + + + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/template" \ + -H "Authorization: Bearer $TOKEN" \ + -F 'config={ + "type": "whip_client", + "endpoint_url": "https://example.com/whip", + "video": { + "resolution": { "width": 1280, "height": 720 }, + "initial": { "root": { "type": "view" } } + }, + "audio": { "initial": { "inputs": [] } } + };type=application/json' \ + -F "template=@dist/App.js" +``` + + + + +```js +const config = { + type: "whip_client", + endpoint_url: "https://example.com/whip", + video: { + resolution: { width: 1280, height: 720 }, + initial: { root: { type: "view" } }, + }, + audio: { initial: { inputs: [] } }, +}; + +const form = new FormData(); +form.append("config", JSON.stringify(config)); +form.append("template", new Blob([bundle]), "App.js"); + +await fetch( + `${COMPOSITION_URL}/api/composition/${composition}/output/main/template`, + { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + body: form, + }, +); +``` + + + + +To feed a whole Fishjam room's peers into the template, continue with [Compose a Fishjam room](./compose-a-fishjam-room). To update the template's on-screen state at runtime, see [Drive a template with events](./drive-a-template-with-events). diff --git a/docs/tutorials/compositions.mdx b/docs/tutorials/compositions.mdx new file mode 100644 index 00000000..328aedb4 --- /dev/null +++ b/docs/tutorials/compositions.mdx @@ -0,0 +1,141 @@ +--- +type: tutorial +title: Compositions +sidebar_position: 3.5 +description: Create your first composition end to end, from two WHIP inputs to a composed WHIP output you can play back. +--- + +# Compositions + +This tutorial walks you through creating your first composition. You will compose two live camera streams into a single side-by-side output and play it back. + +## What you'll build + +A composition with two WebRTC inputs arranged into a tiled layout, published as one WebRTC output. + +``` +[camera 1] ──WHIP──▶ ┐ + ├─▶ composition (tiles) ──WHIP──▶ [viewer via WHEP] +[camera 2] ──WHIP──▶ ┘ +``` + +## What you'll learn + +- How to create a composition. +- How to register inputs and an output with a layout. +- How to publish to and play back from a composition. + +## Prerequisites + +- A Fishjam account. Open the [**Fishjam developer panel**](https://fishjam.io/app) and copy your **Management Token**. +- A WHIP/WHEP client to publish and play streams. Any standard WebRTC tool works. + +All requests go to `https://rtc.fishjam.io` and carry your token as a Bearer header: + +```bash +export COMPOSITION_URL="https://rtc.fishjam.io" +export TOKEN="" +``` + +## Step 1: Create a composition + +```bash +curl -X POST "$COMPOSITION_URL/api/composition" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +The response contains the `composition_id` you will use for every subsequent call: + +```json +{ "composition_id": "abc123", "api_url": "https://rtc.fishjam.io" } +``` + +By default a composition auto-starts and is cleaned up automatically once it has no inputs. Save the id: + +```bash +export COMPOSITION="abc123" +``` + +## Step 2: Register two inputs + +Register two `whip_server` inputs. Each one returns a `bearer_token` that a publisher uses to push media in. + +```bash +for INPUT in camera_1 camera_2; do + curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/input/$INPUT/register" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "type": "whip_server" }' +done +``` + +Each response returns the token for that input: + +```json +{ "bearer_token": "" } +``` + +## Step 3: Register an output with a layout + +Register a `whip_client` output whose scene arranges both inputs in a `tiles` layout and mixes their audio. Replace `endpoint_url` with the WHIP endpoint of the service that should receive the composed stream, for example a [Fishjam livestream](./../explanation/livestreams). + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/register" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "type": "whip_client", + "endpoint_url": "https://example.com/whip", + "video": { + "resolution": { "width": 1280, "height": 720 }, + "initial": { + "root": { + "type": "tiles", + "children": [ + { "type": "input_stream", "input_id": "camera_1" }, + { "type": "input_stream", "input_id": "camera_2" } + ] + } + } + }, + "audio": { + "initial": { + "inputs": [ + { "input_id": "camera_1" }, + { "input_id": "camera_2" } + ] + } + } + }' +``` + +The scene is a tree of components. `tiles` automatically arranges its children into a grid, and each `input_stream` renders one registered input. See [Scenes](../explanation/compositions#scenes) for the full component model. + +## Step 4: Publish and play + +Publish a camera to each input's WHIP endpoint using the input token from Step 2: + +``` +POST https://rtc.fishjam.io/api/composition/abc123/whip/camera_1 +Authorization: Bearer +Content-Type: application/sdp +``` + +Your composed output is now publishing to the `endpoint_url` you set in Step 3. Play it back through the receiving service's WHEP endpoint with any WHEP client. + +## Step 5: Clean up + +Delete the composition when you are done: + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" +``` + +## Next steps + +- [Write and deploy a template](../how-to/compositions/write-and-deploy-a-template) to replace the static scene with a live React layout. +- [Compose a Fishjam room](../how-to/compositions/compose-a-fishjam-room) to feed a whole room's peers in automatically. +- [Choose inputs and outputs](../how-to/compositions/inputs-and-outputs) for other protocols like RTMP and HLS. diff --git a/docusaurus.config.ts b/docusaurus.config.ts index cf709720..fa61516f 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -75,71 +75,10 @@ function injectTypeDocSidebar( ): NormalizedSidebar { return items.map((item) => { if (item.customProps?.id === "generated-api" && item.type === "category") { - const injectedItems: (CustomInjectedCategory | NormalizedSidebarItem)[] = - [ - { - type: "category", - label: "React Native SDK", - link: { type: "doc", id: "api/mobile/index" }, - items: require( - `${version.contentPath}/api/mobile/typedoc-sidebar.cjs`, - ), - }, - { - type: "category", - label: "React SDK", - link: { type: "doc", id: "api/web/index" }, - items: require( - `${version.contentPath}/api/web/typedoc-sidebar.cjs`, - ), - }, - { - type: "category", - label: "Server SDK for JS", - link: { type: "doc", id: "api/server/index" }, - items: require( - `${version.contentPath}/api/server/typedoc-sidebar.cjs`, - ), - }, - ]; - - // The custom-source packages ship after 0.28, so older versioned docs - // don't contain their generated API trees — inject only when present. - const customSourceCategories: ( - | CustomInjectedCategory - | NormalizedSidebarItem - )[] = []; - for (const { label, dir } of [ - { label: "Vision Camera Source", dir: "vision-camera-source" }, - { label: "Custom Video Source", dir: "custom-video-source" }, - ]) { - const sidebarModule = `${version.contentPath}/api/${dir}/typedoc-sidebar.cjs`; - if (fs.existsSync(sidebarModule)) { - customSourceCategories.push({ - type: "category", - label, - link: { type: "doc", id: `api/${dir}/index` }, - items: require(sidebarModule), - }); - } - } - injectedItems.splice(1, 0, ...customSourceCategories); - - injectedItems.push({ - type: "category", - label: "Server SDK for Python", - items: [ - { - type: "autogenerated", - dirName: "api/server-python", - }, - ], - }); - return { ...item, items: [ - ...injectedItems, + ...buildInjectedApiItems(version), ...item.items.filter((element) => element.type === "doc"), ] as NormalizedSidebar, }; @@ -149,6 +88,64 @@ function injectTypeDocSidebar( }); } +function buildInjectedApiItems(version: SidebarItemsGeneratorVersion) { + const injectedItems: (CustomInjectedCategory | NormalizedSidebarItem)[] = [ + { + type: "category", + label: "React Native SDK", + link: { type: "doc", id: "api/mobile/index" }, + items: require(`${version.contentPath}/api/mobile/typedoc-sidebar.cjs`), + }, + { + type: "category", + label: "React SDK", + link: { type: "doc", id: "api/web/index" }, + items: require(`${version.contentPath}/api/web/typedoc-sidebar.cjs`), + }, + { + type: "category", + label: "Server SDK for JS", + link: { type: "doc", id: "api/server/index" }, + items: require(`${version.contentPath}/api/server/typedoc-sidebar.cjs`), + }, + ]; + + // The custom-source packages ship after 0.28, so older versioned docs + // don't contain their generated API trees — inject only when present. + const customSourceCategories: ( + | CustomInjectedCategory + | NormalizedSidebarItem + )[] = []; + for (const { label, dir } of [ + { label: "Vision Camera Source", dir: "vision-camera-source" }, + { label: "Custom Video Source", dir: "custom-video-source" }, + ]) { + const sidebarModule = `${version.contentPath}/api/${dir}/typedoc-sidebar.cjs`; + if (fs.existsSync(sidebarModule)) { + customSourceCategories.push({ + type: "category", + label, + link: { type: "doc", id: `api/${dir}/index` }, + items: require(sidebarModule), + }); + } + } + injectedItems.splice(1, 0, ...customSourceCategories); + + injectedItems.push({ + type: "category", + label: "Server SDK for Python", + items: [ + { + type: "autogenerated", + dirName: "api/server-python", + }, + ], + }); + + return injectedItems; +} + const typedocConfig = { readme: "none", parametersFormat: "table", @@ -275,12 +272,17 @@ const config: Config = { activeBaseRegex: "^/docs(?=/|$)(?!.*(ai-skill|/api(/|$)))", }, { + type: "dropdown", to: "/api/rest", label: "API Reference", position: "left", // Highlight across the whole /api section (rest, reference, and the // typedoc web/mobile/server pages), not just the linked /api/rest. activeBaseRegex: "/api(/|$)", + items: [ + { to: "/api/rest", label: "Server API" }, + { to: "/api/compositions", label: "Composition API" }, + ], }, { to: "/ai-skill", @@ -373,7 +375,19 @@ const config: Config = { route: "/api/rest", showNavLink: false, configuration: { - url: "/docs/api/fishjam-server-openapi.yaml", + sources: [ + { + title: "Server API", + slug: "server", + url: "/docs/api/fishjam-server-openapi.yaml", + default: true, + }, + { + title: "Composition API", + slug: "composition", + url: "/docs/api/composition-openapi.json", + }, + ], hideSearch: true, persistAuth: true, defaultOpenFirstTag: false, @@ -383,6 +397,24 @@ const config: Config = { }, } as ScalarOptions, ], + [ + "@scalar/docusaurus", + { + id: "composition-api", + label: "Composition API", + route: "/api/compositions", + showNavLink: false, + configuration: { + url: "/docs/api/composition-openapi.json", + hideSearch: true, + persistAuth: true, + defaultOpenFirstTag: false, + authentication: { + preferredSecurityScheme: "BearerAuth", + }, + }, + } as ScalarOptions, + ], [ "docusaurus-plugin-typedoc", { diff --git a/package.json b/package.json index 2e063292..b4f32936 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@docusaurus/theme-mermaid": "^3.10.0", "@docusaurus/utils": "^3.10.0", "@fastify/env": "^5.0.2", + "@fishjam-cloud/composition": "link:./packages/js-server-sdk/packages/composition", "@fishjam-cloud/js-server-sdk": "link:./packages/js-server-sdk/packages/js-server-sdk", "@fishjam-cloud/react-client": "link:./packages/web-client-sdk/packages/react-client", "@fishjam-cloud/react-native-client": "link:./packages/web-client-sdk/packages/mobile-client", diff --git a/scripts/composition_openapi_overlay.jq b/scripts/composition_openapi_overlay.jq new file mode 100644 index 00000000..fb092ce1 --- /dev/null +++ b/scripts/composition_openapi_overlay.jq @@ -0,0 +1,92 @@ +# Presentation overlay for the composition OpenAPI document, applied by +# update_api.sh when copying the spec from its source repo. +# +# The generated spec carries no operation summaries and no tag metadata, so +# an API reference renders raw operation ids and machine-named groups. This +# overlay adds human titles and regroups operations, and strips fields marked +# "Internal use only". Belongs in the source generator long-term; keep it +# here until the source spec carries this metadata itself. + +def examples: { + create_composition: { autostart: false, cleanup_without_inputs: false }, + register_input: { type: "whip_server" }, + register_output: { + type: "whip_client", + endpoint_url: "https://example.com/whip", + video: { + resolution: { width: 1280, height: 720 }, + initial: { + root: { + type: "tiles", + children: [ + { type: "input_stream", input_id: "camera_1" }, + { type: "input_stream", input_id: "camera_2" } + ] + } + } + }, + audio: { initial: { inputs: [{ input_id: "camera_1" }, { input_id: "camera_2" }] } } + }, + update_output: { + video: { root: { type: "rescaler", child: { type: "input_stream", input_id: "camera_1" } } } + }, + send_composition_event: { event_name: "SET_CAPTION", data: { text: "Welcome to the stream" } }, + link_room: { fishjam_id: "your-fishjam-id", room_id: "room-uuid" } +}; + +def meta: { + create_composition: { summary: "Create a composition", tag: "Compositions" }, + delete_composition: { summary: "Delete a composition", tag: "Compositions" }, + start: { summary: "Start a composition", tag: "Compositions" }, + reset: { summary: "Reset a composition", tag: "Compositions" }, + register_input: { summary: "Register an input", tag: "Inputs" }, + unregister_input: { summary: "Unregister an input", tag: "Inputs" }, + register_output: { summary: "Register an output", tag: "Outputs" }, + register_template_output: { summary: "Register a templated output", tag: "Outputs" }, + update_output: { summary: "Update an output's scene", tag: "Outputs" }, + request_keyframe: { summary: "Request a keyframe", tag: "Outputs" }, + unregister_output: { summary: "Unregister an output", tag: "Outputs" }, + register_image: { summary: "Register an image", tag: "Renderers" }, + unregister_image: { summary: "Unregister an image", tag: "Renderers" }, + register_font: { summary: "Register a font", tag: "Renderers" }, + send_composition_event: { summary: "Send an event to templates", tag: "Events" }, + ws: { summary: "Subscribe to engine events", tag: "Events" }, + link_room: { summary: "Link a Fishjam room", tag: "Room forwarding" }, + unlink_room: { summary: "Unlink a Fishjam room", tag: "Room forwarding" }, + whip_offer: { summary: "Publish to an input over WHIP", tag: "Media transport" }, + whep_offer: { summary: "Play an output over WHEP", tag: "Media transport" } +}; + +del(.components.schemas.WhipInput.properties.endpoint_override) +| .components.schemas.Mp4Input.description = "Input stream from an MP4 file." +| .components.schemas.Framerate.oneOf[0].title = "Ratio string" +| .components.schemas.Framerate.oneOf[1].title = "Frames per second" +| .components.schemas.PortOrPortRange.oneOf[0].title = "Port range string" +| .components.schemas.PortOrPortRange.oneOf[1].title = "Single port" +| .components.schemas.VideoEncoderBitrate.oneOf[0].title = "Average bitrate" +| .components.schemas.VideoEncoderBitrate.oneOf[1].title = "Average and max bitrate" +| .components.schemas.Response.oneOf[0].title = "Bearer token" +| .components.schemas.Response.oneOf[1].title = "Media durations" +| .components.schemas.Response.oneOf[2].title = "Port" +| .components.schemas.Response.oneOf[3].title = "Empty" +| .paths |= map_values( + map_values( + if type == "object" and has("operationId") and (meta[.operationId] != null) then + . + { summary: meta[.operationId].summary, tags: [meta[.operationId].tag] } + | (if examples[.operationId] != null then + .requestBody.content["application/json"].example = examples[.operationId] + else . end) + else + . + end + ) + ) +| .tags = [ + { name: "Compositions", description: "A composition is a single running compositing session. Create one, start it, and delete it when you are done." }, + { name: "Inputs", description: "Live media sources being composed: WHIP, WHEP, RTMP, HLS, or MP4." }, + { name: "Outputs", description: "Where the composed result is sent, over WHIP or RTMP. Each output carries a scene or renders a template." }, + { name: "Renderers", description: "Shared assets, such as images and fonts, that scenes can reference." }, + { name: "Events", description: "Custom events delivered to templates, and the WebSocket stream of engine events." }, + { name: "Room forwarding", description: "Links a Fishjam room to the composition. Called by Fishjam automatically when a room's track forwarding is created." }, + { name: "Media transport", description: "WHIP publishing into inputs and WHEP playback of outputs." } + ] diff --git a/scripts/update_api.sh b/scripts/update_api.sh index 05c51e68..bdf5b046 100755 --- a/scripts/update_api.sh +++ b/scripts/update_api.sh @@ -37,6 +37,36 @@ copy_openapi() { fi } +# TODO: switch to main once the template-workers branch is merged there. +# Note: until that merge, the branch's checked-in spec lacks the oneOf variant +# titles the committed composition-openapi.json already carries, so re-running +# this script before the merge drops them. +COMPOSITION_BRANCH="template-workers" + +# The composition source repo does not tag semver releases yet, so its +# submodule is checked out at a branch instead of the latest tag. +checkout_submodule_branch() { + local submodule_path + submodule_path=api/$1 + + cd $CWD/$submodule_path + git fetch origin $2 + git checkout FETCH_HEAD --detach &>/dev/null +} + +# The spec is passed through a presentation overlay before publishing: see +# scripts/composition_openapi_overlay.jq. +copy_composition_openapi() { + if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to process the composition spec. Install jq and re-run." >&2 + exit 1 + fi + + jq -f "$CWD/scripts/composition_openapi_overlay.jq" openapi.json \ + >"$ASSETS_DIRECTORY/composition-openapi.json" + echo "Copied openapi.json of composition to the assets directory." +} + PROTO_FILES="server_notifications agent_notifications notifications/shared" copy_protos() { for file in $PROTO_FILES; do @@ -53,4 +83,7 @@ copy_openapi room-manager checkout_submodule protos copy_protos +checkout_submodule_branch composition $COMPOSITION_BRANCH +copy_composition_openapi + echo $'\nSubmodule update and copy complete.' diff --git a/spelling.txt b/spelling.txt index c063804a..204ebef6 100644 --- a/spelling.txt +++ b/spelling.txt @@ -119,3 +119,4 @@ unplugin unorm bgra BGRA +rtmp diff --git a/static/api/composition-openapi.json b/static/api/composition-openapi.json new file mode 100644 index 00000000..544bb7a2 --- /dev/null +++ b/static/api/composition-openapi.json @@ -0,0 +1,3766 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Composition API", + "description": "Real-time video compositing", + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0" + }, + "version": "0.1.0" + }, + "servers": [ + { + "url": "https://rtc.fishjam.io" + } + ], + "paths": { + "/api/composition": { + "post": { + "tags": ["Compositions"], + "operationId": "create_composition", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateCompositionRequest" + }, + "example": { + "autostart": false, + "cleanup_without_inputs": false + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Composition created.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CompositionCreatedResponse" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Create a composition" + } + }, + "/api/composition/{composition_id}/output/{output_id}/template": { + "post": { + "tags": ["Outputs"], + "operationId": "register_template_output", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": ["config", "template"], + "properties": { + "config": { + "$ref": "#/components/schemas/RegisterOutput" + }, + "template": { + "type": "string", + "format": "binary" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Output registered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Register a templated output" + } + }, + "/api/composition/{composition_id}/room": { + "post": { + "tags": ["Room forwarding"], + "operationId": "link_room", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["fishjam_id", "room_id"], + "properties": { + "fishjam_id": { + "type": "string" + }, + "room_id": { + "type": "string" + } + } + }, + "example": { + "fishjam_id": "your-fishjam-id", + "room_id": "room-uuid" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "Composition linked to room." + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Link a Fishjam room" + }, + "delete": { + "tags": ["Room forwarding"], + "operationId": "unlink_room", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Composition unlinked from room." + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Unlink a Fishjam room" + } + }, + "/api/composition/{composition_id}/event": { + "post": { + "tags": ["Events"], + "operationId": "send_composition_event", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "required": ["event_name"], + "properties": { + "event_name": { + "type": "string", + "description": "Name of the event delivered to the composition's templates.", + "example": "START_LIVE", + "maxLength": 128, + "minLength": 1 + }, + "data": { + "description": "Optional arbitrary JSON payload delivered with the event." + } + } + }, + "example": { + "event_name": "SET_CAPTION", + "data": { + "text": "Welcome to the stream" + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Event accepted (dispatched, or a no-op if no output has an active template).", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Send an event to templates" + } + }, + "/api/composition/{composition_id}/ws": { + "get": { + "tags": ["Events"], + "description": "Open a WebSocket to the composition.", + "operationId": "ws", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "101": { + "description": "WebSocket connection established." + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "502": { + "description": "Bad gateway.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "503": { + "description": "Service unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "504": { + "description": "Gateway timeout.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "WebSocketSubprotocolAuth": [] + } + ], + "summary": "Subscribe to engine events" + } + }, + "/api/composition/{composition_id}": { + "delete": { + "tags": ["Compositions"], + "operationId": "delete_composition", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Composition deleted." + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Delete a composition" + } + }, + "/api/composition/{composition_id}/start": { + "post": { + "tags": ["Compositions"], + "operationId": "start", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Composition started.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Start a composition" + } + }, + "/api/composition/{composition_id}/reset": { + "post": { + "tags": ["Compositions"], + "operationId": "reset", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Composition reset.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Reset a composition" + } + }, + "/api/composition/{composition_id}/input/{input_id}/register": { + "post": { + "tags": ["Inputs"], + "operationId": "register_input", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "input_id", + "in": "path", + "description": "Input ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterInput" + }, + "example": { + "type": "whip_server" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Input registered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Register an input" + } + }, + "/api/composition/{composition_id}/output/{output_id}/register": { + "post": { + "tags": ["Outputs"], + "operationId": "register_output", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterOutput" + }, + "example": { + "type": "whip_client", + "endpoint_url": "https://example.com/whip", + "video": { + "resolution": { + "width": 1280, + "height": 720 + }, + "initial": { + "root": { + "type": "tiles", + "children": [ + { + "type": "input_stream", + "input_id": "camera_1" + }, + { + "type": "input_stream", + "input_id": "camera_2" + } + ] + } + } + }, + "audio": { + "initial": { + "inputs": [ + { + "input_id": "camera_1" + }, + { + "input_id": "camera_2" + } + ] + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Output registered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Register an output" + } + }, + "/api/composition/{composition_id}/image/{image_id}/register": { + "post": { + "tags": ["Renderers"], + "operationId": "register_image", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "image_id", + "in": "path", + "description": "Image ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImageSpec" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Image registered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Register an image" + } + }, + "/api/composition/{composition_id}/font/register": { + "post": { + "tags": ["Renderers"], + "operationId": "register_font", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": ["font"], + "properties": { + "font": { + "type": "string", + "format": "binary" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Font registered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Register a font" + } + }, + "/api/composition/{composition_id}/input/{input_id}/unregister": { + "post": { + "tags": ["Inputs"], + "operationId": "unregister_input", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "input_id", + "in": "path", + "description": "Input ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnregisterInput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Input unregistered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Input not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Unregister an input" + } + }, + "/api/composition/{composition_id}/output/{output_id}/unregister": { + "post": { + "tags": ["Outputs"], + "operationId": "unregister_output", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnregisterOutput" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Output unregistered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Output not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Unregister an output" + } + }, + "/api/composition/{composition_id}/image/{image_id}/unregister": { + "post": { + "tags": ["Renderers"], + "operationId": "unregister_image", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "image_id", + "in": "path", + "description": "Image ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UnregisterRenderer" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Image unregistered successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Image not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Unregister an image" + } + }, + "/api/composition/{composition_id}/output/{output_id}/update": { + "post": { + "tags": ["Outputs"], + "operationId": "update_output", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateOutputRequest" + }, + "example": { + "video": { + "root": { + "type": "rescaler", + "child": { + "type": "input_stream", + "input_id": "camera_1" + } + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Output updated successfully.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Update an output's scene" + } + }, + "/api/composition/{composition_id}/output/{output_id}/request_keyframe": { + "post": { + "tags": ["Outputs"], + "operationId": "request_keyframe", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Keyframe request successful.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Response" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Request a keyframe" + } + }, + "/api/composition/{composition_id}/whip/{input_id}": { + "post": { + "tags": ["Media transport"], + "operationId": "whip_offer", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "input_id", + "in": "path", + "description": "Input ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "SDP offer.", + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Session created.", + "headers": { + "location": { + "schema": { + "type": "string" + }, + "description": "Session URL." + } + }, + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition or input not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "summary": "Publish to an input over WHIP" + } + }, + "/api/composition/{composition_id}/whep/{output_id}": { + "post": { + "tags": ["Media transport"], + "operationId": "whep_offer", + "parameters": [ + { + "name": "composition_id", + "in": "path", + "description": "Composition ID.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "output_id", + "in": "path", + "description": "Output ID.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "SDP offer.", + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Session created.", + "headers": { + "location": { + "schema": { + "type": "string" + }, + "description": "Session URL." + } + }, + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + } + }, + "400": { + "description": "Bad request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "404": { + "description": "Composition or output not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "500": { + "description": "Internal server error.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + } + }, + "security": [ + {}, + { + "BearerAuth": [] + } + ], + "summary": "Play an output over WHEP" + } + } + }, + "components": { + "schemas": { + "ApiError": { + "type": "object", + "required": ["message", "http_status_code"], + "properties": { + "message": { + "type": "string" + }, + "http_status_code": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + } + }, + "AspectRatio": { + "type": "string" + }, + "AudioChannels": { + "type": "string", + "enum": ["mono", "stereo"] + }, + "AudioMixingStrategy": { + "type": "string", + "enum": ["sum_clip", "sum_scale"] + }, + "AudioScene": { + "type": "object", + "required": ["inputs"], + "properties": { + "inputs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AudioSceneInput" + } + } + }, + "additionalProperties": false + }, + "AudioSceneInput": { + "type": "object", + "required": ["input_id"], + "properties": { + "input_id": { + "$ref": "#/components/schemas/InputId" + }, + "volume": { + "type": ["number", "null"], + "format": "float", + "description": "(**default=`1.0`**) float in `[0, 2]` range representing input volume" + } + }, + "additionalProperties": false + }, + "BoxShadow": { + "type": "object", + "properties": { + "offset_x": { + "type": ["number", "null"], + "format": "float" + }, + "offset_y": { + "type": ["number", "null"], + "format": "float" + }, + "color": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/RGBAColor" + } + ] + }, + "blur_radius": { + "type": ["number", "null"], + "format": "float" + } + }, + "additionalProperties": false + }, + "Component": { + "oneOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/InputStream" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["input_stream"] + } + } + } + ], + "title": "ComponentInputStream" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/View" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["view"] + } + } + } + ], + "title": "ComponentView" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/Text" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["text"] + } + } + } + ], + "title": "ComponentText" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/Tiles" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["tiles"] + } + } + } + ], + "title": "ComponentTiles" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/Rescaler" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["rescaler"] + } + } + } + ], + "title": "ComponentRescaler" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/Image" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["image"] + } + } + } + ], + "title": "ComponentImage" + } + ] + }, + "ComponentId": { + "type": "string" + }, + "CompositionCreatedResponse": { + "type": "object", + "required": ["composition_id", "api_url"], + "properties": { + "composition_id": { + "$ref": "#/components/schemas/CompositionId" + }, + "api_url": { + "type": "string" + } + } + }, + "CompositionEvent": { + "type": "object", + "required": ["event_name"], + "properties": { + "event_name": { + "type": "string", + "description": "Name of the event delivered to the composition's templates.", + "example": "START_LIVE", + "maxLength": 128, + "minLength": 1 + }, + "data": { + "description": "Optional arbitrary JSON payload delivered with the event." + } + } + }, + "CompositionId": { + "type": "string" + }, + "CreateCompositionRequest": { + "type": "object", + "properties": { + "autostart": { + "type": "boolean", + "description": "If true, outputs will immediately start producing audio and video.\nIf false, call `POST /api/composition/{composition_id}/start` to start the composition.", + "default": true + }, + "cleanup_without_inputs": { + "type": "boolean", + "description": "If true (default), the composition will be cleaned up after 5 minutes when all **inputs**\nhave zero bitrate, regardless of output bitrate. This prevents circular liveness when\ncomposition output is sent to a stream.\nIf false, cleanup only triggers when both inputs and outputs have zero bitrate.", + "default": true + } + } + }, + "EasingFunction": { + "oneOf": [ + { + "type": "object", + "title": "EasingFunctionLinear", + "required": ["function_name"], + "properties": { + "function_name": { + "type": "string", + "enum": ["linear"] + } + } + }, + { + "type": "object", + "title": "EasingFunctionBounce", + "required": ["function_name"], + "properties": { + "function_name": { + "type": "string", + "enum": ["bounce"] + } + } + }, + { + "type": "object", + "title": "EasingFunctionCubicBezier", + "required": ["points", "function_name"], + "properties": { + "points": { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + "function_name": { + "type": "string", + "enum": ["cubic_bezier"] + } + } + } + ], + "description": "Easing functions are used to interpolate between two values over time.\n\nCustom easing functions can be implemented with cubic Bézier.\nThe control points are defined with `points` field by providing four numerical values: `x1`, `y1`, `x2` and `y2`. The `x1` and `x2` values have to be in the range `[0; 1]`. The cubic Bézier result is clamped to the range `[0; 1]`.\nYou can find example control point configurations [here](https://easings.net/)." + }, + "FontUpload": { + "type": "object", + "required": ["font"], + "properties": { + "font": { + "type": "string", + "format": "binary" + } + } + }, + "Framerate": { + "oneOf": [ + { + "type": "string", + "title": "Ratio string" + }, + { + "type": "integer", + "format": "int32", + "minimum": 0, + "title": "Frames per second" + } + ] + }, + "H264EncoderPreset": { + "type": "string", + "enum": [ + "ultrafast", + "superfast", + "veryfast", + "faster", + "fast", + "medium", + "slow", + "slower", + "veryslow", + "placebo" + ] + }, + "HlsInput": { + "type": "object", + "description": "Parameters for an input stream from HLS source.", + "required": ["url"], + "properties": { + "url": { + "type": "string", + "description": "URL to HLS playlist" + } + }, + "additionalProperties": false + }, + "HorizontalAlign": { + "type": "string", + "enum": ["left", "right", "justified", "center"] + }, + "Image": { + "type": "object", + "required": ["image_id"], + "properties": { + "id": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ComponentId", + "description": "Id of a component." + } + ] + }, + "image_id": { + "$ref": "#/components/schemas/RendererId", + "description": "Id of an image. It identifies an image registered using a [`register image`](../routes.md#register-image) request." + }, + "width": { + "type": ["number", "null"], + "format": "float", + "description": "Width of the image in pixels.\nIf `height` is not explicitly provided, the image will automatically adjust its height to maintain its original aspect ratio relative to the width." + }, + "height": { + "type": ["number", "null"], + "format": "float", + "description": "Height of the image in pixels.\nIf `width` is not explicitly provided, the image will automatically adjust its width to maintain its original aspect ratio relative to the height." + } + }, + "additionalProperties": false + }, + "ImageSpec": { + "oneOf": [ + { + "type": "object", + "title": "ImageSpecPng", + "required": ["asset_type"], + "properties": { + "url": { + "type": ["string", "null"] + }, + "asset_type": { + "type": "string", + "enum": ["png"] + } + } + }, + { + "type": "object", + "title": "ImageSpecJpeg", + "required": ["asset_type"], + "properties": { + "url": { + "type": ["string", "null"] + }, + "asset_type": { + "type": "string", + "enum": ["jpeg"] + } + } + }, + { + "type": "object", + "title": "ImageSpecSvg", + "required": ["asset_type"], + "properties": { + "url": { + "type": ["string", "null"] + }, + "resolution": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/Resolution" + } + ] + }, + "asset_type": { + "type": "string", + "enum": ["svg"] + } + } + }, + { + "type": "object", + "title": "ImageSpecGif", + "required": ["asset_type"], + "properties": { + "url": { + "type": ["string", "null"] + }, + "asset_type": { + "type": "string", + "enum": ["gif"] + } + } + }, + { + "type": "object", + "title": "ImageSpecAuto", + "required": ["asset_type"], + "properties": { + "url": { + "type": ["string", "null"] + }, + "asset_type": { + "type": "string", + "enum": ["auto"] + } + } + } + ] + }, + "InputId": { + "type": "string" + }, + "InputStream": { + "type": "object", + "required": ["input_id"], + "properties": { + "id": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ComponentId", + "description": "Id of a component." + } + ] + }, + "input_id": { + "$ref": "#/components/schemas/InputId", + "description": "Id of an input. It identifies a stream registered using a [`RegisterInputStream`](../routes.md#register-input) request." + } + }, + "additionalProperties": false + }, + "Interpolation": { + "type": "string", + "enum": ["linear", "spring"] + }, + "Mp4Input": { + "type": "object", + "description": "Input stream from an MP4 file.", + "properties": { + "url": { + "type": ["string", "null"], + "description": "URL of the MP4 file." + }, + "loop": { + "type": ["boolean", "null"], + "description": "(**default=`false`**) If input should be played in the loop. Added in v0.4.0" + } + }, + "additionalProperties": false + }, + "OpusEncoderPreset": { + "type": "string", + "enum": ["quality", "voip", "lowest_latency"] + }, + "OutputEndCondition": { + "type": "object", + "description": "This type defines when end of an input stream should trigger end of the output stream. Only one of those fields can be set at the time.\nUnless specified otherwise the input stream is considered finished/ended when:\n- TCP connection was dropped/closed.\n- RTCP Goodbye packet (`BYE`) was received.\n- Mp4 track has ended.\n- Input was unregistered already (or never registered).", + "properties": { + "any_of": { + "type": ["array", "null"], + "items": { + "$ref": "#/components/schemas/InputId" + }, + "description": "Terminate output stream if any of the input streams from the list are finished." + }, + "all_of": { + "type": ["array", "null"], + "items": { + "$ref": "#/components/schemas/InputId" + }, + "description": "Terminate output stream if all the input streams from the list are finished." + }, + "any_input": { + "type": ["boolean", "null"], + "description": "Terminate output stream if any of the input streams ends. This includes streams added after the output was registered. In particular, output stream will **not be** terminated if no inputs were ever connected." + }, + "all_inputs": { + "type": ["boolean", "null"], + "description": "Terminate output stream if all the input streams finish. In particular, output stream will **be** terminated if no inputs were ever connected." + } + }, + "additionalProperties": false + }, + "OutputId": { + "type": "string" + }, + "OutputRtmpClientAudioOptions": { + "type": "object", + "required": ["initial"], + "properties": { + "mixing_strategy": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/AudioMixingStrategy", + "description": "(**default=\"sum_clip\"**) Specifies how audio should be mixed." + } + ] + }, + "send_eos_when": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OutputEndCondition", + "description": "Condition for termination of the output stream based on the input streams states. If output includes both audio and video streams, then EOS needs to be sent for every type." + } + ] + }, + "channels": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/AudioChannels", + "description": "Channels configuration." + } + ] + }, + "initial": { + "$ref": "#/components/schemas/AudioScene", + "description": "Initial audio mixer configuration for output." + } + }, + "additionalProperties": false + }, + "OutputRtmpClientVideoOptions": { + "type": "object", + "required": ["resolution", "initial"], + "properties": { + "resolution": { + "$ref": "#/components/schemas/Resolution", + "description": "Output resolution in pixels." + }, + "send_eos_when": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OutputEndCondition", + "description": "Condition for termination of the output stream based on the input streams states. If output includes both audio and video streams, then EOS needs to be sent for every type." + } + ] + }, + "initial": { + "$ref": "#/components/schemas/VideoScene", + "description": "Root of a component tree/scene that should be rendered for the output. Use [`update_output` request](../routes.md#update-output) to update this value after registration. [Learn more](../../concept/component.md)." + } + }, + "additionalProperties": false + }, + "OutputWhipAudioOptions": { + "type": "object", + "required": ["initial"], + "properties": { + "mixing_strategy": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/AudioMixingStrategy", + "description": "(**default=\"sum_clip\"**) Specifies how audio should be mixed." + } + ] + }, + "send_eos_when": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OutputEndCondition", + "description": "Condition for termination of output stream based on the input streams states." + } + ] + }, + "channels": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/AudioChannels", + "description": "Specifies channels configuration." + } + ] + }, + "encoder_preferences": { + "type": ["array", "null"], + "items": { + "$ref": "#/components/schemas/WhipAudioEncoderOptions" + }, + "description": "Codec preferences list." + }, + "initial": { + "$ref": "#/components/schemas/AudioScene", + "description": "Initial audio mixer configuration for output." + } + }, + "additionalProperties": false + }, + "OutputWhipVideoOptions": { + "type": "object", + "required": ["resolution", "initial"], + "properties": { + "resolution": { + "$ref": "#/components/schemas/Resolution", + "description": "Output resolution in pixels." + }, + "send_eos_when": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OutputEndCondition", + "description": "Defines when output stream should end if some of the input streams are finished. If output includes both audio and video streams, then EOS needs to be sent on both." + } + ] + }, + "initial": { + "$ref": "#/components/schemas/VideoScene", + "description": "Root of a component tree/scene that should be rendered for the output." + } + }, + "additionalProperties": false + }, + "Overflow": { + "type": "string", + "enum": ["visible", "hidden", "fit"] + }, + "PixelFormat": { + "type": "string", + "enum": ["yuv420p", "yuv422p", "yuv444p"] + }, + "PortOrPortRange": { + "oneOf": [ + { + "type": "string", + "title": "Port range string" + }, + { + "type": "integer", + "format": "int32", + "minimum": 0, + "title": "Single port" + } + ] + }, + "RGBAColor": { + "type": "string" + }, + "RegisterInput": { + "oneOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/RtmpInput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["rtmp_server"] + } + } + } + ], + "title": "RegisterInputRtmpServer" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/Mp4Input" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["mp4"] + } + } + } + ], + "title": "RegisterInputMp4" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/WhipInput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["whip_server"] + } + } + } + ], + "title": "RegisterInputWhipServer" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/WhepInput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["whep_client"] + } + } + } + ], + "title": "RegisterInputWhepClient" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/HlsInput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["hls"] + } + } + } + ], + "title": "RegisterInputHls" + } + ] + }, + "RegisterOutput": { + "oneOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/RtmpOutput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["rtmp_client"] + } + } + } + ], + "title": "RegisterOutputRtmpClient" + }, + { + "allOf": [ + { + "$ref": "#/components/schemas/WhipOutput" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["whip_client"] + } + } + } + ], + "title": "RegisterOutputWhipClient" + } + ] + }, + "RegisterTemplateOutput": { + "type": "object", + "required": ["config", "template"], + "properties": { + "config": { + "$ref": "#/components/schemas/RegisterOutput" + }, + "template": { + "type": "string", + "format": "binary" + } + } + }, + "RendererId": { + "type": "string" + }, + "RescaleMode": { + "type": "string", + "enum": ["fit", "fill"] + }, + "Rescaler": { + "type": "object", + "required": ["child"], + "properties": { + "id": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ComponentId", + "description": "Id of a component." + } + ] + }, + "child": { + "$ref": "#/components/schemas/Component", + "description": "List of component's children." + }, + "mode": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/RescaleMode", + "description": "(**default=`\"fit\"`**) Resize mode:" + } + ] + }, + "horizontal_align": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/HorizontalAlign", + "description": "(**default=`\"center\"`**) Horizontal alignment." + } + ] + }, + "vertical_align": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/VerticalAlign", + "description": "(**default=`\"center\"`**) Vertical alignment." + } + ] + }, + "width": { + "type": ["number", "null"], + "format": "float", + "description": "Width of a component in pixels (without a border). Exact behavior might be different\nbased on the parent component:\n- If the parent component is a layout, check sections \"Absolute positioning\" and \"Static\n positioning\" of that component.\n- If the parent component is not a layout, then this field is required." + }, + "height": { + "type": ["number", "null"], + "format": "float", + "description": "Height of a component in pixels (without a border). Exact behavior might be different\nbased on the parent component:\n- If the parent component is a layout, check sections \"Absolute positioning\" and \"Static\n positioning\" of that component.\n- If the parent component is not a layout, then this field is required." + }, + "top": { + "type": ["number", "null"], + "format": "float", + "description": "Distance in pixels between this component's top edge and its parent's top edge (including a border).\nIf this field is defined, then the component will ignore a layout defined by its parent." + }, + "left": { + "type": ["number", "null"], + "format": "float", + "description": "Distance in pixels between this component's left edge and its parent's left edge (including a border).\nIf this field is defined, this element will be absolutely positioned, instead of being\nlaid out by its parent." + }, + "bottom": { + "type": ["number", "null"], + "format": "float", + "description": "Distance in pixels between the bottom edge of this component and the bottom edge of its\nparent (including a border). If this field is defined, this element will be absolutely\npositioned, instead of being laid out by its parent." + }, + "right": { + "type": ["number", "null"], + "format": "float", + "description": "Distance in pixels between this component's right edge and its parent's right edge.\nIf this field is defined, this element will be absolutely positioned, instead of being\nlaid out by its parent." + }, + "rotation": { + "type": ["number", "null"], + "format": "float", + "description": "Rotation of a component in degrees. If this field is defined, this element will be\nabsolutely positioned, instead of being laid out by its parent." + }, + "transition": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/Transition", + "description": "Defines how this component will behave during a scene update. This will only have an\neffect if the previous scene already contained a `Rescaler` component with the same id." + } + ] + }, + "border_radius": { + "type": ["number", "null"], + "format": "float", + "description": "(**default=`0.0`**) Radius of a rounded corner." + }, + "border_width": { + "type": ["number", "null"], + "format": "float", + "description": "(**default=`0.0`**) Border width." + }, + "border_color": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/RGBAColor", + "description": "(**default=`\"#00000000\"`**) Border color in a `\"#RRGGBBAA\"` format." + } + ] + }, + "box_shadow": { + "type": ["array", "null"], + "items": { + "$ref": "#/components/schemas/BoxShadow" + }, + "description": "List of box shadows." + } + }, + "additionalProperties": false + }, + "Resolution": { + "type": "object", + "required": ["width", "height"], + "properties": { + "width": { + "type": "integer", + "description": "Width in pixels.", + "minimum": 0 + }, + "height": { + "type": "integer", + "description": "Height in pixels.", + "minimum": 0 + } + } + }, + "Response": { + "oneOf": [ + { + "type": "object", + "required": ["bearer_token"], + "properties": { + "bearer_token": { + "type": "string" + } + }, + "title": "Bearer token" + }, + { + "type": "object", + "properties": { + "video_duration_ms": { + "type": ["integer", "null"], + "format": "int64", + "minimum": 0 + }, + "audio_duration_ms": { + "type": ["integer", "null"], + "format": "int64", + "minimum": 0 + } + }, + "title": "Media durations" + }, + { + "type": "object", + "properties": { + "port": { + "type": ["integer", "null"], + "format": "int32", + "minimum": 0 + } + }, + "title": "Port" + }, + { + "type": "object", + "title": "Empty" + } + ] + }, + "RoomLink": { + "type": "object", + "required": ["fishjam_id", "room_id"], + "properties": { + "fishjam_id": { + "type": "string" + }, + "room_id": { + "type": "string" + } + } + }, + "RtmpInput": { + "type": "object", + "required": ["app", "stream_key"], + "properties": { + "app": { + "type": "string", + "description": "The RTMP application name.\nThis is the first path segment of the RTMP stream URL that Smelter listens on for incoming streams.\nFormat: `rtmp://://://`." + } + } + }, + "tags": [ + { + "name": "Compositions", + "description": "A composition is a single running compositing session. Create one, start it, and delete it when you are done." + }, + { + "name": "Inputs", + "description": "Live media sources being composed: WHIP, WHEP, RTMP, HLS, or MP4." + }, + { + "name": "Outputs", + "description": "Where the composed result is sent, over WHIP or RTMP. Each output carries a scene or renders a template." + }, + { + "name": "Renderers", + "description": "Shared assets, such as images and fonts, that scenes can reference." + }, + { + "name": "Events", + "description": "Custom events delivered to templates, and the WebSocket stream of engine events." + }, + { + "name": "Room forwarding", + "description": "Links a Fishjam room to the composition. Called by Fishjam automatically when a room's track forwarding is created." + }, + { + "name": "Media transport", + "description": "WHIP publishing into inputs and WHEP playback of outputs." + } + ] +} diff --git a/yarn.lock b/yarn.lock index efa7881d..b78869c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3719,6 +3719,12 @@ __metadata: languageName: node linkType: hard +"@fishjam-cloud/composition@link:./packages/js-server-sdk/packages/composition::locator=fishjam-docs%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@fishjam-cloud/composition@link:./packages/js-server-sdk/packages/composition::locator=fishjam-docs%40workspace%3A." + languageName: node + linkType: soft + "@fishjam-cloud/js-server-sdk@link:./packages/js-server-sdk/packages/js-server-sdk::locator=fishjam-docs%40workspace%3A.": version: 0.0.0-use.local resolution: "@fishjam-cloud/js-server-sdk@link:./packages/js-server-sdk/packages/js-server-sdk::locator=fishjam-docs%40workspace%3A." @@ -11931,6 +11937,7 @@ __metadata: "@docusaurus/types": "npm:^3.10.0" "@docusaurus/utils": "npm:^3.10.0" "@fastify/env": "npm:^5.0.2" + "@fishjam-cloud/composition": "link:./packages/js-server-sdk/packages/composition" "@fishjam-cloud/js-server-sdk": "link:./packages/js-server-sdk/packages/js-server-sdk" "@fishjam-cloud/react-client": "link:./packages/web-client-sdk/packages/react-client" "@fishjam-cloud/react-native-client": "link:./packages/web-client-sdk/packages/mobile-client"