From 49299add74e563652918a595561a5ec065a2974b Mon Sep 17 00:00:00 2001 From: Dev Shah Date: Tue, 9 Jun 2026 11:04:05 -0700 Subject: [PATCH 1/2] feat(integrations): add Resemble Detect + Intelligence block Four tools (detect / intelligence / watermark detect+apply) + a Resemble block. Bearer auth; async detection polls to completion. Registered in tool/block registries with a ResembleIcon. --- apps/sim/blocks/blocks/resemble.ts | 123 ++++++++++++++++++++ apps/sim/blocks/registry.ts | 2 + apps/sim/components/icons.tsx | 19 +++ apps/sim/tools/registry.ts | 10 ++ apps/sim/tools/resemble/detect.ts | 54 +++++++++ apps/sim/tools/resemble/index.ts | 5 + apps/sim/tools/resemble/intelligence.ts | 50 ++++++++ apps/sim/tools/resemble/types.ts | 36 ++++++ apps/sim/tools/resemble/utils.ts | 63 ++++++++++ apps/sim/tools/resemble/watermark_apply.ts | 48 ++++++++ apps/sim/tools/resemble/watermark_detect.ts | 32 +++++ 11 files changed, 442 insertions(+) create mode 100644 apps/sim/blocks/blocks/resemble.ts create mode 100644 apps/sim/tools/resemble/detect.ts create mode 100644 apps/sim/tools/resemble/index.ts create mode 100644 apps/sim/tools/resemble/intelligence.ts create mode 100644 apps/sim/tools/resemble/types.ts create mode 100644 apps/sim/tools/resemble/utils.ts create mode 100644 apps/sim/tools/resemble/watermark_apply.ts create mode 100644 apps/sim/tools/resemble/watermark_detect.ts diff --git a/apps/sim/blocks/blocks/resemble.ts b/apps/sim/blocks/blocks/resemble.ts new file mode 100644 index 00000000000..a6c59107a51 --- /dev/null +++ b/apps/sim/blocks/blocks/resemble.ts @@ -0,0 +1,123 @@ +import { ResembleIcon } from '@/components/icons' +import type { BlockConfig } from '@/blocks/types' +import { AuthMode, IntegrationType } from '@/blocks/types' +import type { ResembleResponse } from '@/tools/resemble/types' + +export const ResembleBlock: BlockConfig = { + type: 'resemble', + name: 'Resemble', + description: 'Deepfake detection, media intelligence, and watermarking', + longDescription: + 'Integrate Resemble AI media safety into your workflow: detect deepfakes in audio/image/video, analyze media intelligence, and apply or detect invisible watermarks.', + docsLink: 'https://docs.resemble.ai', + category: 'tools', + integrationType: IntegrationType.Security, + tags: ['deepfake-detection', 'media-safety'], + bgColor: '#2E1AC4', + icon: ResembleIcon, + authMode: AuthMode.ApiKey, + + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + options: [ + { label: 'Deepfake Detection', id: 'resemble_detect' }, + { label: 'Media Intelligence', id: 'resemble_intelligence' }, + { label: 'Detect Watermark', id: 'resemble_watermark_detect' }, + { label: 'Apply Watermark', id: 'resemble_watermark_apply' }, + ], + value: () => 'resemble_detect', + }, + { + id: 'url', + title: 'Media URL', + type: 'short-input', + placeholder: 'https://example.com/media.mp4', + required: true, + }, + // Detection toggles + { id: 'runIntelligence', title: 'Run Intelligence', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { id: 'audioSourceTracing', title: 'Audio Source Tracing', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { id: 'visualize', title: 'Visualize', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { id: 'useReverseSearch', title: 'Reverse Image Search', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { id: 'useOodDetector', title: 'OOD Detector', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { id: 'zeroRetentionMode', title: 'Zero-Retention Mode', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { + id: 'modelTypes', + title: 'Model Type', + type: 'dropdown', + options: [ + { label: 'Auto', id: 'auto' }, + { label: 'Image', id: 'image' }, + { label: 'Talking Head', id: 'talking_head' }, + ], + value: () => 'auto', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + // Intelligence options + { + id: 'mediaType', + title: 'Media Type', + type: 'dropdown', + options: [ + { label: 'Auto', id: 'auto' }, + { label: 'Audio', id: 'audio' }, + { label: 'Video', id: 'video' }, + { label: 'Image', id: 'image' }, + ], + value: () => 'auto', + condition: { field: 'operation', value: 'resemble_intelligence' }, + }, + // Apply-watermark options + { id: 'strength', title: 'Strength (0–1)', type: 'short-input', placeholder: '0.2', condition: { field: 'operation', value: 'resemble_watermark_apply' } }, + { id: 'customMessage', title: 'Custom Message', type: 'short-input', placeholder: 'resembleai', condition: { field: 'operation', value: 'resemble_watermark_apply' } }, + { + id: 'apiKey', + title: 'API Key', + type: 'short-input', + placeholder: 'Enter your Resemble API key', + required: true, + password: true, + }, + ], + + tools: { + access: ['resemble_detect', 'resemble_intelligence', 'resemble_watermark_detect', 'resemble_watermark_apply'], + config: { + tool: (params) => { + switch (params.operation) { + case 'resemble_intelligence': + return 'resemble_intelligence' + case 'resemble_watermark_detect': + return 'resemble_watermark_detect' + case 'resemble_watermark_apply': + return 'resemble_watermark_apply' + default: + return 'resemble_detect' + } + }, + }, + }, + + inputs: { + operation: { type: 'string', description: 'Operation to perform' }, + url: { type: 'string', description: 'Public HTTPS URL to the media' }, + runIntelligence: { type: 'boolean', description: 'Also run media intelligence' }, + audioSourceTracing: { type: 'boolean', description: 'Trace the source platform of fake audio' }, + visualize: { type: 'boolean', description: 'Generate heatmap artifacts' }, + useReverseSearch: { type: 'boolean', description: 'Image-only reverse image search' }, + useOodDetector: { type: 'boolean', description: 'Out-of-distribution detection' }, + zeroRetentionMode: { type: 'boolean', description: 'Auto-delete media after analysis' }, + modelTypes: { type: 'string', description: 'auto | image | talking_head' }, + mediaType: { type: 'string', description: 'auto | audio | video | image' }, + strength: { type: 'number', description: 'Watermark strength 0–1' }, + customMessage: { type: 'string', description: 'Watermark message' }, + apiKey: { type: 'string', description: 'Resemble API key' }, + }, + + outputs: { + result: { type: 'json', description: 'Result from the selected Resemble operation' }, + }, +} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 10371ed1646..415315bd0ae 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -189,6 +189,7 @@ import { SearchBlock } from '@/blocks/blocks/search' import { SecretsManagerBlock } from '@/blocks/blocks/secrets_manager' import { SendGridBlock } from '@/blocks/blocks/sendgrid' import { SentryBlock } from '@/blocks/blocks/sentry' +import { ResembleBlock } from '@/blocks/blocks/resemble' import { SerperBlock } from '@/blocks/blocks/serper' import { ServiceNowBlock } from '@/blocks/blocks/servicenow' import { SESBlock } from '@/blocks/blocks/ses' @@ -461,6 +462,7 @@ export const registry: Record = { search: SearchBlock, sendgrid: SendGridBlock, sentry: SentryBlock, + resemble: ResembleBlock, serper: SerperBlock, servicenow: ServiceNowBlock, sftp: SftpBlock, diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index dce91bf9720..637ddbb9c47 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -7391,3 +7391,22 @@ export function WizaIcon(props: SVGProps) { ) } + +export function ResembleIcon(props: SVGProps) { + return ( + + + + + + + ) +} diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 83ef9e9ebcd..34385eb99b0 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -2664,6 +2664,12 @@ import { updateIssueTool, updateProjectTool, } from '@/tools/sentry' +import { + detectTool as resembleDetectTool, + intelligenceTool as resembleIntelligenceTool, + watermarkApplyTool as resembleWatermarkApplyTool, + watermarkDetectTool as resembleWatermarkDetectTool, +} from '@/tools/resemble' import { serperSearchTool } from '@/tools/serper' import { servicenowCreateRecordTool, @@ -3843,6 +3849,10 @@ export const tools: Record = { github_repo_info_v2: githubRepoInfoV2Tool, github_latest_commit: githubLatestCommitTool, github_latest_commit_v2: githubLatestCommitV2Tool, + resemble_detect: resembleDetectTool, + resemble_intelligence: resembleIntelligenceTool, + resemble_watermark_detect: resembleWatermarkDetectTool, + resemble_watermark_apply: resembleWatermarkApplyTool, serper_search: serperSearchTool, similarweb_website_overview: similarwebWebsiteOverviewTool, similarweb_traffic_visits: similarwebTrafficVisitsTool, diff --git a/apps/sim/tools/resemble/detect.ts b/apps/sim/tools/resemble/detect.ts new file mode 100644 index 00000000000..13ad7dcdde5 --- /dev/null +++ b/apps/sim/tools/resemble/detect.ts @@ -0,0 +1,54 @@ +import type { ResembleDetectParams, ResembleResponse } from '@/tools/resemble/types' +import { authHeaders, baseOf, pollResource, rItem, sanitize } from '@/tools/resemble/utils' +import type { ToolConfig } from '@/tools/types' + +export const detectTool: ToolConfig = { + id: 'resemble_detect', + name: 'Resemble Deepfake Detection', + description: 'Detect whether media (audio, image, or video) is a deepfake / AI-generated.', + version: '1.0.0', + params: { + apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, + url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, + runIntelligence: { type: 'boolean', required: false, visibility: 'user-only', description: 'Also run media intelligence' }, + audioSourceTracing: { type: 'boolean', required: false, visibility: 'user-only', description: 'Trace the source platform of fake audio' }, + visualize: { type: 'boolean', required: false, visibility: 'user-only', description: 'Generate heatmap artifacts' }, + useReverseSearch: { type: 'boolean', required: false, visibility: 'user-only', description: 'Image-only reverse image search' }, + useOodDetector: { type: 'boolean', required: false, visibility: 'user-only', description: 'Out-of-distribution detection' }, + zeroRetentionMode: { type: 'boolean', required: false, visibility: 'user-only', description: 'Auto-delete media after analysis' }, + modelTypes: { type: 'string', required: false, visibility: 'user-only', description: 'auto | image | talking_head' }, + maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll for the result' }, + baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + }, + request: { + url: (p) => `${baseOf(p)}/detect`, + method: 'POST', + headers: (p) => authHeaders(p), + body: (p) => { + const b: Record = { url: p.url } + if (p.runIntelligence) b.intelligence = true + if (p.audioSourceTracing) b.audio_source_tracing = true + if (p.visualize) b.visualize = true + if (p.useReverseSearch) b.use_reverse_search = true + if (p.useOodDetector) b.use_ood_detector = true + if (p.zeroRetentionMode) b.zero_retention_mode = true + if (p.modelTypes && p.modelTypes !== 'auto') b.model_types = p.modelTypes + return b + }, + }, + transformResponse: async (response: Response, params?: ResembleDetectParams) => { + let data: any + try { + data = await response.json() + } catch { + data = { raw: await response.text() } + } + if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + const uuid = rItem(data).uuid + if (uuid && params) { + data = await pollResource(baseOf(params), `/detect/${uuid}`, authHeaders(params), params.maxWaitSeconds || 120) + } + return { success: true, output: { result: sanitize(data) } } + }, + outputs: { result: { type: 'json', description: 'Detection result (label, score, metrics, optional intelligence).' } }, +} diff --git a/apps/sim/tools/resemble/index.ts b/apps/sim/tools/resemble/index.ts new file mode 100644 index 00000000000..4129c51bfd9 --- /dev/null +++ b/apps/sim/tools/resemble/index.ts @@ -0,0 +1,5 @@ +export { detectTool } from '@/tools/resemble/detect' +export { intelligenceTool } from '@/tools/resemble/intelligence' +export { watermarkDetectTool } from '@/tools/resemble/watermark_detect' +export { watermarkApplyTool } from '@/tools/resemble/watermark_apply' +export * from '@/tools/resemble/types' diff --git a/apps/sim/tools/resemble/intelligence.ts b/apps/sim/tools/resemble/intelligence.ts new file mode 100644 index 00000000000..1b721969e73 --- /dev/null +++ b/apps/sim/tools/resemble/intelligence.ts @@ -0,0 +1,50 @@ +import type { ResembleIntelligenceParams, ResembleResponse } from '@/tools/resemble/types' +import { authHeaders, baseOf, pollResource, rItem, sanitize } from '@/tools/resemble/utils' +import type { ToolConfig } from '@/tools/types' + +const TERMINAL = new Set(['completed', 'failed', 'error', 'cancelled', 'success']) + +export const intelligenceTool: ToolConfig = { + id: 'resemble_intelligence', + name: 'Resemble Media Intelligence', + description: 'Analyze media for transcription, translation, speaker info, emotion, and misinformation.', + version: '1.0.0', + params: { + apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, + url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, + structuredJson: { type: 'boolean', required: false, visibility: 'user-only', description: 'Return structured JSON fields' }, + mediaType: { type: 'string', required: false, visibility: 'user-only', description: 'auto | audio | video | image' }, + maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll' }, + baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + }, + request: { + url: (p) => `${baseOf(p)}/intelligence`, + method: 'POST', + headers: (p) => authHeaders(p), + body: (p) => { + const b: Record = { url: p.url, json: p.structuredJson !== false } + if (p.mediaType && p.mediaType !== 'auto') b.media_type = p.mediaType + return b + }, + }, + transformResponse: async (response: Response, params?: ResembleIntelligenceParams) => { + let data: any + try { + data = await response.json() + } catch { + data = { raw: await response.text() } + } + if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + const it = rItem(data) + const status = (it.status || '').toString().toLowerCase() + if (it.uuid && status && !TERMINAL.has(status) && params) { + try { + data = await pollResource(baseOf(params), `/intelligence/${it.uuid}`, authHeaders(params), params.maxWaitSeconds || 120) + } catch { + /* poll path may vary; return submit payload */ + } + } + return { success: true, output: { result: sanitize(data) } } + }, + outputs: { result: { type: 'json', description: 'Structured intelligence analysis.' } }, +} diff --git a/apps/sim/tools/resemble/types.ts b/apps/sim/tools/resemble/types.ts new file mode 100644 index 00000000000..b495755446f --- /dev/null +++ b/apps/sim/tools/resemble/types.ts @@ -0,0 +1,36 @@ +import type { ToolResponse } from '@/tools/types' + +export interface ResembleBaseParams { + apiKey: string + baseUrl?: string + maxWaitSeconds?: number +} + +export interface ResembleDetectParams extends ResembleBaseParams { + url: string + runIntelligence?: boolean + audioSourceTracing?: boolean + visualize?: boolean + useReverseSearch?: boolean + useOodDetector?: boolean + zeroRetentionMode?: boolean + modelTypes?: string +} + +export interface ResembleIntelligenceParams extends ResembleBaseParams { + url: string + structuredJson?: boolean + mediaType?: string +} + +export interface ResembleWatermarkParams extends ResembleBaseParams { + url: string + strength?: number + customMessage?: string +} + +export interface ResembleResponse extends ToolResponse { + output: { + result: any + } +} diff --git a/apps/sim/tools/resemble/utils.ts b/apps/sim/tools/resemble/utils.ts new file mode 100644 index 00000000000..a26cb64ea9f --- /dev/null +++ b/apps/sim/tools/resemble/utils.ts @@ -0,0 +1,63 @@ +export const DEFAULT_BASE_URL = 'https://app.resemble.ai/api/v2' +const TERMINAL = new Set(['completed', 'failed', 'error', 'cancelled', 'success']) + +export function baseOf(params: any): string { + return (params?.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, '') +} + +export function authHeaders(params: any, extra?: Record): Record { + return { + Authorization: `Bearer ${params?.apiKey}`, + 'Content-Type': 'application/json', + Accept: 'application/json', + ...(extra || {}), + } +} + +export function rItem(d: any): any { + return d && typeof d === 'object' && d.item && typeof d.item === 'object' ? d.item : d || {} +} + +export function sanitize(d: any, n = 200): any { + if (Array.isArray(d)) return d.map((x) => sanitize(x, n)) + if (d && typeof d === 'object') { + const o: any = {} + for (const k of Object.keys(d)) o[k] = sanitize(d[k], n) + return o + } + if (typeof d === 'string' && d.startsWith('data:') && d.length > n) { + return `` + } + return d +} + +export async function getJson(url: string, headers: Record): Promise { + const r = await fetch(url, { headers }) + let j: any + try { + j = await r.json() + } catch { + j = { raw: await r.text() } + } + if (r.status >= 400) throw new Error((j && j.message) || `Resemble API error: HTTP ${r.status}`) + return j +} + +export async function pollResource( + base: string, + path: string, + headers: Record, + maxWaitSeconds = 120 +): Promise { + const deadline = Date.now() + Math.max(1, maxWaitSeconds) * 1000 + let delay = 2000 + let last = await getJson(`${base}${path}`, headers) + while (true) { + const s = (rItem(last).status || '').toString().toLowerCase() + if (!s || TERMINAL.has(s)) return last + if (Date.now() >= deadline) return last + await new Promise((r) => setTimeout(r, delay)) + delay = Math.min(10000, delay + 1000) + last = await getJson(`${base}${path}`, headers) + } +} diff --git a/apps/sim/tools/resemble/watermark_apply.ts b/apps/sim/tools/resemble/watermark_apply.ts new file mode 100644 index 00000000000..db36032fa61 --- /dev/null +++ b/apps/sim/tools/resemble/watermark_apply.ts @@ -0,0 +1,48 @@ +import type { ResembleResponse, ResembleWatermarkParams } from '@/tools/resemble/types' +import { authHeaders, baseOf, pollResource, rItem, sanitize } from '@/tools/resemble/utils' +import type { ToolConfig } from '@/tools/types' + +export const watermarkApplyTool: ToolConfig = { + id: 'resemble_watermark_apply', + name: 'Resemble Apply Watermark', + description: 'Apply an invisible Resemble provenance watermark and return the watermarked media (audio-first).', + version: '1.0.0', + params: { + apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, + url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, + strength: { type: 'number', required: false, visibility: 'user-only', description: 'Watermark strength 0.0–1.0 (image/video only)' }, + customMessage: { type: 'string', required: false, visibility: 'user-only', description: 'Message to embed (image/video only)' }, + maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll' }, + baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + }, + request: { + url: (p) => `${baseOf(p)}/watermark/apply`, + method: 'POST', + headers: (p) => authHeaders(p, { Prefer: 'wait' }), + body: (p) => { + const b: Record = { url: p.url } + if (p.strength != null) b.strength = Number(p.strength) + if (p.customMessage) b.custom_message = p.customMessage + return b + }, + }, + transformResponse: async (response: Response, params?: ResembleWatermarkParams) => { + let data: any + try { + data = await response.json() + } catch { + data = { raw: await response.text() } + } + if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + const it = rItem(data) + if (!(it.watermarked_media || it.url) && it.uuid && params) { + try { + data = await pollResource(baseOf(params), `/watermark/apply/${it.uuid}/result`, authHeaders(params), params.maxWaitSeconds || 120) + } catch { + /* fall through */ + } + } + return { success: true, output: { result: sanitize(data) } } + }, + outputs: { result: { type: 'json', description: 'Watermarked media result.' } }, +} diff --git a/apps/sim/tools/resemble/watermark_detect.ts b/apps/sim/tools/resemble/watermark_detect.ts new file mode 100644 index 00000000000..19f47a19dfa --- /dev/null +++ b/apps/sim/tools/resemble/watermark_detect.ts @@ -0,0 +1,32 @@ +import type { ResembleResponse, ResembleWatermarkParams } from '@/tools/resemble/types' +import { authHeaders, baseOf, sanitize } from '@/tools/resemble/utils' +import type { ToolConfig } from '@/tools/types' + +export const watermarkDetectTool: ToolConfig = { + id: 'resemble_watermark_detect', + name: 'Resemble Detect Watermark', + description: 'Check whether media contains a Resemble watermark.', + version: '1.0.0', + params: { + apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, + url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, + baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + }, + request: { + url: (p) => `${baseOf(p)}/watermark/detect`, + method: 'POST', + headers: (p) => authHeaders(p, { Prefer: 'wait' }), + body: (p) => ({ url: p.url }), + }, + transformResponse: async (response: Response) => { + let data: any + try { + data = await response.json() + } catch { + data = { raw: await response.text() } + } + if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + return { success: true, output: { result: sanitize(data) } } + }, + outputs: { result: { type: 'json', description: 'Watermark detection result.' } }, +} From 3f843c55da5a4c0e0bdcc7ca6c68d973230879f6 Mon Sep 17 00:00:00 2001 From: Dev Shah Date: Thu, 11 Jun 2026 13:38:55 -0700 Subject: [PATCH 2/2] fix(resemble): address review feedback on polling and block UI - pollResource now throws when a job does not finish within maxWaitSeconds instead of returning a non-terminal payload as success - completion is decided by a per-resource predicate: watermark apply results have no status field, so done = watermarked_media/url present; a missing status no longer ends polling early - intelligence polls the documented GET /intelligences/{uuid} route - poll errors propagate instead of being swallowed (intelligence, watermark apply) - TERMINAL is exported from utils and reused, removing the duplicate set - add Structured JSON switch to the Intelligence operation UI - read response bodies once via text() to avoid double body consumption - drop BlockConfig.tags after upstream BlockMeta refactor --- apps/sim/blocks/blocks/resemble.ts | 74 ++++++++++++--- apps/sim/tools/registry.ts | 12 +-- apps/sim/tools/resemble/detect.ts | 99 +++++++++++++++++---- apps/sim/tools/resemble/index.ts | 4 +- apps/sim/tools/resemble/intelligence.ts | 75 ++++++++++++---- apps/sim/tools/resemble/utils.ts | 24 +++-- apps/sim/tools/resemble/watermark_apply.ts | 69 ++++++++++---- apps/sim/tools/resemble/watermark_detect.ts | 29 ++++-- 8 files changed, 305 insertions(+), 81 deletions(-) diff --git a/apps/sim/blocks/blocks/resemble.ts b/apps/sim/blocks/blocks/resemble.ts index a6c59107a51..a1f206575ca 100644 --- a/apps/sim/blocks/blocks/resemble.ts +++ b/apps/sim/blocks/blocks/resemble.ts @@ -12,7 +12,6 @@ export const ResembleBlock: BlockConfig = { docsLink: 'https://docs.resemble.ai', category: 'tools', integrationType: IntegrationType.Security, - tags: ['deepfake-detection', 'media-safety'], bgColor: '#2E1AC4', icon: ResembleIcon, authMode: AuthMode.ApiKey, @@ -38,12 +37,42 @@ export const ResembleBlock: BlockConfig = { required: true, }, // Detection toggles - { id: 'runIntelligence', title: 'Run Intelligence', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, - { id: 'audioSourceTracing', title: 'Audio Source Tracing', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, - { id: 'visualize', title: 'Visualize', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, - { id: 'useReverseSearch', title: 'Reverse Image Search', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, - { id: 'useOodDetector', title: 'OOD Detector', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, - { id: 'zeroRetentionMode', title: 'Zero-Retention Mode', type: 'switch', condition: { field: 'operation', value: 'resemble_detect' } }, + { + id: 'runIntelligence', + title: 'Run Intelligence', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + { + id: 'audioSourceTracing', + title: 'Audio Source Tracing', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + { + id: 'visualize', + title: 'Visualize', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + { + id: 'useReverseSearch', + title: 'Reverse Image Search', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + { + id: 'useOodDetector', + title: 'OOD Detector', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, + { + id: 'zeroRetentionMode', + title: 'Zero-Retention Mode', + type: 'switch', + condition: { field: 'operation', value: 'resemble_detect' }, + }, { id: 'modelTypes', title: 'Model Type', @@ -57,6 +86,13 @@ export const ResembleBlock: BlockConfig = { condition: { field: 'operation', value: 'resemble_detect' }, }, // Intelligence options + { + id: 'structuredJson', + title: 'Structured JSON', + type: 'switch', + defaultValue: true, + condition: { field: 'operation', value: 'resemble_intelligence' }, + }, { id: 'mediaType', title: 'Media Type', @@ -71,8 +107,20 @@ export const ResembleBlock: BlockConfig = { condition: { field: 'operation', value: 'resemble_intelligence' }, }, // Apply-watermark options - { id: 'strength', title: 'Strength (0–1)', type: 'short-input', placeholder: '0.2', condition: { field: 'operation', value: 'resemble_watermark_apply' } }, - { id: 'customMessage', title: 'Custom Message', type: 'short-input', placeholder: 'resembleai', condition: { field: 'operation', value: 'resemble_watermark_apply' } }, + { + id: 'strength', + title: 'Strength (0–1)', + type: 'short-input', + placeholder: '0.2', + condition: { field: 'operation', value: 'resemble_watermark_apply' }, + }, + { + id: 'customMessage', + title: 'Custom Message', + type: 'short-input', + placeholder: 'resembleai', + condition: { field: 'operation', value: 'resemble_watermark_apply' }, + }, { id: 'apiKey', title: 'API Key', @@ -84,7 +132,12 @@ export const ResembleBlock: BlockConfig = { ], tools: { - access: ['resemble_detect', 'resemble_intelligence', 'resemble_watermark_detect', 'resemble_watermark_apply'], + access: [ + 'resemble_detect', + 'resemble_intelligence', + 'resemble_watermark_detect', + 'resemble_watermark_apply', + ], config: { tool: (params) => { switch (params.operation) { @@ -111,6 +164,7 @@ export const ResembleBlock: BlockConfig = { useOodDetector: { type: 'boolean', description: 'Out-of-distribution detection' }, zeroRetentionMode: { type: 'boolean', description: 'Auto-delete media after analysis' }, modelTypes: { type: 'string', description: 'auto | image | talking_head' }, + structuredJson: { type: 'boolean', description: 'Return structured JSON fields' }, mediaType: { type: 'string', description: 'auto | audio | video | image' }, strength: { type: 'number', description: 'Watermark strength 0–1' }, customMessage: { type: 'string', description: 'Watermark message' }, diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index d972b48de28..5c0b6a2c697 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -2435,6 +2435,12 @@ import { redisTtlTool, } from '@/tools/redis' import { reductoParserTool, reductoParserV2Tool } from '@/tools/reducto' +import { + detectTool as resembleDetectTool, + intelligenceTool as resembleIntelligenceTool, + watermarkApplyTool as resembleWatermarkApplyTool, + watermarkDetectTool as resembleWatermarkDetectTool, +} from '@/tools/resemble' import { resendCreateContactTool, resendDeleteContactTool, @@ -2792,12 +2798,6 @@ import { updateIssueTool, updateProjectTool, } from '@/tools/sentry' -import { - detectTool as resembleDetectTool, - intelligenceTool as resembleIntelligenceTool, - watermarkApplyTool as resembleWatermarkApplyTool, - watermarkDetectTool as resembleWatermarkDetectTool, -} from '@/tools/resemble' import { serperSearchTool } from '@/tools/serper' import { servicenowAggregateTool, diff --git a/apps/sim/tools/resemble/detect.ts b/apps/sim/tools/resemble/detect.ts index 13ad7dcdde5..9e23676cc8a 100644 --- a/apps/sim/tools/resemble/detect.ts +++ b/apps/sim/tools/resemble/detect.ts @@ -8,17 +8,72 @@ export const detectTool: ToolConfig = { description: 'Detect whether media (audio, image, or video) is a deepfake / AI-generated.', version: '1.0.0', params: { - apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, - url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, - runIntelligence: { type: 'boolean', required: false, visibility: 'user-only', description: 'Also run media intelligence' }, - audioSourceTracing: { type: 'boolean', required: false, visibility: 'user-only', description: 'Trace the source platform of fake audio' }, - visualize: { type: 'boolean', required: false, visibility: 'user-only', description: 'Generate heatmap artifacts' }, - useReverseSearch: { type: 'boolean', required: false, visibility: 'user-only', description: 'Image-only reverse image search' }, - useOodDetector: { type: 'boolean', required: false, visibility: 'user-only', description: 'Out-of-distribution detection' }, - zeroRetentionMode: { type: 'boolean', required: false, visibility: 'user-only', description: 'Auto-delete media after analysis' }, - modelTypes: { type: 'string', required: false, visibility: 'user-only', description: 'auto | image | talking_head' }, - maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll for the result' }, - baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Resemble API key', + }, + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Public HTTPS URL to the media', + }, + runIntelligence: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Also run media intelligence', + }, + audioSourceTracing: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Trace the source platform of fake audio', + }, + visualize: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Generate heatmap artifacts', + }, + useReverseSearch: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Image-only reverse image search', + }, + useOodDetector: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Out-of-distribution detection', + }, + zeroRetentionMode: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Auto-delete media after analysis', + }, + modelTypes: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'auto | image | talking_head', + }, + maxWaitSeconds: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Max seconds to poll for the result', + }, + baseUrl: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'API base URL override', + }, }, request: { url: (p) => `${baseOf(p)}/detect`, @@ -37,18 +92,30 @@ export const detectTool: ToolConfig = { }, }, transformResponse: async (response: Response, params?: ResembleDetectParams) => { + const text = await response.text() let data: any try { - data = await response.json() + data = JSON.parse(text) } catch { - data = { raw: await response.text() } + data = { raw: text } } - if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + if (!response.ok) + throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) const uuid = rItem(data).uuid if (uuid && params) { - data = await pollResource(baseOf(params), `/detect/${uuid}`, authHeaders(params), params.maxWaitSeconds || 120) + data = await pollResource( + baseOf(params), + `/detect/${uuid}`, + authHeaders(params), + params.maxWaitSeconds || 120 + ) } return { success: true, output: { result: sanitize(data) } } }, - outputs: { result: { type: 'json', description: 'Detection result (label, score, metrics, optional intelligence).' } }, + outputs: { + result: { + type: 'json', + description: 'Detection result (label, score, metrics, optional intelligence).', + }, + }, } diff --git a/apps/sim/tools/resemble/index.ts b/apps/sim/tools/resemble/index.ts index 4129c51bfd9..3b163ece905 100644 --- a/apps/sim/tools/resemble/index.ts +++ b/apps/sim/tools/resemble/index.ts @@ -1,5 +1,5 @@ export { detectTool } from '@/tools/resemble/detect' export { intelligenceTool } from '@/tools/resemble/intelligence' -export { watermarkDetectTool } from '@/tools/resemble/watermark_detect' -export { watermarkApplyTool } from '@/tools/resemble/watermark_apply' export * from '@/tools/resemble/types' +export { watermarkApplyTool } from '@/tools/resemble/watermark_apply' +export { watermarkDetectTool } from '@/tools/resemble/watermark_detect' diff --git a/apps/sim/tools/resemble/intelligence.ts b/apps/sim/tools/resemble/intelligence.ts index 1b721969e73..5302679366b 100644 --- a/apps/sim/tools/resemble/intelligence.ts +++ b/apps/sim/tools/resemble/intelligence.ts @@ -1,21 +1,57 @@ import type { ResembleIntelligenceParams, ResembleResponse } from '@/tools/resemble/types' -import { authHeaders, baseOf, pollResource, rItem, sanitize } from '@/tools/resemble/utils' +import { + authHeaders, + baseOf, + pollResource, + rItem, + sanitize, + TERMINAL, +} from '@/tools/resemble/utils' import type { ToolConfig } from '@/tools/types' -const TERMINAL = new Set(['completed', 'failed', 'error', 'cancelled', 'success']) - export const intelligenceTool: ToolConfig = { id: 'resemble_intelligence', name: 'Resemble Media Intelligence', - description: 'Analyze media for transcription, translation, speaker info, emotion, and misinformation.', + description: + 'Analyze media for transcription, translation, speaker info, emotion, and misinformation.', version: '1.0.0', params: { - apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, - url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, - structuredJson: { type: 'boolean', required: false, visibility: 'user-only', description: 'Return structured JSON fields' }, - mediaType: { type: 'string', required: false, visibility: 'user-only', description: 'auto | audio | video | image' }, - maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll' }, - baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Resemble API key', + }, + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Public HTTPS URL to the media', + }, + structuredJson: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Return structured JSON fields', + }, + mediaType: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'auto | audio | video | image', + }, + maxWaitSeconds: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Max seconds to poll', + }, + baseUrl: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'API base URL override', + }, }, request: { url: (p) => `${baseOf(p)}/intelligence`, @@ -28,21 +64,24 @@ export const intelligenceTool: ToolConfig { + const text = await response.text() let data: any try { - data = await response.json() + data = JSON.parse(text) } catch { - data = { raw: await response.text() } + data = { raw: text } } - if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + if (!response.ok) + throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) const it = rItem(data) const status = (it.status || '').toString().toLowerCase() if (it.uuid && status && !TERMINAL.has(status) && params) { - try { - data = await pollResource(baseOf(params), `/intelligence/${it.uuid}`, authHeaders(params), params.maxWaitSeconds || 120) - } catch { - /* poll path may vary; return submit payload */ - } + data = await pollResource( + baseOf(params), + `/intelligences/${it.uuid}`, + authHeaders(params), + params.maxWaitSeconds || 120 + ) } return { success: true, output: { result: sanitize(data) } } }, diff --git a/apps/sim/tools/resemble/utils.ts b/apps/sim/tools/resemble/utils.ts index a26cb64ea9f..f0d89aeb214 100644 --- a/apps/sim/tools/resemble/utils.ts +++ b/apps/sim/tools/resemble/utils.ts @@ -1,5 +1,5 @@ export const DEFAULT_BASE_URL = 'https://app.resemble.ai/api/v2' -const TERMINAL = new Set(['completed', 'failed', 'error', 'cancelled', 'success']) +export const TERMINAL = new Set(['completed', 'failed', 'error', 'cancelled', 'success']) export function baseOf(params: any): string { return (params?.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, '') @@ -33,29 +33,37 @@ export function sanitize(d: any, n = 200): any { export async function getJson(url: string, headers: Record): Promise { const r = await fetch(url, { headers }) + const text = await r.text() let j: any try { - j = await r.json() + j = JSON.parse(text) } catch { - j = { raw: await r.text() } + j = { raw: text } } if (r.status >= 400) throw new Error((j && j.message) || `Resemble API error: HTTP ${r.status}`) return j } +export function statusDone(d: any): boolean { + return TERMINAL.has((rItem(d).status || '').toString().toLowerCase()) +} + export async function pollResource( base: string, path: string, headers: Record, - maxWaitSeconds = 120 + maxWaitSeconds = 120, + isDone: (d: any) => boolean = statusDone ): Promise { - const deadline = Date.now() + Math.max(1, maxWaitSeconds) * 1000 + const wait = Math.max(1, maxWaitSeconds) + const deadline = Date.now() + wait * 1000 let delay = 2000 let last = await getJson(`${base}${path}`, headers) while (true) { - const s = (rItem(last).status || '').toString().toLowerCase() - if (!s || TERMINAL.has(s)) return last - if (Date.now() >= deadline) return last + if (isDone(last)) return last + if (Date.now() >= deadline) { + throw new Error(`Resemble job did not complete within ${wait}s (GET ${path})`) + } await new Promise((r) => setTimeout(r, delay)) delay = Math.min(10000, delay + 1000) last = await getJson(`${base}${path}`, headers) diff --git a/apps/sim/tools/resemble/watermark_apply.ts b/apps/sim/tools/resemble/watermark_apply.ts index db36032fa61..c3b67f8c582 100644 --- a/apps/sim/tools/resemble/watermark_apply.ts +++ b/apps/sim/tools/resemble/watermark_apply.ts @@ -5,15 +5,46 @@ import type { ToolConfig } from '@/tools/types' export const watermarkApplyTool: ToolConfig = { id: 'resemble_watermark_apply', name: 'Resemble Apply Watermark', - description: 'Apply an invisible Resemble provenance watermark and return the watermarked media (audio-first).', + description: + 'Apply an invisible Resemble provenance watermark and return the watermarked media (audio-first).', version: '1.0.0', params: { - apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Resemble API key' }, - url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Public HTTPS URL to the media' }, - strength: { type: 'number', required: false, visibility: 'user-only', description: 'Watermark strength 0.0–1.0 (image/video only)' }, - customMessage: { type: 'string', required: false, visibility: 'user-only', description: 'Message to embed (image/video only)' }, - maxWaitSeconds: { type: 'number', required: false, visibility: 'user-only', description: 'Max seconds to poll' }, - baseUrl: { type: 'string', required: false, visibility: 'user-only', description: 'API base URL override' }, + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Resemble API key', + }, + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Public HTTPS URL to the media', + }, + strength: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Watermark strength 0.0–1.0 (image/video only)', + }, + customMessage: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Message to embed (image/video only)', + }, + maxWaitSeconds: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Max seconds to poll', + }, + baseUrl: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'API base URL override', + }, }, request: { url: (p) => `${baseOf(p)}/watermark/apply`, @@ -27,20 +58,28 @@ export const watermarkApplyTool: ToolConfig { + const text = await response.text() let data: any try { - data = await response.json() + data = JSON.parse(text) } catch { - data = { raw: await response.text() } + data = { raw: text } } - if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + if (!response.ok) + throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) const it = rItem(data) if (!(it.watermarked_media || it.url) && it.uuid && params) { - try { - data = await pollResource(baseOf(params), `/watermark/apply/${it.uuid}/result`, authHeaders(params), params.maxWaitSeconds || 120) - } catch { - /* fall through */ - } + // The apply result has no `status` field — done means the media URL is present. + data = await pollResource( + baseOf(params), + `/watermark/apply/${it.uuid}/result`, + authHeaders(params), + params.maxWaitSeconds || 120, + (d) => { + const r = rItem(d) + return !!(r.watermarked_media || r.url) + } + ) } return { success: true, output: { result: sanitize(data) } } }, diff --git a/apps/sim/tools/resemble/watermark_detect.ts b/apps/sim/tools/resemble/watermark_detect.ts index 19f47a19dfa..ca7a5a2767a 100644 --- a/apps/sim/tools/resemble/watermark_detect.ts +++ b/apps/sim/tools/resemble/watermark_detect.ts @@ -8,9 +8,24 @@ export const watermarkDetectTool: ToolConfig `${baseOf(p)}/watermark/detect`, @@ -19,13 +34,15 @@ export const watermarkDetectTool: ToolConfig ({ url: p.url }), }, transformResponse: async (response: Response) => { + const text = await response.text() let data: any try { - data = await response.json() + data = JSON.parse(text) } catch { - data = { raw: await response.text() } + data = { raw: text } } - if (!response.ok) throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) + if (!response.ok) + throw new Error((data && data.message) || `Resemble API error: HTTP ${response.status}`) return { success: true, output: { result: sanitize(data) } } }, outputs: { result: { type: 'json', description: 'Watermark detection result.' } },