import { createWriteStream } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { extensionForMime, normalizeMimeType } from "@openclaw/media-core/mime"; import type { Command } from "commander"; import { resolveAgentDir } from "../../agents/agent-scope.js"; import { assertOkOrThrowHttpError, assertProviderBinaryResponseContent, } from "../../agents/provider-http-errors.js"; import { resolveAgentModelPrimaryValue } from "../../config/model-input.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { readResponseWithLimit } from "../../infra/http-body.js"; import { buildMediaUnderstandingRegistry } from "../../media-understanding/provider-registry.js"; import { describeVideoFile } from "../../media-understanding/runtime.js"; import { resolveGeneratedMediaMaxBytes } from "../../media/configured-max-bytes.js"; import { fetchWithTimeoutGuarded, resolveProviderHttpRequestConfig, sanitizeConfiguredModelProviderRequest, } from "../../plugin-sdk/provider-http.js"; import { defaultRuntime } from "../../runtime.js"; import { generateVideo, listRuntimeVideoGenerationProviders, } from "../../video-generation/runtime.js"; import type { VideoGenerationResolution } from "../../video-generation/types.js"; import { runCommandWithRuntime } from "../cli-utils.js"; import { getModelsCommandSecretTargetIds } from "../command-secret-targets.js"; import { publishOutputFileAtomically, writeOutputAsset } from "../media-output.js"; import { prepareLocalCapabilityAccountSecrets } from "./local-account-secrets.js"; import type { CapabilityEnvelope } from "./metadata.js"; import { emitJsonOrText, formatEnvelopeForText } from "./output.js"; import { parseOptionalFiniteNumber, parseOptionalTimeoutMs, providerHasGenericConfig, registerLocalProvidersCommand, requireProviderModelOverride, resolveCapabilityAgentOption, resolveCapabilityProviderAgentId, resolveLocalCapabilityRuntimeConfig, resolveSelectedProviderFromModelRef, } from "./shared.js"; const GENERATED_VIDEO_DOWNLOAD_TIMEOUT_MS = 120_000; function normalizeVideoResolution(raw: string | undefined): VideoGenerationResolution | undefined { const normalized = raw?.trim().toUpperCase(); if (!normalized) { return undefined; } if ( normalized === "360P" || normalized === "480P" || normalized === "540P" || normalized === "720P" || normalized === "768P" || normalized === "1080P" ) { return normalized; } throw new Error("video resolution must be one of 360P, 480P, 540P, 720P, 768P, or 1080P"); } async function fetchGeneratedVideoDownload(params: { cfg: OpenClawConfig; provider: string; url: string; }) { const providerConfig = params.cfg.models?.providers?.[params.provider]; const { allowPrivateNetwork, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: params.url, defaultBaseUrl: params.url, request: sanitizeConfiguredModelProviderRequest(providerConfig?.request), provider: params.provider, capability: "video", transport: "http", }); const result = await fetchWithTimeoutGuarded( params.url, { method: "GET" }, GENERATED_VIDEO_DOWNLOAD_TIMEOUT_MS, fetch, { ...(allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}), ...(dispatcherPolicy ? { dispatcherPolicy } : {}), auditContext: `${params.provider}-generated-video-download`, }, ); try { await assertOkOrThrowHttpError( result.response, `${params.provider} generated video download failed`, ); assertProviderBinaryResponseContent( result.response, `${params.provider} generated video download`, "video", ); return result; } catch (error) { await result.release(); throw error; } } async function runVideoGenerate(params: { prompt: string; model?: string; output?: string; size?: string; aspectRatio?: string; resolution?: VideoGenerationResolution; durationSeconds?: number; audio?: boolean; watermark?: boolean; timeoutMs?: number; agent?: string; }) { requireProviderModelOverride(params.model); const cfg = await resolveLocalCapabilityRuntimeConfig({ commandName: "infer video.generate", targetIds: getModelsCommandSecretTargetIds(), }); const agentId = resolveCapabilityProviderAgentId(cfg, params.agent, "infer video.generate"); await prepareLocalCapabilityAccountSecrets({ cfg, agentId }); const agentDir = resolveAgentDir(cfg, agentId); const result = await generateVideo({ cfg, agentDir, prompt: params.prompt, modelOverride: params.model, size: params.size, aspectRatio: params.aspectRatio, resolution: params.resolution, durationSeconds: params.durationSeconds, audio: params.audio, watermark: params.watermark, timeoutMs: params.timeoutMs, }); const outputs = await Promise.all( result.videos.map(async (video, index) => { if (!video.buffer && !video.url) { throw new Error(`Video asset at index ${index} has neither buffer nor url`); } let videoBuffer = video.buffer; if (!videoBuffer && video.url) { const download = await fetchGeneratedVideoDownload({ cfg, provider: result.provider, url: video.url, }); const response = download.response; try { if (params.output && response.body) { const mimeType = normalizeMimeType(video.mimeType); const ext = extensionForMime(mimeType) || path.extname(video.fileName ?? "") || path.extname(params.output); const resolvedOutput = path.resolve(params.output); const parsed = path.parse(resolvedOutput); const filePath = result.videos.length <= 1 ? path.join(parsed.dir, `${parsed.name}${ext}`) : path.join(parsed.dir, `${parsed.name}-${String(index + 1)}${ext}`); const size = await publishOutputFileAtomically({ filePath, writeTemp: async (tempPath) => { await pipeline( Readable.fromWeb( response.body as import("node:stream/web").ReadableStream, ), createWriteStream(tempPath, { flags: "wx" }), ); const writtenSize = (await fs.stat(tempPath)).size; if (writtenSize === 0) { throw new Error("Generated media output is empty."); } return writtenSize; }, }); return { path: filePath, mimeType: video.mimeType, size }; } // Provider-supplied video URLs are untrusted external sources, and the // in-memory fallback (no --output) must not buffer an unbounded body: // generated videos routinely exceed tens of MiB and a hostile/buggy // provider could exhaust process memory. Cap the read (fail-closed: // overflow cancels the stream and throws rather than silently // truncating) using the same shared bounded reader the rest of the // media stack relies on. The --output branch above already streams // straight to disk, so only this buffered path needs the guard. The // overflow error reports only the provider label and byte cap (never // the raw URL, which may be signed/tokenized) to match the sibling // generated-media downloaders. const videoMaxBytes = resolveGeneratedMediaMaxBytes(cfg, "video"); videoBuffer = await readResponseWithLimit(response, videoMaxBytes, { onOverflow: ({ maxBytes }) => new Error( `${result.provider} generated video download exceeds ${maxBytes} bytes; pass --output to stream large videos to disk`, ), }); if (videoBuffer.byteLength === 0) { throw new Error("Generated media output is empty."); } } finally { await download.release(); } } return { ...(await writeOutputAsset({ buffer: videoBuffer!, mimeType: video.mimeType, originalFilename: video.fileName, outputPath: params.output, outputIndex: index, outputCount: result.videos.length, subdir: "generated", })), }; }), ); return { ok: true, capability: "video.generate", transport: "local" as const, provider: result.provider, model: result.model, attempts: result.attempts, outputs, } satisfies CapabilityEnvelope; } async function runVideoDescribe(params: { file: string; model?: string; agent?: string }) { const cfg = await resolveLocalCapabilityRuntimeConfig({ commandName: "infer video.describe", targetIds: getModelsCommandSecretTargetIds(), }); const agentId = resolveCapabilityProviderAgentId(cfg, params.agent, "infer video describe"); await prepareLocalCapabilityAccountSecrets({ cfg, agentId }); const agentDir = resolveAgentDir(cfg, agentId); const activeModel = requireProviderModelOverride(params.model); const result = await describeVideoFile({ filePath: path.resolve(params.file), cfg, agentId, agentDir, activeModel, }); if (!result.text) { throw new Error(`No description returned for video: ${path.resolve(params.file)}`); } return { ok: true, capability: "video.describe", transport: "local" as const, provider: result.provider, model: result.model, attempts: [], outputs: [{ path: path.resolve(params.file), text: result.text, kind: "video.description" }], } satisfies CapabilityEnvelope; } export function registerVideoCapabilityCommands(capability: Command): void { const video = capability .command("video") .description("Video generation and description") .option("--agent ", "Agent whose model and auth state should be used"); video .command("generate") .description("Generate video") .requiredOption("--prompt ", "Prompt text") .option("--model ", "Model override") .option("--size ", "Size hint like 1280x720") .option("--aspect-ratio ", "Aspect ratio hint like 16:9") .option("--resolution ", "Resolution hint: 360P, 480P, 540P, 720P, 768P, or 1080P") .option("--duration ", "Target duration in seconds") .option("--audio", "Enable generated audio when supported") .option("--watermark", "Request provider watermark when supported") .option("--timeout-ms ", "Provider request timeout in milliseconds") .option("--output ", "Output path") .option( "--agent ", "Agent whose saved provider auth is used (default: agents.defaults.systemAgent.agentId, then the sole agent)", ) .option("--json", "Output JSON", false) .action(async (opts, command) => { await runCommandWithRuntime(defaultRuntime, async () => { const result = await runVideoGenerate({ prompt: String(opts.prompt), agent: resolveCapabilityAgentOption(command, opts.agent), model: opts.model as string | undefined, output: opts.output as string | undefined, size: opts.size as string | undefined, aspectRatio: opts.aspectRatio as string | undefined, resolution: normalizeVideoResolution(opts.resolution as string | undefined), durationSeconds: parseOptionalFiniteNumber(opts.duration, "--duration"), audio: opts.audio === true ? true : undefined, watermark: opts.watermark === true ? true : undefined, timeoutMs: parseOptionalTimeoutMs(opts.timeoutMs), }); emitJsonOrText(defaultRuntime, Boolean(opts.json), result, formatEnvelopeForText); }); }); video .command("describe") .description("Describe one video file") .requiredOption("--file ", "Video file") .option("--agent ", "Agent whose model and auth state should be used") .option("--model ", "Model override") .option("--json", "Output JSON", false) .action(async (opts, command) => { await runCommandWithRuntime(defaultRuntime, async () => { const result = await runVideoDescribe({ file: String(opts.file), agent: resolveCapabilityAgentOption(command, opts.agent), model: opts.model as string | undefined, }); emitJsonOrText(defaultRuntime, Boolean(opts.json), result, formatEnvelopeForText); }); }); registerLocalProvidersCommand( video, "List video generation and description providers", (cfg, agentId) => { const selectedGenerationProvider = resolveSelectedProviderFromModelRef( resolveAgentModelPrimaryValue(cfg.agents?.defaults?.mediaModels?.video), ); return { generation: listRuntimeVideoGenerationProviders({ config: cfg }).map((provider) => ({ available: true, configured: selectedGenerationProvider === provider.id || providerHasGenericConfig({ cfg, providerId: provider.id, agentId }), selected: selectedGenerationProvider === provider.id, id: provider.id, label: provider.label, defaultModel: provider.defaultModel, models: provider.models ?? [], capabilities: provider.capabilities, })), description: [...buildMediaUnderstandingRegistry(undefined, cfg).values()] .filter((provider) => provider.capabilities?.includes("video")) .map((provider) => ({ available: true, configured: providerHasGenericConfig({ cfg, providerId: provider.id, agentId }), selected: false, id: provider.id, capabilities: provider.capabilities, defaultModels: provider.defaultModels, })), }; }, (value) => JSON.stringify(value, null, 2), ); }