sushilideaclan01's picture
change the
c4f9173
Raw
History Blame Contribute Delete
34.5 kB
import type { ClipMetadata, SegmentsPayload, StreamEvent, VeoSegment } from '@/types';
/**
* In dev, use same-origin `/api` so the Vite proxy handles SSE (EventSource) without CORS.
* Ngrok and other tunnels often omit Access-Control-Allow-Origin on streamed responses.
* Set `VITE_PUBLIC_API_IN_DEV=true` only if your API sends proper CORS for localhost.
* Production: set `VITE_API_BASE_URL` to your deployed API origin.
*/
const API_BASE =
import.meta.env.DEV && import.meta.env.VITE_PUBLIC_API_IN_DEV !== 'true'
? ''
: (import.meta.env.VITE_API_BASE_URL || '');
/**
* Hero preview in the UI: prefer the storefront CDN URL when we have it (always a real image).
* Otherwise use blob/data URLs as-is. For hosted /api/images/... URLs, use a path-only URL in dev
* so the Vite proxy loads pixels (ngrok-free and some tunnels return HTML for absolute image GETs).
*/
export function imageSrcForHeroPreview(
heroRemoteUrl: string | null,
imagePreview: string | null
): string | null {
if (heroRemoteUrl) return heroRemoteUrl;
if (!imagePreview) return null;
if (imagePreview.startsWith('blob:') || imagePreview.startsWith('data:')) return imagePreview;
if (imagePreview.includes('/api/images/') && import.meta.env.DEV) {
try {
return new URL(imagePreview).pathname;
} catch {
return imagePreview.startsWith('/') ? imagePreview : null;
}
}
return imagePreview;
}
const GPT_IMAGE_EDIT_MAX_REFS = 4;
/**
* Build 2–4 distinct product image URLs for GPT Image `edits` (e.g. gpt-image-2).
* Merges the hosted hero (this run), storefront hero, and scraped gallery so models
* see multiple angles when available — not just a single frame.
*/
export function pickProductReferenceUrlsForGpt(options: {
hostedUrl: string | null;
heroRemoteUrl: string | null;
scrapedImageUrls: string[];
max?: number;
}): string[] {
const max = Math.min(GPT_IMAGE_EDIT_MAX_REFS, Math.max(1, options.max ?? GPT_IMAGE_EDIT_MAX_REFS));
const out: string[] = [];
const seen = new Set<string>();
const push = (u: string | null | undefined): boolean => {
const s = (u ?? '').trim();
if (!s || seen.has(s)) return false;
seen.add(s);
out.push(s);
return out.length >= max;
};
if (push(options.hostedUrl)) return out;
if (push(options.heroRemoteUrl)) return out;
for (const u of options.scrapedImageUrls) {
if (push(u)) break;
}
return out;
}
async function parseError(res: Response, fallback: string): Promise<string> {
const ct = res.headers.get('content-type');
try {
if (ct?.includes('application/json')) {
const j = await res.json();
return j.detail || j.message || fallback;
}
const t = await res.text();
return t?.trim() || fallback;
} catch {
return fallback;
}
}
export interface HealthStatus {
status: string;
service?: string;
kie_configured: boolean;
/** Seedance can fall back to Replicate when KIE fails if this is true */
replicate_configured?: boolean;
openai_configured: boolean;
gpt_image_model?: string;
ffmpeg_available?: boolean;
ffprobe_available?: boolean;
public_base_url?: string;
server_port?: number;
}
export async function checkHealth(): Promise<HealthStatus> {
const url = `${API_BASE}/health`;
const res = await fetch(url);
if (!res.ok) throw new Error(await parseError(res, 'Health check failed'));
return res.json();
}
export async function uploadImage(file: File): Promise<{ url: string }> {
const fd = new FormData();
fd.append('file', file);
const res = await fetch(`${API_BASE}/api/upload-image`, { method: 'POST', body: fd });
if (!res.ok) throw new Error(await parseError(res, 'Upload failed'));
return res.json();
}
export interface ScrapeProductResponse {
product_name: string;
description: string;
price: string;
offers: string;
target_audience: string;
product_images: string;
brand: string;
category: string;
image_urls: string[];
source_url: string;
[key: string]: unknown;
}
export async function scrapeProductPage(url: string): Promise<ScrapeProductResponse> {
const res = await fetch(`${API_BASE}/api/showcase/scrape`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url }),
});
if (!res.ok) throw new Error(await parseError(res, 'Scrape failed'));
return res.json();
}
export async function hostImageFromUrl(url: string): Promise<{ url: string; source_url: string }> {
const res = await fetch(`${API_BASE}/api/host-image-url`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url }),
});
if (!res.ok) throw new Error(await parseError(res, 'Could not host image'));
return res.json();
}
export async function generateDirectConcepts(params: {
productName: string;
description?: string;
price?: string;
targetAudience?: string;
count?: number;
ugcCount?: number;
modelShowcaseCount?: number;
featureHighlightCount?: number;
}): Promise<{
concepts: string[];
concept_groups?: {
ugc: string[];
model_showcase: string[];
feature_highlight: string[];
};
concept_execution_plans?: Record<
string,
{
duration_seconds: number;
render_mode: 'direct_seedance' | 'segmented';
reference_frame_prompts: string[];
}
>;
}> {
const res = await fetch(`${API_BASE}/api/showcase/direct-concepts`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
product_name: params.productName,
tagline: params.description ?? '',
mood: params.price ? `price: ${params.price}` : '',
target_audience: params.targetAudience ?? '',
count: params.count ?? 10,
ugc_count: params.ugcCount ?? 4,
model_showcase_count: params.modelShowcaseCount ?? 3,
feature_highlight_count: params.featureHighlightCount ?? 3,
}),
});
if (!res.ok) throw new Error(await parseError(res, 'Concept generation failed'));
return res.json();
}
export async function regenerateDirectConcept(params: {
productName: string;
description?: string;
price?: string;
targetAudience?: string;
exclude?: string[];
index?: number;
}): Promise<{ concept: string }> {
const res = await fetch(`${API_BASE}/api/showcase/direct-concept-one`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
product_name: params.productName,
tagline: params.description ?? '',
mood: params.price ? `price: ${params.price}` : '',
target_audience: params.targetAudience ?? '',
exclude: params.exclude ?? [],
index: params.index ?? 1,
}),
});
if (!res.ok) throw new Error(await parseError(res, 'Concept regeneration failed'));
return res.json();
}
/** Must match backend `showcase_prompts._CONCEPTS` keys. */
export const SHOWCASE_CONCEPT_OPTIONS = [
{
id: 'luxury_studio',
label: 'Luxury studio launch',
description: 'Slow dolly, pedestal, whisper VO — flagship packshot grammar.',
category: 'Commercial',
},
{
id: 'ugc_authentic',
label: 'UGC / social authentic',
description: 'Handheld, desk or counter, creator energy — short-form hooks.',
category: 'Social',
},
{
id: 'tech_minimal',
label: 'Tech / minimal',
description: 'Void set, hard edge light, precise motion — device-film calm.',
category: 'Commercial',
},
{
id: 'lifestyle_natural',
label: 'Lifestyle / natural light',
description: 'Real rooms, sun paths, slow living — editorial home story.',
category: 'Commercial',
},
{
id: 'bold_editorial',
label: 'Bold / color editorial',
description: 'Gels, graphic shadows, campaign-poster energy.',
category: 'Experimental',
},
{
id: 'unboxing_asmr',
label: 'Unboxing / desk ASMR',
description: 'Top-down peel, satisfying motion, whisper pacing.',
category: 'Social',
},
{
id: 'high_energy_sports',
label: 'High-energy sports',
description: 'Kinetic camera, impact cuts, performance-first momentum.',
category: 'Commercial',
},
{
id: 'moody_cinematic_noir',
label: 'Moody cinematic noir',
description: 'Chiaroscuro lighting, suspense pacing, dramatic reveals.',
category: 'Experimental',
},
{
id: 'playful_stopmotion_style',
label: 'Playful stop-motion style',
description: 'Tabletop whimsy, snappy object beats, colorful transitions.',
category: 'Experimental',
},
{
id: 'nature_outdoor_adventure',
label: 'Nature / outdoor adventure',
description: 'Golden trails, outdoor scale, utility-forward storytelling.',
category: 'Commercial',
},
] as const;
export type ShowcaseConceptId = (typeof SHOWCASE_CONCEPT_OPTIONS)[number]['id'];
export async function showcasePlanStream(
params: {
productName: string;
description?: string;
price?: string;
targetAudience: string;
shotCount: number;
/** Optional override; when omitted backend chooses 4/6/8 from concept + product brief. */
secondsPerSegment?: number;
/** Creative arc for shot planning (template + GPT). */
creativeConcept?: ShowcaseConceptId;
image?: File | null;
/** Original CDN URL for vision (optional); file upload takes precedence when present. */
heroImageUrl?: string;
},
onEvent: (e: StreamEvent) => void,
signal?: AbortSignal
): Promise<SegmentsPayload> {
const fd = new FormData();
fd.append('productName', params.productName);
fd.append('tagline', params.description ?? '');
fd.append('mood', params.price ? `price: ${params.price}` : '');
fd.append('targetAudience', params.targetAudience);
fd.append('shotCount', String(params.shotCount));
if (params.secondsPerSegment != null) {
fd.append('secondsPerSegment', String(params.secondsPerSegment));
}
fd.append('creativeConcept', params.creativeConcept ?? 'luxury_studio');
if (params.image) fd.append('image', params.image);
if (params.heroImageUrl) fd.append('heroImageUrl', params.heroImageUrl);
const res = await fetch(`${API_BASE}/api/showcase/plan-stream`, {
method: 'POST',
body: fd,
signal,
});
if (!res.ok) throw new Error(await parseError(res, 'Shot plan failed'));
if (!res.body) throw new Error('No response body');
const reader = res.body.getReader();
const dec = new TextDecoder();
let buf = '';
let final: SegmentsPayload | null = null;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
const lines = buf.split('\n');
buf = lines.pop() || '';
for (const line of lines) {
if (!line.trim()) continue;
const ev = JSON.parse(line) as StreamEvent;
onEvent(ev);
if (ev.event === 'complete') final = ev.payload;
if (ev.event === 'error') throw new Error(ev.message);
}
}
} finally {
reader.releaseLock();
}
if (!final) throw new Error('Stream ended without complete payload');
return final;
}
export interface KlingGenerateResponse {
taskId: string;
status: string;
}
/** Parse segment_info.duration ("4s", "8s") for Veo / trim. */
export function segmentClipSeconds(segment: { segment_info?: { duration?: string } }): 4 | 6 | 8 {
const raw = segment.segment_info?.duration ?? '';
const m = String(raw).match(/^(\d+)/);
if (m) {
const n = parseInt(m[1], 10);
if (n === 4 || n === 6 || n === 8) return n;
}
return 8;
}
export type SegmentVideoModel = 'veo3_fast' | 'seedance-2' | 'seedance-2-fast';
export type SeedanceSegmentModel = 'seedance-2' | 'seedance-2-fast';
/** UI segment model → KIE `jobs/createTask` model id. */
export function kieSeedanceModelId(model: SeedanceSegmentModel): string {
return model === 'seedance-2-fast' ? 'bytedance/seedance-2-fast' : 'bytedance/seedance-2';
}
export function isSeedanceSegmentModel(model: SegmentVideoModel): model is SeedanceSegmentModel {
return model === 'seedance-2' || model === 'seedance-2-fast';
}
/** Replicate model page (pricing tiers documented there). */
export const REPLICATE_SEEDANCE_20_URL = 'https://replicate.com/bytedance/seedance-2.0';
/**
* Replicate bills by **output video seconds**. Image / reference inputs use the `non_video_in` column
* on the Seedance 2.0 pricing grid (as of early 2026 on the model page).
*/
export const REPLICATE_SEEDANCE_20_NON_VIDEO_IN_USD_PER_SEC: Record<'480p' | '720p' | '1080p', number> = {
'480p': 0.08,
'720p': 0.18,
'1080p': 0.45,
};
export function replicateSeedance20NonVideoInUsdPerSec(
resolution: '480p' | '720p' | '1080p'
): number {
return REPLICATE_SEEDANCE_20_NON_VIDEO_IN_USD_PER_SEC[resolution];
}
/** Total USD estimate for Replicate Seedance 2.0 (non–video-in) at a given resolution and output seconds. */
export function estimateReplicateSeedance20Usd(params: {
resolution: '480p' | '720p' | '1080p';
totalOutputSeconds: number;
}): { usdPerSecond: number; estimatedUsd: number } {
const usdPerSecond = replicateSeedance20NonVideoInUsdPerSec(params.resolution);
const estimatedUsd = usdPerSecond * Math.max(0, params.totalOutputSeconds);
return { usdPerSecond, estimatedUsd };
}
/** Heuristic: product or shot text is likely wearable jewelry (rings, earrings, etc.). */
export function textSuggestsJewelry(...parts: (string | undefined | null)[]): boolean {
const t = parts.map((p) => String(p || '').toLowerCase()).join(' ');
if (!t.trim()) return false;
if (/\bring\s+light\b/.test(t)) return false;
return /\b(?:rings?|necklaces?|bracelets?|earrings?|pendants?|bangles?|anklets?|chokers?|lockets?|studs?|hoops?|cartilage|lobe|jewelry|jewellery|mangalsutra|nose\s*pin|925|14k|18k|karat|carat|sterling)\b/.test(
t
);
}
/** Seedance / video instructions when jewelry appears on a person. */
export function jewelryWearPlacementPromptBlock(): string {
return (
'[WEAR_PLACEMENT — jewelry on body] ' +
'When a person wears the hero piece it must rest on real anatomy with believable contact—no floating, no metal clipping through skin, no z-fighting. ' +
'Scale must match the references (band width, stone size vs finger or ear). ' +
'Rings: default to the ring finger (fourth digit) of the visible hand unless a reference clearly shows another finger; band sits between knuckles with consistent top-facing stone orientation. ' +
'Earrings: pierced placement at lobe or cartilage as the SKU implies; symmetric pair when the product is a pair. ' +
'Necklaces: natural gravity drape at collarbone/sternum; clasp hidden at nape when appropriate. ' +
'Bracelets / anklets: centered on wrist or ankle joint, not mid-forearm or calf. ' +
'Keep the exact same SKU silhouette and metal color as the reference set—no resized cartoon prop or mirrored design error on the body.'
);
}
/** Flatten structured segment JSON into a Seedance text prompt (max 20k on API). */
export function segmentToSeedancePrompt(segment: VeoSegment, productName?: string): string {
const ch = segment.character_description;
const sc = segment.scene_continuity;
const at = segment.action_timeline;
const lines: string[] = [];
if (productName?.trim()) lines.push(`Product: ${productName.trim()}.`);
if (ch?.current_state) lines.push(ch.current_state);
if (sc?.environment) lines.push(sc.environment);
if (sc?.camera_position) lines.push(`Camera: ${sc.camera_position}`);
if (sc?.camera_movement) lines.push(`Motion: ${sc.camera_movement}`);
if (sc?.lighting_state) lines.push(`Lighting: ${sc.lighting_state}`);
if (sc?.background_elements) lines.push(`Background: ${sc.background_elements}`);
if (at?.dialogue) lines.push(`VO: ${at.dialogue}`);
const sync = at?.synchronized_actions;
const syncLines: string[] = [];
if (sync && typeof sync === 'object') {
for (const [k, v] of Object.entries(sync)) {
if (v) syncLines.push(`${k}: ${v}`);
}
}
for (const s of syncLines) lines.push(s);
let text = lines.filter(Boolean).join('\n').trim();
if (!text) {
text =
'Cinematic premium product showcase, photoreal, smooth camera, shallow depth of field.';
}
const jewelryCtx = [
productName,
ch?.current_state,
sc?.environment,
sc?.background_elements,
at?.dialogue,
...syncLines,
];
if (textSuggestsJewelry(...jewelryCtx)) {
text = `${text}\n\n${jewelryWearPlacementPromptBlock()}`;
}
if (text.length > 20000) text = text.slice(0, 20000);
return text;
}
/** Max prompt length accepted by `/api/seedance/create` (matches backend). */
export const SEEDANCE_PROMPT_MAX_CHARS = 20000;
export type DirectSeedancePromptParams = {
productName: string;
description: string;
price: string;
targetAudience: string;
aspectRatio: string;
durationSeconds: number;
conceptText: string;
frameHints?: string[];
/** When false, model should favor clean ambience over heavy VO. */
generateAudio?: boolean;
};
/**
* Parse grouped direct-concept strings from the backend:
* `core idea. Angle: ... . Trigger: ...`
*/
export function parseDirectConceptLine(conceptText: string): {
core: string;
angle?: string;
trigger?: string;
} {
let rest = conceptText.trim();
let trigger: string | undefined;
const tri = rest.match(/\.\s*Trigger:\s*(.+)$/i);
if (tri && tri.index !== undefined) {
trigger = tri[1].trim();
rest = rest.slice(0, tri.index).trim();
}
let core = rest;
let angle: string | undefined;
const ang = rest.match(/\.\s*Angle:\s*(.+)$/i);
if (ang && ang.index !== undefined) {
angle = ang[1].trim();
core = rest.slice(0, ang.index).trim();
}
if (!core && (angle || trigger)) {
core = [angle, trigger].filter(Boolean).join(' — ') || conceptText.trim();
}
return { core: core || conceptText.trim(), angle, trigger };
}
function _beatLabels(durationSeconds: number, hintCount: number): string[] {
const d = Math.max(4, Math.min(15, Math.floor(durationSeconds)));
if (hintCount <= 0) return [];
if (hintCount === 1) return [`0–${d}s`];
if (hintCount === 2) return [`0–${Math.round(d / 2)}s`, `${Math.round(d / 2)}${d}s`];
const out: string[] = [];
for (let i = 0; i < hintCount; i++) {
const start = Math.round((i * d) / hintCount);
const end = i === hintCount - 1 ? d : Math.round(((i + 1) * d) / hintCount);
out.push(`${start}${end}s`);
}
return out;
}
/**
* Structured Seedance prompt for single-clip “direct concept” ads.
* De-duplicates angle/trigger when already present in `conceptText`, adds chronological beats, and trims safely.
*/
export function buildDirectSeedancePrompt(p: DirectSeedancePromptParams): string {
const name = p.productName.trim();
const ar = p.aspectRatio.trim() || '9:16';
const dur = Math.max(4, Math.min(15, Math.floor(p.durationSeconds)));
const parsed = parseDirectConceptLine(p.conceptText);
const hints = (p.frameHints ?? []).map((h) => h.trim()).filter(Boolean).slice(0, 4);
const labels = _beatLabels(dur, hints.length);
const sections: string[] = [];
sections.push(
`[TASK] One continuous ${dur}s ${ar} photoreal product commercial for ${name ? `"${name}"` : 'this product'}. ` +
'Single storyline with smooth transitions—avoid unrelated stock montage cuts.'
);
sections.push(`[CREATIVE] ${parsed.core}`);
if (parsed.angle) sections.push(`[ANGLE] ${parsed.angle}`);
if (parsed.trigger) sections.push(`[PSYCHOLOGY_TRIGGER] ${parsed.trigger}`);
if (hints.length > 0) {
const beats = hints.map((h, i) => `${labels[i]}: ${h}`).join('\n');
sections.push(
`[SHOT_BEATS — chronological, one flowing take]\n${beats}\n` +
'Blend these beats without jarring jumps; keep the product readable in every beat.'
);
}
const facts: string[] = [];
const rawDesc = p.description.trim();
const desc =
rawDesc.length > 2800 ? `${rawDesc.slice(0, 2800).trim()}…` : rawDesc;
if (desc) facts.push(`Description: ${desc}`);
if (p.price.trim()) facts.push(`Price (optional supers / tone only): ${p.price.trim()}`);
if (p.targetAudience.trim()) facts.push(`Audience / tone: ${p.targetAudience.trim()}`);
if (facts.length) sections.push(`[PRODUCT_CONTEXT]\n${facts.join('\n')}`);
const jewelryLike = textSuggestsJewelry(name, desc, parsed.core, p.conceptText, ...hints);
sections.push(
'[REFERENCE_IMAGES] Multiple angles may be provided—treat them as one SKU. ' +
'Match silhouette, metal tone, stone layout, and proportions exactly; do not invent a different design.'
);
if (jewelryLike) {
sections.push(jewelryWearPlacementPromptBlock());
}
sections.push(
'[LOOK] Soft commercial key + gentle fill, natural skin where people appear, shallow depth when it helps legibility, ' +
'smooth gimbal or slow dolly—no handheld shake unless the creative clearly asks for UGC.'
);
const negBase =
'Extra fingers, warped hands, melting metal, duplicated bands, unreadable text walls, fake logos, harsh unrelated location jumps.';
const negJewelry =
'wrong-finger ring placement, floating or hovering jewelry, metal intersecting palm or ear, mirrored asymmetric design, novelty oversized prop scale on body.';
sections.push(`[NEGATIVE] ${negBase}${jewelryLike ? ` ${negJewelry}` : ''}`);
if (p.generateAudio !== false) {
sections.push(
'[AUDIO] Light tasteful ambience; optional short VO that supports the angle—avoid overpowering music unless the brief implies it.'
);
} else {
sections.push('[AUDIO] No voiceover; subtle room tone or silence is fine.');
}
let text = sections.join('\n\n').trim();
if (text.length > SEEDANCE_PROMPT_MAX_CHARS) {
text = text.slice(0, SEEDANCE_PROMPT_MAX_CHARS);
}
return text;
}
export async function seedanceCreate(body: {
prompt: string;
reference_image_urls: string[];
aspect_ratio: string;
duration: number;
resolution?: string;
generate_audio?: boolean;
/** KIE model, e.g. `bytedance/seedance-2` or `bytedance/seedance-2-fast`. */
model?: string;
}): Promise<{ taskId: string }> {
const res = await fetch(`${API_BASE}/api/seedance/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!res.ok) throw new Error(await parseError(res, 'Seedance task failed'));
return res.json();
}
export function createSeedanceEventSource(taskId: string): EventSource {
return new EventSource(`${API_BASE}/api/seedance/events/${encodeURIComponent(taskId)}`);
}
export async function seedanceStatus(taskId: string): Promise<{
state?: string;
url?: string | null;
failMsg?: string | null;
}> {
const res = await fetch(`${API_BASE}/api/seedance/status/${encodeURIComponent(taskId)}`);
if (!res.ok) throw new Error(await parseError(res, 'Seedance status check failed'));
return res.json();
}
export function waitForSeedanceVideo(taskId: string, timeoutMs = 600000): Promise<string> {
return new Promise((resolve, reject) => {
let settled = false;
const es = createSeedanceEventSource(taskId);
let poller: number | null = null;
const pollEveryMs = 10000;
const settleSuccess = (url: string) => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
clearTimeout(t);
es.close();
resolve(url);
};
const settleFail = (msg?: string | null) => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
clearTimeout(t);
es.close();
reject(new Error(msg || 'Seedance generation failed'));
};
const inspect = (data: { state?: string; url?: string | null; failMsg?: string | null }) => {
if (data.state === 'success' && data.url) {
settleSuccess(data.url);
} else if (data.state === 'fail') {
settleFail(data.failMsg);
}
};
const startFallbackPolling = () => {
if (poller !== null || settled) return;
poller = window.setInterval(async () => {
if (settled) return;
try {
const data = await seedanceStatus(taskId);
inspect(data);
} catch {
/* ignore transient poll errors; timeout handles terminal failure */
}
}, pollEveryMs);
};
// Fast path: handle already-finished tasks without waiting for callback/SSE.
void seedanceStatus(taskId).then(inspect).catch(() => {
/* ignore transient status errors; SSE path remains active */
});
const t = setTimeout(() => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
es.close();
reject(new Error('Seedance generation timed out'));
}, timeoutMs);
es.onmessage = (ev) => {
if (settled) return;
try {
const data = JSON.parse(ev.data) as {
state?: string;
url?: string | null;
failMsg?: string | null;
};
inspect(data);
} catch {
/* ignore malformed SSE payloads */
}
};
es.onerror = () => {
// EventSource retries automatically; poll only as a degraded-path fallback.
startFallbackPolling();
};
});
}
export async function klingGenerate(body: {
prompt: string | object;
imageUrls?: string[];
model?: string;
aspectRatio?: string;
generationType?: string;
seeds?: number;
voiceType?: string;
/** 4, 6, or 8 — forwarded to the API so the provider can honor shot length */
durationSeconds?: number;
}): Promise<KlingGenerateResponse> {
const res = await fetch(`${API_BASE}/api/veo/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!res.ok) throw new Error(await parseError(res, 'Video start failed'));
return res.json();
}
export function createKlingEventSource(taskId: string): EventSource {
return new EventSource(`${API_BASE}/api/veo/events/${taskId}`);
}
export async function klingStatus(taskId: string): Promise<{
status?: string;
url?: string | null;
error?: string | null;
message?: string | null;
}> {
const res = await fetch(`${API_BASE}/api/veo/status/${encodeURIComponent(taskId)}`);
if (!res.ok) throw new Error(await parseError(res, 'Video status check failed'));
return res.json();
}
export function waitForKlingVideo(taskId: string, timeoutMs = 420000): Promise<string> {
return new Promise((resolve, reject) => {
let settled = false;
const es = createKlingEventSource(taskId);
let poller: number | null = null;
const pollEveryMs = 10000;
const settleSuccess = (url: string) => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
clearTimeout(t);
es.close();
resolve(url);
};
const settleFail = (msg?: string | null) => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
clearTimeout(t);
es.close();
reject(new Error(msg || 'Generation failed'));
};
const inspect = (data: {
status?: string;
url?: string | null;
error?: string | null;
message?: string | null;
}) => {
if (data.status === 'succeeded' && data.url) {
settleSuccess(data.url);
} else if (data.status === 'failed' || data.status === 'cancelled') {
settleFail(data.error || data.message);
}
};
const startFallbackPolling = () => {
if (poller !== null || settled) return;
poller = window.setInterval(async () => {
if (settled) return;
try {
const data = await klingStatus(taskId);
inspect(data);
} catch {
/* ignore transient poll errors; timeout handles terminal failure */
}
}, pollEveryMs);
};
// Fast path for already-finished tasks.
void klingStatus(taskId).then(inspect).catch(() => {
/* ignore transient status errors; SSE path remains active */
});
const t = setTimeout(() => {
if (settled) return;
settled = true;
if (poller !== null) window.clearInterval(poller);
es.close();
reject(new Error('Video generation timed out'));
}, timeoutMs);
es.onmessage = (ev) => {
if (settled) return;
try {
const data = JSON.parse(ev.data);
inspect(data);
} catch {
/* ignore */
}
};
es.onerror = () => {
// EventSource retries automatically; poll only as a degraded-path fallback.
startFallbackPolling();
};
});
}
export async function downloadVideo(
url: string,
opts?: { trimSeconds?: 4 | 6 | 8 }
): Promise<Blob> {
const q = new URLSearchParams({ url });
if (opts?.trimSeconds != null) q.set('trimSeconds', String(opts.trimSeconds));
const res = await fetch(`${API_BASE}/api/veo/download?${q}`);
if (!res.ok) throw new Error('Download failed');
return res.blob();
}
function loadVideoFromFile(file: File): Promise<HTMLVideoElement> {
return new Promise((resolve, reject) => {
const v = document.createElement('video');
v.preload = 'metadata';
v.src = URL.createObjectURL(file);
v.onloadedmetadata = () => resolve(v);
v.onerror = () => {
URL.revokeObjectURL(v.src);
reject(new Error('Could not read video'));
};
});
}
export async function getVideoDuration(file: File): Promise<number> {
const v = await loadVideoFromFile(file);
try {
return v.duration;
} finally {
URL.revokeObjectURL(v.src);
}
}
export async function mergeVideos(blobs: Blob[], clips: ClipMetadata[]): Promise<Blob> {
const fd = new FormData();
fd.append('clips_data', JSON.stringify(clips));
blobs.forEach((b, i) => fd.append('files', b, `clip_${i}.mp4`));
const res = await fetch(`${API_BASE}/api/export/merge`, { method: 'POST', body: fd });
if (!res.ok) throw new Error(await parseError(res, 'Merge failed'));
return res.blob();
}
/**
* Run async work on `items` with at most `concurrency` in flight. Results are in original order.
* Use for I/O-bound pipelines (e.g. several segment renders) without unbounded provider load.
*/
export async function mapWithConcurrency<T, R>(
items: T[],
concurrency: number,
mapper: (item: T, index: number) => Promise<R>,
onProgress?: (completed: number, total: number) => void
): Promise<R[]> {
const n = items.length;
if (n === 0) return [];
const cap = Math.max(1, Math.min(concurrency, n));
const results: R[] = new Array(n);
let next = 0;
let finished = 0;
async function worker(): Promise<void> {
while (true) {
const i = next++;
if (i >= n) return;
results[i] = await mapper(items[i], i);
finished++;
onProgress?.(finished, n);
}
}
await Promise.all(Array.from({ length: cap }, () => worker()));
return results;
}
/** Default parallel segment pipelines (GPT keyframe + Veo/Seedance). Reduce if providers rate-limit. */
export const SEGMENT_RENDER_CONCURRENCY = 3;
/** Build merge metadata for full-length clips in order. */
export async function clipsFromBlobs(blobs: Blob[]): Promise<ClipMetadata[]> {
return Promise.all(
blobs.map(async (b, i) => {
const file = new File([b], `c${i}.mp4`, { type: 'video/mp4' });
const dur = await getVideoDuration(file);
return {
index: i,
startTime: 0,
endTime: dur,
type: 'video' as const,
};
})
);
}
export async function generateSegmentFirstFrame(params: {
segment: VeoSegment;
referenceImageUrls: string[];
aspectRatio: string;
productName: string;
}): Promise<{ url: string; model: string; size: string }> {
const candidateRefs = Array.from(
new Set(
params.referenceImageUrls
.map((u) => (u || '').trim())
.filter(Boolean)
)
).slice(0, GPT_IMAGE_EDIT_MAX_REFS);
const vettedRefs: string[] = [];
for (const url of candidateRefs) {
// Preflight only our hosted image URLs to avoid CORS false negatives on third-party CDNs.
if (!url.includes('/api/images/')) {
vettedRefs.push(url);
continue;
}
try {
const probe = await fetch(url, { method: 'GET' });
if (probe.ok) vettedRefs.push(url);
} catch {
// Network/CORS error on probe: keep URL and let backend decide.
vettedRefs.push(url);
}
}
const referenceImageUrls = vettedRefs.length > 0 ? vettedRefs : candidateRefs;
const res = await fetch(`${API_BASE}/api/showcase/segment-first-frame`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
segment: params.segment,
reference_image_urls: referenceImageUrls,
aspect_ratio: params.aspectRatio,
product_name: params.productName,
}),
});
if (!res.ok) throw new Error(await parseError(res, 'GPT Image first frame failed'));
return res.json();
}
export async function generateSegmentVideo(
segment: VeoSegment,
imageUrl: string,
aspectRatio: string,
seed: number,
voiceType: string,
opts: {
model?: SegmentVideoModel;
productName?: string;
seedanceResolution?: '480p' | '720p' | '1080p';
promptOverride?: string;
referenceImageUrls?: string[];
} = {}
): Promise<Blob> {
const clipSec = segmentClipSeconds(segment);
const model = opts.model ?? 'seedance-2-fast';
if (isSeedanceSegmentModel(model)) {
const prompt =
opts.promptOverride?.trim() || segmentToSeedancePrompt(segment, opts.productName);
const { taskId } = await seedanceCreate({
prompt,
reference_image_urls:
opts.referenceImageUrls && opts.referenceImageUrls.length > 0
? opts.referenceImageUrls
: [imageUrl],
aspect_ratio: aspectRatio,
duration: clipSec,
resolution: opts.seedanceResolution ?? '480p',
generate_audio: voiceType.trim().toLowerCase() !== 'none',
model: kieSeedanceModelId(model),
});
const url = await waitForSeedanceVideo(taskId);
return downloadVideo(url, { trimSeconds: clipSec });
}
const { taskId } = await klingGenerate({
prompt: opts.promptOverride?.trim() || segment,
imageUrls: [imageUrl],
model: 'veo3_fast',
aspectRatio,
generationType: 'FIRST_AND_LAST_FRAMES_2_VIDEO',
seeds: seed,
voiceType,
durationSeconds: clipSec,
});
const url = await waitForKlingVideo(taskId);
return downloadVideo(url, { trimSeconds: clipSec });
}