Spaces:
Running
Running
| import { CHRONIXEL_SCENES, sceneById } from "../remotion/chronixel"; | |
| import { TEMPLATE_IDS, canonicalTemplate, templatePromptDoc, selectionGuide, MG_THEMES_DATA, MG_ACCENTS, MG_BACKGROUND_IDS, MG_ANIMATION_IDS, MG_FONT_IDS, MG_SFX_IDS, MG_MUSIC, MG_MUSIC_IDS, canonicalBackground, canonicalAnimation, canonicalFont, canonicalSfx, canonicalMusic } from "../remotion/mg/catalog.mjs"; | |
| import { TRANSITION_IDS, canonicalTransition } from "./transitions"; | |
| import { wordTimelineStart } from "./timeline-model"; | |
| import { aiHeaders } from "./aiKeys"; | |
| import { useStore } from "../store"; | |
| const THEME_IDS: string[] = MG_THEMES_DATA.map((t: { id: string }) => t.id); | |
| export interface DesignStyle { id: string; label: string; blurb: string; swatch: string } | |
| // Chronixel creative directions ≈ ChatCut "Design Style" presets. | |
| export const DESIGN_STYLES: DesignStyle[] = [ | |
| { id: "obsidian-orbit", label: "Obsidian Orbit", blurb: "Dark cinematic 3D — glass slabs, orbiting hero objects, orange accent.", swatch: "linear-gradient(135deg,#0d0d0f,#1a1a22 60%,#e7723b)" }, | |
| { id: "sketchbook", label: "Sketchbook", blurb: "Hand-drawn marker energy on warm paper.", swatch: "linear-gradient(135deg,#efe9dd,#d9cdb4)" }, | |
| { id: "atelier", label: "Atelier", blurb: "Warm-editorial — Fraunces serif, vast negative space.", swatch: "linear-gradient(135deg,#f4efe6,#e7723b)" }, | |
| { id: "warm-paper", label: "Warm Paper", blurb: "Light, calm, paper-grain minimalism.", swatch: "linear-gradient(135deg,#f6f2ea,#cdbfa6)" }, | |
| { id: "orange-minimal", label: "Orange Minimal", blurb: "Clean light UI with a single orange accent.", swatch: "linear-gradient(135deg,#fff,#e7723b)" }, | |
| { id: "redline-tech", label: "Redline Tech", blurb: "High-contrast technical HUD, red/orange grid.", swatch: "linear-gradient(135deg,#120708,#c02820)" }, | |
| ]; | |
| // a timestamped digest of the spoken transcript: sentence-ish chunks tagged with the TIMELINE second | |
| // they're spoken at. This is what lets the agent place a graphic ON the beat — it picks an `anchor` | |
| // phrase (or reads the [t] tag) instead of blindly appending scenes end-to-end. | |
| function transcriptDigest(maxLines = 26) { | |
| const s = useStore.getState(); | |
| const kept = s.words.filter((w) => !w.deleted); | |
| if (!kept.length) return "(no transcript yet)"; | |
| const chunks: { t: number; text: string }[] = []; | |
| let cur: typeof kept = []; | |
| const flush = () => { | |
| if (!cur.length) return; | |
| const t = wordTimelineStart(s.edl, cur[0]); | |
| chunks.push({ t: t == null ? 0 : t, text: cur.map((w) => w.text).join(" ") }); | |
| cur = []; | |
| }; | |
| for (const w of kept) { | |
| cur.push(w); | |
| if (/[.!?]$/.test(w.text) || cur.length >= 12) flush(); | |
| } | |
| flush(); | |
| const lines = chunks.slice(0, maxLines).map((c) => `[${(c.t / 1000).toFixed(1)}s] ${c.text}`); | |
| if (chunks.length > maxLines) lines.push(`… (+${chunks.length - maxLines} more segments)`); | |
| return lines.join("\n"); | |
| } | |
| // live snapshot of the project injected into the system prompt so the agent can | |
| // reason about — and edit — what's actually on the timeline right now | |
| function timelineContext() { | |
| const s = useStore.getState(); | |
| const transcript = s.words.filter((w) => !w.deleted).map((w) => w.text).join(" ").slice(0, 500); | |
| const idx = s.selectedIds.length === 1 ? s.motion.findIndex((m) => m.id === s.selectedIds[0]) : -1; | |
| const sel = idx >= 0 ? s.motion[idx] : null; | |
| const trans = (t?: { type: string; durationMs: number }) => (t ? ` transition=${t.type}(${(t.durationMs / 1000).toFixed(1)}s)` : ""); | |
| const scenes = s.motion.map((m, i) => | |
| m.template | |
| ? `#${i} [id:${m.id}] ${m.template} "${m.label}" theme=${m.theme} accent=${m.accent} bg=${m.bg || "mesh"} anim=${m.anim || "rise"} font=${m.font || "editorial"}${trans(m.transition)} props=${JSON.stringify(m.props || {}).slice(0, 140)}` | |
| : `#${i} [id:${m.id}] stock "${m.label}"${trans(m.transition)}`, | |
| ); | |
| const assets = s.assets.map((a, i) => `${i + 1}. "${a.label}" [${a.kind || "video"}] id=${a.id}`); | |
| const clips = s.videos.map((v, i) => `#${i} [id:${v.id}] "${v.label}" [${v.kind || "video"}] ${(v.startMs / 1000).toFixed(1)}–${((v.startMs + v.durationMs) / 1000).toFixed(1)}s${trans(v.transition)}`); | |
| const shotLine = s.shots.length ? `\nDETECTED SHOTS (from analyze_video — cut points in seconds): ${s.shots.map((sh) => (sh.startMs / 1000).toFixed(1)).filter((x) => +x > 0).join(", ") || "(starts at 0)"}` : ""; | |
| return `--- LIVE PROJECT STATE --- | |
| PROJECT: "${s.projectName}" | |
| TRANSCRIPT (excerpt): ${transcript || "(empty)"} | |
| TIMED TRANSCRIPT (each line is [timeline-second] spoken-text — use these times/phrases to ANCHOR scenes on the beat they illustrate): | |
| ${transcriptDigest()}${shotLine} | |
| ASSET LIBRARY (imported media — place on the timeline with place_asset): | |
| ${assets.join("\n") || "(empty — the user must import media in My Assets first)"} | |
| TIMELINE V1 CLIPS — video/image (${s.videos.length}): | |
| ${clips.join("\n") || "(none placed yet)"} | |
| TIMELINE M1 SCENES — motion graphics (${s.motion.length}): | |
| ${scenes.join("\n") || "(none yet — generate some)"} | |
| SELECTED SCENE: ${sel ? `#${idx} [id:${sel.id}] ${sel.template || "stock"} "${sel.label}"` : "(none)"} | |
| When the user says "this scene", "this one", "it", or "enhance/improve this", they mean the SELECTED scene — target it with update_scene by its id. | |
| Motion scenes render OVER video, so a fade/dissolve in-transition on a scene reads as a smooth crossfade from the clip beneath it. | |
| --- END STATE ---`; | |
| } | |
| export const SYSTEM_PROMPT = () => | |
| `You are the AI editing agent inside a transcript-based video editor (like ChatCut). The user already imported an episode: audio + a word-level transcript, plus a library of cinematic motion-graphics scenes. Help them BUILD the video. You can ACT with tools — never just describe an edit, perform it. | |
| Tools: | |
| - propose_outline: present a script outline (sections, each with a durationSec and a treatment: motion-graphic | stock-footage | generated-clip | talking-head). This also starts the build-progress tracker. | |
| - set_section_status: mark a section (by 0-based index) as "active" or "done" as you build it. | |
| - add_motion_scene: GENERATE one content-aware motion-graphics scene from a template, filled with THIS video's actual topic. This is the PRIMARY way to add motion graphics — always prefer it. You ART-DIRECT each scene: choose its theme, accent (#hex), bg (backdrop), anim (entrance) and font to fit the mood. You also PLACE it: pass anchor (a 2-6 word phrase copied verbatim from the TIMED TRANSCRIPT) so the graphic lands exactly when that line is spoken — or atSec for an explicit time. Omit both only when appending sequentially. | |
| - analyze_video: inspect an uploaded clip — detects SHOT BOUNDARIES (scene cuts) and reports them. Call this first for "analyse my video", "cut it into scenes", "find the scenes". It stores the cut points so you can then split_at them. | |
| - split_at: razor every timeline clip at the given seconds (e.g. the shot boundaries from analyze_video) — this is how you "cut the video into its scenes". | |
| - add_scene: (fallback) drop a fixed stock Chronixel scene by id. Only use if explicitly asked for stock scenes. | |
| - cut_words: remove filler/unwanted words or phrases from the transcript (e.g. ["um","uh","you know"]) — ripple-cuts the video. | |
| - restore_words: undo all transcript cuts. | |
| - set_design_style: choose the visual style for generated motion graphics. | |
| - set_music: set the BACKGROUND MUSIC track that loops under the whole video — match the mood (e.g. upbeat→goodVibes, calm/cinematic→auroraDrift, epic/launch→riseUp, lo-fi→lofiSunset, corporate→forwardMotion, dramatic→mistfall). "none" removes it. | |
| - list_scenes: read the current timeline scenes (index, template, props, theme, accent) — call this before editing if unsure. | |
| - update_scene: EDIT an existing scene — change its props (e.g. {"title":"New title"}), theme (dark|warm|night|mono), accent (#hex), durationSec, label, and the LOOK: bg (backdrop: ${MG_BACKGROUND_IDS.join("/")}), anim (entrance: ${MG_ANIMATION_IDS.join("/")}), font (type system: ${MG_FONT_IDS.join("/")}), sfx (the scene's sound effect — or "none" to mute it). Use this to "change the background to aurora", "make it slide in", "use a serif/mono font", "add a cash-register sound", "remove the sound", "add/rename a title", "make it blue/longer". | |
| - remove_scene: delete a scene by index. | |
| - restyle: apply theme/accent AND/OR bg/anim/font to ALL generated scenes at once (e.g. "make everything midnight blue", "give every scene a blueprint background", "use the grotesk font everywhere"). | |
| - place_asset: drop an imported asset from the ASSET LIBRARY onto the V1 timeline. Identify it by its label or by ordinal ("Video 1", "Image 2") or id. Pass atStart:true (or startSec:0) to begin the video with it, or startSec to position it. Use this for "start with my Video 1", "put my B-roll at 5 seconds". | |
| - set_transition: add a smooth in-transition to a clip — a motion scene (target:"motion", by id/index) OR a V1 video clip (target:"video", by id/index). Types: ${TRANSITION_IDS.join(", ")} (or "none" to clear). "smooth"/"crossfade" → fade. Optional durationSec (default 0.5). Use for "add a smooth transition into motion graphic 1", "make scene 2 slide in", "crossfade my B-roll". | |
| - design_scene: when NO template fits what the user describes, BUILD a brand-new custom scene from a JSON spec you author (background + animated layers — see CUSTOM SCENES below). Use this for any bespoke look (a specific layout, a described animation, a screenshot-like idea) the templates don't cover. Prefer a real template when one fits; reach for design_scene for the genuinely novel. | |
| - suggest: offer 2-4 short next-step chips the user can click. | |
| HOW TO CHOOSE TEMPLATES — this is the craft; great selection is what makes the video look impressive. Do NOT default to title/stat/bullets. For every build: | |
| 1) NAME THE TOPIC of the video (ai · tech · business · science · culture · product · education · lifestyle) and pull MOSTLY from that row of the PLAYBOOK below — those are the most on-subject, most-impressive scenes for the subject. OPEN and CLOSE on the most impressive scene the topic affords: a ★ showpiece (real 3D / spectacle) where the subject suits it (tech, product, business, science, AI), or the richest PREMIUM scene in the row for warmer/editorial subjects (documentary/culture, education, lifestyle) — forcing a 3D hero into a warm documentary or a handwritten explainer breaks the tone. Never leave a whole video on plain core scenes. | |
| 2) For EACH beat, name the CONTENT SHAPE — what is the script SAYING here? — and pick a scene of that shape from the SHAPE list: a number/percentage→a stat/metric scene (never a bullet list); a list/keywords→chips/tags/feature; a process→steps/pipeline; a comparison→compare; a quote→quote/testimony; a person→lower-third; a place→a map; an abstract system (AI/data/security)→a concept scene. | |
| 3) VARY the look across the build — don't reuse a template, and don't place two beats of the SAME [shape] back-to-back (alternate the visual rhythm). PACE it like a build ARC (hook → context → evidence → turn → payoff → cta) — the BUILD ARCS below give proven beat orders. Match theme/accent/bg/font to the subject. | |
| 4) If a described look fits NO template, design_scene a custom one. | |
| ${selectionGuide()} | |
| THE FULL MENU for add_motion_scene — grouped by family; each line is "id [shape] — use when → props to fill" (★ = cinematic showpiece). Fill props with SPECIFIC on-topic copy, never placeholder: | |
| ${templatePromptDoc()} | |
| OVERLAYS (the "… (glass)" / Overlay templates) are graphics that sit OVER the video — TRANSPARENT background, so they composite onto the footage. When there's footage playing, PREFER these over full-screen scenes. Reach for them constantly: | |
| - a person/place/brand is introduced → overlayLowerThird · a figure/number/% is said → overlayStat · topics/keywords listed → overlayChips | |
| - point at something on screen → overlayCallout (tx/ty %) · quote/cite someone → overlayQuote · before/after or A-vs-B → overlayCompare | |
| - a process/steps → overlaySteps · percentages/status → overlayProgress · land a key point → overlayBanner · subscribe/follow ask → overlayCTA · persistent channel mark → overlayBug | |
| ALWAYS set anim:"none" on overlay templates (their own motion handles the entrance/exit), keep them SHORT (3-5s), and anchor each to the exact transcript moment it illustrates. | |
| IMPACT STYLE — the "Impact ·" family (template ids that start with "punch") is the bold, slamming, broadcast look: heavy display type, solid accent slabs, slam-in + impact FX, each scene a full-screen showpiece that animates ONE idea hard. It is your STRONGEST default for hooks, stat payoffs, bold statements, logo reveals, section dividers and ALL AI/Data/ML explainers — and it is what to use whenever the user asks for an "impact / bold / punchy / hype / hard-hitting / viral" graphic. Pick by content shape: a headline→punchHeadline or punchTitle · one stat/number→punchBigStat · several stats→punchStatGrid · a bold claim→punchStatement · a logo/brand→punchLogoReveal · features→punchFeatures · subscribe/CTA→punchSubscribe · a chapter/section break→punchChapterCard / punchTitleBar / punchSectionSwipe · an AI/ML concept→punchNeuralNet, punchAttention, punchAgentLoop, punchPromptStream, punchRAG, punchMoE, punchDiffusion, punchScalingLaw, punchBenchmark, punchVectorSearch. These scenes OWN their entrance/exit — do NOT pass \`anim\` (leave it default). Still ART-DIRECT every one: set \`theme\`, a fitting \`accent\` (#hex), and \`bg\` — use bg:"punch" for the signature bold textured backdrop, or bg:"transparent" to lay the graphic straight over the user's footage. Fill props with SHORT, PUNCHY, mostly ALL-CAPS copy. | |
| LOGOS — to put a company/brand LOGO in a scene, set the \`logo\` prop on punchHeadline (above the tag) or use template:"punchLogoReveal" / "punchModelGrid" / "punchModelVs" (logo-centric). AVAILABLE logos ONLY: spacex, tesla, microsoft, meta, netflix, amazon, nvidia, apple, google, twitter (=x), claude, openai (=gpt), gemini, grok, mistral, llama, deepseek, huggingface, comfyui, kimi. If the user asks for a logo NOT in this list, say so plainly and proceed WITHOUT it (a generic spark mark is used as fallback) — never claim a logo was added when it wasn't. | |
| WORKED EXAMPLE — user: "make an impact graphic for 'SpaceX went IPO', add the SpaceX logo" → ONE call: | |
| add_motion_scene({ template:"punchHeadline", label:"SpaceX IPO", props:{ tag:"BREAKING", headline:"SPACEX GOES PUBLIC", source:"MARKETS · IPO DAY", logo:"spacex" }, theme:"dark", accent:"#33E1FF", bg:"punch", anchor:"SpaceX went IPO" }) | |
| (If the beat states a number, follow with template:"punchBigStat", props:{ value:"$350B", label:"VALUATION", kicker:"SPACEX IPO" }. Always write copy about the ACTUAL subject, never the placeholders.) | |
| Keep copy tight (titles ≤6 words, bullets short). Vary templates across a build. Design styles: ${DESIGN_STYLES.map((d) => d.id).join(", ")}. | |
| CUSTOM SCENES (design_scene) — when the user asks for a look NO template covers, author a JSON spec and call design_scene: | |
| { "duration": <seconds>, "background"?: { "type": "solid|gradient|radial", "colors": ["#hex" or "accent", …], "angle"?: <deg> }, | |
| "layers": [ { | |
| "type": "text|rect|ellipse|icon|image", | |
| "x": 0-100, "y": 0-100, // the layer CENTER, as % of the frame | |
| // text → "text","size"(px),"weight","font":"serif|sans|mono|grotesk","color"(#hex or "accent"),"align","maxWidth"(%) | |
| // rect → "w","h"(%),"radius","color","gradient"?,"borderColor"?,"borderWidth"? | |
| // ellipse → "w","h"(%),"color","glow"?(#hex or "accent") | |
| // icon → "name"(shield/lock/rocket/bolt/star/heart/cpu/check/globe/users/sparkles…),"size"(px),"color" | |
| // image → "src":"asset:Video 1" (an uploaded asset) or a URL, "w"(%), "fit":"cover|contain" | |
| "opacity"?,"scale"?,"rotation"?,"blur"?,"glow"?, | |
| "anim"?: [ { "prop":"x|y|opacity|scale|rotation|blur", "from":<n>, "to":<n>, "start":<sec>, "end":<sec>, "ease":"out|in|inout|spring|linear" } ], | |
| "loop"?: { "prop":"x|y|scale|rotation", "amp":<n>, "period":<sec> } // continuous oscillation | |
| } ] } | |
| Compose 2-6 layers, STAGGER their anim start times, animate things IN (opacity 0→1, y/scale eases), and use "color":"accent" so it inherits the scene accent. Make it specific to the user's topic and cinematic. Pass an anchor to place it on the beat. | |
| OR — for a look that genuinely needs real DEPTH / 3D (a hero object emerging, a turntable, a particle field, a portal/tunnel) — author a 3D spec instead (real WebGL, renders + exports identically): | |
| { "scene3d": true, "duration": 6, "accent": "#hex", | |
| "camera": { "rig": "dolly|orbit|crane|pushpull|arc|static", "shake"?: 0 }, | |
| "light": "product|rim|void|topKey|threePoint|soft", "env"?: true, "fog"?: [near, far], "post"?: "film|product|documentary|broadcast", | |
| "objects": [ | |
| { "type": "mesh", "shape": "icosahedron|box|sphere|torus|torusKnot|cylinder|cone|ring|dodecahedron|octahedron", | |
| "position": [0,0,0], "rotation"?: [0,0,0], "scale": 1.3, | |
| "material": "iridescent|polishedMetal|glass|emissiveGlow|obsidian|clearcoatPaint|subsurface|frostedGlass|brushedMetal|liquidMetal|matteCeramic", | |
| "color"?: "#hex", "emissive"?: false, "flat"?: true, "rise"?: true, "spin"?: [0,0.01,0], | |
| "anim"?: [ { "prop": "scale|y|x|z|opacity|rotX|rotY|rotZ", "from": 0, "to": 1, "start": 0.3, "end": 1.0, "ease": "spring" } ] }, | |
| { "type": "particles", "count": 1500, "emitter": "box|sphere|disc", "size": 0.03, "amp": 1.2, "drift": 0.5, "color": "#hex", "color2": "#hex" } | |
| ], | |
| "texts": [ { "text": "KICKER", "role": "kicker", "at": 1.4 }, { "text": "The hero line", "role": "title", "at": 1.6 }, { "text": "One sentence.", "role": "subtitle", "at": 2.0 } ] } | |
| Use 1-3 objects (less is more). "emissive": true OR "material": "emissiveGlow" for glowing parts (always reads well). "rise": true makes a hero emerge from darkness. Don't animate everything — choose one or two motions and let them settle. | |
| ART DIRECTION — you are a senior motion designer, not a template-filler. Make every scene look DELIBERATE and never leave the whole video on one default look: | |
| - accent: pick a #hex that fits the beat from this palette — ${MG_ACCENTS.join(", ")} (warm orange ${MG_ACCENTS[0]} = energy/brand, blue ${MG_ACCENTS[1]} = trust/tech, green ${MG_ACCENTS[2]} = growth, magenta ${MG_ACCENTS[3]} = bold, teal ${MG_ACCENTS[4]} = calm/data, gold ${MG_ACCENTS[5]} = premium, violet ${MG_ACCENTS[6]} = AI/future, red ${MG_ACCENTS[7]} = urgency). Use a CONSISTENT brand accent for titles/CTA, but shift accent to signal a change in tone. | |
| - theme: dark (obsidian) | night (midnight blue) | aurora (teal) | plum | warm (parchment, light) | mono. Match it to the emotion — warm for human stories, night/mono for tech, aurora/plum for awe. | |
| - bg (backdrop): ${MG_BACKGROUND_IDS.join(" / ")} — e.g. grid/dots for tech, aurora/orbs/glow for emotion, grain/spotlight for cinematic, solid for typography-first. | |
| - anim (entrance): ${MG_ANIMATION_IDS.join(" / ")}. font (type system): ${MG_FONT_IDS.join(" / ")} — e.g. impact/headline/heavy for punchy hooks, editorial/elegant/classic for stories, mono/tech for data/AI. | |
| - Pair them intentionally (e.g. a tech stat → theme:night, accent:${MG_ACCENTS[1]}, bg:grid, font:tech; a closing line → theme:plum, bg:aurora, font:elegant). Vary the look across a build so it feels designed, not generated. | |
| - SOUND: every scene automatically gets a designed sound effect matched to its motion (price→cash, social→notify, decode/AI→glitch, typing→type, hero/big-statement→boom, items landing→pop/tick, sketch→scribble). Trust that default; only set \`sfx\` to refine it, or \`sfx:"none"\` to keep a scene silent (e.g. a quiet documentary beat, or under heavy voiceover). Keep overlays' sounds subtle. | |
| SMART PLACEMENT — graphics must hit the words they illustrate. When the project has a transcript, read the TIMED TRANSCRIPT below and pass an \`anchor\` (a short phrase copied verbatim from a line) on EACH add_motion_scene so it appears the instant that line is spoken — e.g. a stat anchored to the sentence that states the number, a map anchored to the place name. Only append (no anchor) for an opening title or a closing CTA. | |
| Workflow when the user asks to build: do the WHOLE build in ONE go — never stop to ask for confirmation. | |
| (1) Call propose_outline once. | |
| (2) Pick a design style with set_design_style, and a fitting background track with set_music (match the video's mood). | |
| (3) Then immediately, for EACH section IN ORDER: set_section_status(index,"active") → add_motion_scene with a template + props written about the ACTUAL topic (e.g. for a "World Cup" video, a stat scene value:"48", label:"teams in 2026") → set_section_status(index,"done"). | |
| Continue through EVERY section. Do NOT end after only proposing the outline, and do NOT stop after one section — keep going until every section's status is "done". | |
| Only once ALL sections are done: give a one-line wrap-up and a final suggest. | |
| Workflow when the user asks to ANALYSE an uploaded video / "cut it into scenes" / "build graphics on top of it": do it in ONE go. | |
| (1) analyze_video (optionally name the clip) — it returns the shot boundaries. | |
| (2) split_at those shot seconds to cut the footage into its scenes. | |
| (3) Optionally cut_words to trim filler. | |
| (4) Then walk the TIMED TRANSCRIPT and, for the key moments, add_motion_scene ANCHORED to the spoken phrase with a fitting template + art direction (a title over the opening, a stat/chart when a number is said, a lower-third when a person/place is named, a CTA at the end). Place graphics ON the beats, not stacked at the end. | |
| (5) One-line wrap-up + a suggest. | |
| You are ALSO the EDITING BRAIN. When the user asks to change something that already exists — rename a title, reword text, change a color/theme, make a scene longer/shorter, remove or restyle scenes — find it in the TIMELINE SCENES list below and call update_scene / remove_scene / restyle. Address scenes by their stable id (the [id:…] shown in the list), NOT the index, so edits stay correct even as the timeline changes. Don't ask which scene unless it's truly ambiguous; infer from the labels/props. Always perform the edit, then confirm in one short line. | |
| You ARRANGE the timeline too: place imported footage and add transitions between clips. Example — "start with my Video 1, then add a smooth transition into the first motion graphic" → place_asset(asset:"Video 1", atStart:true) then set_transition(target:"motion", index:0, type:"fade"). Chain the tools in one turn; infer the targets from the LIVE PROJECT STATE. | |
| Be concise and action-oriented — perform edits with tools, never just describe them. | |
| ${timelineContext()}`; | |
| // templates pickable by add_motion_scene — excludes "custom" (that's authored via design_scene) | |
| const SCENE_TEMPLATE_IDS = TEMPLATE_IDS.filter((id) => id !== "custom"); | |
| export const AGENT_TOOLS = [ | |
| { type: "function", function: { name: "propose_outline", description: "Show a script outline as a table and start the build tracker.", parameters: { type: "object", properties: { sections: { type: "array", items: { type: "object", properties: { title: { type: "string" }, durationSec: { type: "number" }, treatment: { type: "string", enum: ["motion-graphic", "stock-footage", "generated-clip", "talking-head"] } }, required: ["title", "treatment"] } } }, required: ["sections"] } } }, | |
| { type: "function", function: { name: "set_section_status", description: "Mark a section (0-based index) active or done.", parameters: { type: "object", properties: { index: { type: "number" }, status: { type: "string", enum: ["pending", "active", "done"] } }, required: ["index", "status"] } } }, | |
| { type: "function", function: { name: "add_motion_scene", description: "Generate ONE content-aware, ART-DIRECTED motion-graphics scene from a template, filled with this video's actual topic, and place it on the beat it illustrates. Prefer this over add_scene.", parameters: { type: "object", properties: { template: { type: "string", enum: SCENE_TEMPLATE_IDS }, label: { type: "string", description: "2-4 word name" }, durationSec: { type: "number" }, props: { type: "object", description: "Template props filled with specific on-topic copy — see the template list in the system prompt for each template's props." }, theme: { type: "string", enum: THEME_IDS, description: "color theme to fit the mood" }, accent: { type: "string", description: "#hex accent from the palette" }, bg: { type: "string", enum: MG_BACKGROUND_IDS, description: "backdrop look" }, anim: { type: "string", enum: MG_ANIMATION_IDS, description: "entrance animation" }, font: { type: "string", enum: MG_FONT_IDS, description: "type system" }, sfx: { type: "string", enum: [...MG_SFX_IDS, "none"], description: "sound effect at the scene start — match the motion (e.g. price→cash, social→notify, decode/AI→glitch, typing→type, hero→boom/whoosh, items→pop/tick); \"none\" = silent. Omit to use the template's default sound." }, anchor: { type: "string", description: "a 2-6 word phrase copied VERBATIM from the timed transcript — the scene starts when that line is spoken" }, atSec: { type: "number", description: "explicit start time in seconds (used if no anchor)" } }, required: ["template", "props"] } } }, | |
| { type: "function", function: { name: "analyze_video", description: "Analyse an uploaded clip: detect shot boundaries (scene cuts). Call before 'cut into scenes'. Returns the cut points (seconds) and stashes them so you can split_at them.", parameters: { type: "object", properties: { asset: { type: "string", description: "which clip — label, 'Video 1', or id; defaults to the first video in the project" } } } } }, | |
| { type: "function", function: { name: "split_at", description: "Razor every timeline clip at each given second — used to cut footage into scenes at the shot boundaries from analyze_video.", parameters: { type: "object", properties: { seconds: { type: "array", items: { type: "number" }, description: "timeline seconds to cut at" } }, required: ["seconds"] } } }, | |
| { type: "function", function: { name: "add_scene", description: "(fallback) Add a fixed stock Chronixel scene by id at the playhead.", parameters: { type: "object", properties: { sceneId: { type: "string" } }, required: ["sceneId"] } } }, | |
| { type: "function", function: { name: "cut_words", description: "Cut filler/unwanted words or phrases from the transcript.", parameters: { type: "object", properties: { phrases: { type: "array", items: { type: "string" } } }, required: ["phrases"] } } }, | |
| { type: "function", function: { name: "restore_words", description: "Restore all previously cut words.", parameters: { type: "object", properties: {} } } }, | |
| { type: "function", function: { name: "set_design_style", description: "Set the visual style for generated motion graphics.", parameters: { type: "object", properties: { styleId: { type: "string" } }, required: ["styleId"] } } }, | |
| { type: "function", function: { name: "set_music", description: "Choose the BACKGROUND MUSIC track that loops under the whole video, matched to the video's mood. \"none\" removes it.", parameters: { type: "object", properties: { music: { type: "string", enum: [...MG_MUSIC_IDS, "none"], description: "track by mood — " + MG_MUSIC.map((m) => `${m.id} (${m.genre}, ${m.mood})`).join("; ") } }, required: ["music"] } } }, | |
| { type: "function", function: { name: "list_scenes", description: "Read the current timeline scenes (id, index, template, props, theme, accent).", parameters: { type: "object", properties: {} } } }, | |
| { type: "function", function: { name: "update_scene", description: "Edit an existing scene. Prefer addressing by its stable id (from list_scenes / the state block) over index. Change props, theme, accent, durationSec, label, or the LOOK (bg/anim/font).", parameters: { type: "object", properties: { id: { type: "string", description: "stable scene id, e.g. m3 (preferred)" }, index: { type: "number", description: "0-based timeline index (fallback)" }, props: { type: "object", description: "Partial props to merge, e.g. {\"title\":\"New\"} or {\"value\":\"50%\"}" }, theme: { type: "string", enum: THEME_IDS }, accent: { type: "string", description: "#hex" }, bg: { type: "string", enum: MG_BACKGROUND_IDS, description: "backdrop style" }, anim: { type: "string", enum: MG_ANIMATION_IDS, description: "entrance animation" }, font: { type: "string", enum: MG_FONT_IDS, description: "type system" }, sfx: { type: "string", enum: [...MG_SFX_IDS, "none"], description: "change the scene's sound effect, or \"none\" to remove it" }, durationSec: { type: "number" }, label: { type: "string" } } } } }, | |
| { type: "function", function: { name: "remove_scene", description: "Delete a scene by its stable id (preferred) or 0-based index.", parameters: { type: "object", properties: { id: { type: "string" }, index: { type: "number" } } } } }, | |
| { type: "function", function: { name: "restyle", description: "Apply theme/accent and/or bg/anim/font to ALL generated scenes at once.", parameters: { type: "object", properties: { theme: { type: "string", enum: THEME_IDS }, accent: { type: "string", description: "#hex" }, bg: { type: "string", enum: MG_BACKGROUND_IDS }, anim: { type: "string", enum: MG_ANIMATION_IDS }, font: { type: "string", enum: MG_FONT_IDS } } } } }, | |
| { type: "function", function: { name: "place_asset", description: "Drop an imported asset from the ASSET LIBRARY onto the V1 timeline. Identify it by label, ordinal ('Video 1') or id.", parameters: { type: "object", properties: { asset: { type: "string", description: "label, 'Video 1'/'Image 2', or id" }, atStart: { type: "boolean", description: "true → place at 0s (begin the video with it)" }, startSec: { type: "number", description: "explicit start in seconds (overrides atStart)" }, durationSec: { type: "number", description: "optional clip length" } }, required: ["asset"] } } }, | |
| { type: "function", function: { name: "set_transition", description: "Add/clear an in-transition on a clip: a motion scene (target 'motion', by id/index) or a V1 video clip (target 'video', by id/index).", parameters: { type: "object", properties: { target: { type: "string", enum: ["motion", "video"], description: "which lane the clip is on (default motion)" }, id: { type: "string", description: "stable clip id (m… for a scene, vi… for a video) — preferred" }, index: { type: "number", description: "0-based index within the target lane (fallback)" }, type: { type: "string", enum: [...TRANSITION_IDS, "none"], description: "transition style; 'none' clears it" }, durationSec: { type: "number", description: "length in seconds (default 0.5)" } }, required: ["type"] } } }, | |
| { type: "function", function: { name: "add_track", description: "Add a timeline track (lane). A 'video' track stacks ON TOP of the others (it composites over lanes below); an 'audio' track is for voiceover/music and mixes in. New clips land on the most recently added track.", parameters: { type: "object", properties: { kind: { type: "string", enum: ["video", "audio"] }, name: { type: "string", description: "optional lane name, e.g. 'B-roll' or 'Music'" } }, required: ["kind"] } } }, | |
| { type: "function", function: { name: "split_clip", description: "Razor-split clips at the current playhead — cuts every clip the playhead crosses (or just the selected scenes) into two. Move the playhead first if needed.", parameters: { type: "object", properties: {} } } }, | |
| { type: "function", function: { name: "design_scene", description: "Design a BRAND-NEW custom motion-graphics scene from scratch when NO template fits the user's request — you author a JSON spec (background + animated layers; see CUSTOM SCENES in the system prompt) and it is rendered and placed on the timeline. Use this for any bespoke look the templates don't cover.", parameters: { type: "object", properties: { spec: { type: "object", description: "the scene spec: { duration, background, layers:[…] } following the CUSTOM SCENES schema" }, label: { type: "string", description: "2-4 word name for the scene" }, accent: { type: "string", description: "#hex accent the spec can reference via color:\"accent\"" }, theme: { type: "string", enum: THEME_IDS, description: "base color theme" }, anchor: { type: "string", description: "a 2-6 word phrase copied from the timed transcript — places the scene on that beat" }, atSec: { type: "number", description: "explicit start time in seconds (used if no anchor)" } }, required: ["spec"] } } }, | |
| { type: "function", function: { name: "suggest", description: "Offer 2-4 short next-step suggestion chips.", parameters: { type: "object", properties: { chips: { type: "array", items: { type: "string" } } }, required: ["chips"] } } }, | |
| ]; | |
| export type OutlineSection = { title: string; durationSec?: number; treatment: string }; | |
| export type AgentEvent = | |
| | { type: "assistant"; text: string } | |
| | { type: "outline"; sections: OutlineSection[] } | |
| | { type: "action"; text: string } | |
| | { type: "suggest"; chips: string[] }; | |
| // map the chosen design style → a template theme | |
| const STYLE_THEME: Record<string, string> = { "obsidian-orbit": "dark", "redline-tech": "night", atelier: "warm", "warm-paper": "warm", "orange-minimal": "warm", sketchbook: "warm" }; | |
| const VALID_TEMPLATES = TEMPLATE_IDS; | |
| const canonTemplate = (t: unknown) => canonicalTemplate(t); | |
| // resolve a scene by stable id (preferred) or a coerced, range-checked index | |
| function resolveScene<T extends { id: string }>(motion: T[], args: { id?: unknown; index?: unknown }): T | null { | |
| if (typeof args.id === "string") { const m = motion.find((x) => x.id === args.id); if (m) return m; } | |
| const idx = Number(args.index); | |
| if (Number.isInteger(idx) && idx >= 0 && idx < motion.length) return motion[idx]; | |
| return null; | |
| } | |
| // resolve a library asset by id, ordinal ("Video 1" / "Image 2"), label substring, or bare number | |
| function resolveAsset(assets: { id: string; label: string; kind?: string }[], q: unknown) { | |
| const raw = String(q ?? "").trim(); | |
| const s = raw.toLowerCase(); | |
| let a = assets.find((x) => x.id === raw); if (a) return a; | |
| const m = s.match(/(video|image|clip|asset)\s*#?\s*(\d+)/); | |
| if (m) { | |
| const n = parseInt(m[2], 10) - 1; | |
| const kind = m[1] === "image" ? "image" : m[1] === "video" ? "video" : null; | |
| if (kind) { const pool = assets.filter((x) => (x.kind || "video") === kind); if (pool[n]) return pool[n]; } // explicit kind → don't fall back to the other kind | |
| else if (assets[n]) return assets[n]; // "clip 2" / "asset 2" → positional across all kinds | |
| } | |
| a = assets.find((x) => x.label.toLowerCase() === s) || assets.find((x) => x.label.toLowerCase().includes(s) && s.length > 1); | |
| if (a) return a; | |
| if (/^\d+$/.test(s)) { const i = parseInt(s, 10) - 1; if (assets[i]) return assets[i]; } | |
| return null; | |
| } | |
| // resolve a transition target across both lanes — id wins, else (target + index) | |
| function resolveTransitionTarget(s2: ReturnType<typeof useStore.getState>, args: { id?: unknown; index?: unknown; target?: unknown }): { id: string; label: string; lane: "motion" | "video" } | null { | |
| const id = typeof args.id === "string" ? args.id : ""; | |
| if (id) { | |
| const m = s2.motion.find((x) => x.id === id); if (m) return { id: m.id, label: m.label, lane: "motion" }; | |
| const v = s2.videos.find((x) => x.id === id); if (v) return { id: v.id, label: v.label, lane: "video" }; | |
| } | |
| const tgt = String(args.target ?? "").toLowerCase(); | |
| const idx = Number(args.index); | |
| const wantVideo = tgt.startsWith("vid") || tgt === "v1" || tgt === "clip"; | |
| if (wantVideo) { | |
| // explicit video target → resolve a video clip or fail; never silently hit the motion lane | |
| if (Number.isInteger(idx) && idx >= 0 && s2.videos[idx]) { const v = s2.videos[idx]; return { id: v.id, label: v.label, lane: "video" }; } | |
| return null; | |
| } | |
| if (Number.isInteger(idx) && idx >= 0 && idx < s2.motion.length) { const m = s2.motion[idx]; return { id: m.id, label: m.label, lane: "motion" }; } | |
| if (Number.isInteger(idx) && idx >= 0 && s2.videos[idx]) { const v = s2.videos[idx]; return { id: v.id, label: v.label, lane: "video" }; } | |
| return null; | |
| } | |
| // Some models (Llama 3.x via Groq) emit tool calls as TEXT — `<function=NAME>{json}</function>` | |
| // — instead of structured tool_calls. Parse those so the agent still acts. Brace-matched | |
| // so nested JSON (e.g. props:{...}) is captured correctly. | |
| export function parseTextToolCalls(text: string): { name: string; args: string }[] { | |
| const calls: { name: string; args: string }[] = []; | |
| const re = /<function=([a-zA-Z_]+)\s*>/g; | |
| let m: RegExpExecArray | null; | |
| while ((m = re.exec(text))) { | |
| let i = re.lastIndex; | |
| while (i < text.length && text[i] !== "{") i++; | |
| if (i >= text.length) continue; | |
| let depth = 0, inStr = false, esc = false, end = -1; | |
| for (let j = i; j < text.length; j++) { | |
| const c = text[j]; | |
| if (inStr) { if (esc) esc = false; else if (c === "\\") esc = true; else if (c === '"') inStr = false; continue; } | |
| if (c === '"') inStr = true; else if (c === "{") depth++; else if (c === "}" && --depth === 0) { end = j; break; } | |
| } | |
| if (end < 0) continue; | |
| calls.push({ name: m[1], args: text.slice(i, end + 1) }); | |
| } | |
| return calls; | |
| } | |
| const stripFunctionCalls = (text: string) => text.replace(/<function=[\s\S]*$/, "").trim(); | |
| // dev/e2e hook: invoke the real tool-dispatch path (resolvers + store mutations) without the LLM | |
| if ((import.meta as { env?: { DEV?: boolean } }).env?.DEV && typeof window !== "undefined") { | |
| (window as unknown as { __agentTool: (n: string, a: unknown) => Promise<string> }).__agentTool = (n, a) => execTool(n, a, () => {}); | |
| } | |
| // accept a #hex (3/6 digit, with or without leading #) → normalized #hex, else undefined | |
| const hexAccent = (v: unknown): string | undefined => { const s = String(v ?? "").trim(); return /^#?[0-9a-fA-F]{6}$/.test(s) || /^#?[0-9a-fA-F]{3}$/.test(s) ? (s.startsWith("#") ? s : "#" + s) : undefined; }; | |
| const VALID_THEMES = new Set(THEME_IDS); | |
| async function execTool(name: string, args: any, push: (e: AgentEvent) => void): Promise<string> { | |
| const st = useStore.getState(); | |
| switch (name) { | |
| case "add_motion_scene": { | |
| const tpl = canonTemplate(args.template); | |
| if (!tpl) return `error: unknown template "${args.template}". Valid: ${VALID_TEMPLATES.join(", ")}`; | |
| // art direction: honor the agent's look choices, fall back to the design-style theme / brand accent | |
| const theme = typeof args.theme === "string" && VALID_THEMES.has(args.theme) ? args.theme : (STYLE_THEME[st.designStyle] || "dark"); | |
| const accent = hexAccent(args.accent) || "#E7723B"; | |
| const bg = typeof args.bg === "string" ? canonicalBackground(args.bg) || undefined : undefined; | |
| const anim = typeof args.anim === "string" ? canonicalAnimation(args.anim) || undefined : undefined; | |
| const font = typeof args.font === "string" ? canonicalFont(args.font) || undefined : undefined; | |
| const sfx = typeof args.sfx === "string" ? canonicalSfx(args.sfx) || undefined : undefined; // "none" → silent; unknown → template default | |
| // smart placement: anchor to the spoken phrase, else an explicit second, else append end-to-end | |
| let startMs: number | undefined; | |
| let where = "appended"; | |
| if (typeof args.anchor === "string" && args.anchor.trim()) { const t = st.timeForPhrase(args.anchor); if (t != null) { startMs = t; where = `@${(t / 1000).toFixed(1)}s ("${args.anchor.trim().slice(0, 30)}")`; } } | |
| if (startMs == null && typeof args.atSec === "number" && args.atSec >= 0) { startMs = Math.round(args.atSec * 1000); where = `@${args.atSec}s`; } | |
| st.addGeneratedScene({ template: tpl, props: args.props || {}, label: args.label || tpl, durationSec: args.durationSec, theme, accent, bg, anim, font, sfx }, startMs); | |
| push({ type: "action", text: `Generated ${tpl} scene${args.label ? ` “${args.label}”` : ""}${startMs != null ? ` ${where}` : ""}` }); | |
| return `added content-aware ${tpl} scene (${where}, theme=${theme} accent=${accent}${bg ? ` bg=${bg}` : ""}${font ? ` font=${font}` : ""})`; | |
| } | |
| case "analyze_video": { | |
| const s2 = useStore.getState(); | |
| // pick the clip: agent arg → matching asset/clip; else first video asset; else first placed video; else base audio | |
| let src = ""; let label = ""; | |
| if (typeof args.asset === "string" && args.asset.trim()) { | |
| const a = resolveAsset(s2.assets, args.asset); const full = a && s2.assets.find((x) => x.id === a.id); if (full) { src = full.src; label = full.label; } | |
| if (!src) { const v = s2.videos.find((x) => x.label.toLowerCase().includes(String(args.asset).toLowerCase())); if (v) { src = v.src; label = v.label; } } | |
| } | |
| if (!src) { const a = s2.assets.find((x) => (x.kind || "video") === "video"); if (a) { src = a.src; label = a.label; } } | |
| if (!src) { const v = s2.videos.find((x) => (x.kind || "video") === "video"); if (v) { src = v.src; label = v.label; } } | |
| if (!src && s2.audioSrc) { src = s2.audioSrc; label = "the source audio"; } | |
| if (!src) return "error: no video to analyse — import a clip in My Assets (or scan-to-upload from your phone) first"; | |
| try { | |
| const res = await fetch("/api/analyze", { method: "POST", headers: { "Content-Type": "application/json", ...aiHeaders() }, body: JSON.stringify({ src }) }); | |
| const d = await res.json(); | |
| if (!res.ok || !d || !Array.isArray(d.shots)) return `error: analysis failed — ${(d && d.error) || res.status}`; | |
| st.setShots(d.shots); | |
| const cuts = d.shots.map((s: { startMs: number }) => +(s.startMs / 1000).toFixed(1)).filter((x: number) => x > 0); | |
| push({ type: "action", text: `Analysed “${label}” — ${d.shots.length} scene${d.shots.length === 1 ? "" : "s"} detected` }); | |
| return `analysed "${label}": ${(d.durationMs / 1000).toFixed(1)}s, ${d.shots.length} shots. Cut points (s): ${cuts.length ? cuts.join(", ") : "(single continuous shot)"} — call split_at with these to cut it into scenes.`; | |
| } catch (e) { return `error: analysis failed — ${String((e as Error)?.message || e)}`; } | |
| } | |
| case "split_at": { | |
| const secs = Array.isArray(args.seconds) ? args.seconds.filter((n: unknown) => typeof n === "number") : []; | |
| if (!secs.length) return "error: no cut points provided"; | |
| const n = useStore.getState().splitAtSeconds(secs); | |
| push({ type: "action", text: n ? `Cut the footage into ${n + 1} scene${n === 0 ? "" : "s"} (${n} cut${n === 1 ? "" : "s"})` : "No clips to cut at those points" }); | |
| return `made ${n} cut(s) at ${secs.length} point(s)`; | |
| } | |
| case "propose_outline": | |
| st.captureBuildCheckpoint(); // snapshot pre-build state so the whole run can be reverted in one click | |
| st.setSections((args.sections || []).map((s: OutlineSection) => ({ title: s.title, treatment: s.treatment, durationSec: s.durationSec, status: "pending" as const }))); | |
| push({ type: "outline", sections: args.sections || [] }); | |
| return "outline displayed and tracker started"; | |
| case "set_section_status": | |
| st.setSectionStatus(args.index ?? 0, args.status); | |
| push({ type: "action", text: `Section ${(args.index ?? 0) + 1} → ${args.status}` }); | |
| return "status updated"; | |
| case "add_scene": { | |
| const def = sceneById(args.sceneId); | |
| if (!def) return `error: unknown sceneId "${args.sceneId}". Valid ids: ${CHRONIXEL_SCENES.map((s) => s.id).join(", ")}`; | |
| // append after the last scene so a build lays out end-to-end (not all stacked at 0) | |
| const nextStart = st.motion.reduce((mx, m) => Math.max(mx, m.startMs + m.durationMs), 0); | |
| st.addMotion(def.id, def.label, def.durationInFrames, nextStart); | |
| push({ type: "action", text: `Added scene “${def.label}” to the timeline` }); | |
| return `added ${def.id} at ${(nextStart / 1000).toFixed(1)}s`; | |
| } | |
| case "cut_words": { | |
| const n = st.cutPhrases(args.phrases || []); | |
| push({ type: "action", text: n ? `Cut ${n} word${n === 1 ? "" : "s"} from the transcript` : "No matching words found to cut" }); | |
| return `cut ${n} words`; | |
| } | |
| case "restore_words": | |
| st.restoreAll(); | |
| push({ type: "action", text: "Restored all cut words" }); | |
| return "restored"; | |
| case "set_design_style": | |
| st.setDesignStyle(args.styleId); | |
| push({ type: "action", text: `Design style set to “${args.styleId}”` }); | |
| return `style set ${args.styleId}`; | |
| case "set_music": { | |
| const mid = canonicalMusic(args.music); // "" = none | |
| useStore.getState().setMusic(mid); | |
| const mt = MG_MUSIC.find((x) => x.id === mid); | |
| push({ type: "action", text: mt ? `Background music → “${mt.label}”` : "Removed background music" }); | |
| return mt ? `set background music ${mid} (${mt.genre} · ${mt.mood})` : "background music cleared"; | |
| } | |
| case "list_scenes": | |
| return JSON.stringify(useStore.getState().motion.map((m, i) => ({ id: m.id, index: i, template: m.template || "stock", label: m.label, theme: m.theme, accent: m.accent, durationSec: +(m.durationMs / 1000).toFixed(1), props: m.props }))); | |
| case "update_scene": { | |
| const s2 = useStore.getState(); | |
| const item = resolveScene(s2.motion, args); | |
| if (!item) return `error: no scene matching ${args.id ?? `index ${args.index}`} (timeline has ${s2.motion.length} scenes; call list_scenes to refresh ids/indices)`; | |
| const patch: Record<string, unknown> = {}; | |
| if (typeof args.theme === "string") patch.theme = args.theme; | |
| if (typeof args.accent === "string") patch.accent = args.accent; | |
| if (typeof args.bg === "string") { const v = canonicalBackground(args.bg); if (v) patch.bg = v; } | |
| if (typeof args.anim === "string") { const v = canonicalAnimation(args.anim); if (v) patch.anim = v; } | |
| if (typeof args.font === "string") { const v = canonicalFont(args.font); if (v) patch.font = v; } | |
| if (typeof args.sfx === "string") { const v = canonicalSfx(args.sfx); if (v) patch.sfx = v; } // "none" → silent | |
| if (typeof args.durationSec === "number") patch.durationMs = Math.round(Math.max(1, args.durationSec) * 1000); | |
| if (typeof args.label === "string") patch.label = args.label; | |
| const hasProps = args.props && typeof args.props === "object"; | |
| if (!Object.keys(patch).length && !hasProps) return `error: nothing to change for scene ${item.id}`; | |
| s2.pushHistory(); | |
| if (Object.keys(patch).length) s2.updateMotion(item.id, patch); | |
| if (hasProps) s2.updateMotionProps(item.id, args.props); | |
| push({ type: "action", text: `Edited scene “${args.label || item.label}”` }); | |
| return `updated scene ${item.id}`; | |
| } | |
| case "remove_scene": { | |
| const s2 = useStore.getState(); | |
| const item = resolveScene(s2.motion, args); | |
| if (!item) return `error: no scene matching ${args.id ?? `index ${args.index}`}`; | |
| s2.removeMotion(item.id); | |
| push({ type: "action", text: `Removed scene “${item.label}”` }); | |
| return `removed scene ${item.id}`; | |
| } | |
| case "restyle": { | |
| const s2 = useStore.getState(); | |
| s2.pushHistory(); | |
| const bg = typeof args.bg === "string" ? canonicalBackground(args.bg) : ""; | |
| const anim = typeof args.anim === "string" ? canonicalAnimation(args.anim) : ""; | |
| const font = typeof args.font === "string" ? canonicalFont(args.font) : ""; | |
| let n = 0; | |
| for (const m of s2.motion) { | |
| if (!m.template) continue; | |
| const patch: Record<string, unknown> = {}; | |
| if (typeof args.theme === "string") patch.theme = args.theme; | |
| if (typeof args.accent === "string") patch.accent = args.accent; | |
| if (bg) patch.bg = bg; | |
| if (anim) patch.anim = anim; | |
| if (font) patch.font = font; | |
| if (Object.keys(patch).length) { s2.updateMotion(m.id, patch); n++; } | |
| } | |
| const bits = [args.theme && `→ ${args.theme}`, args.accent && `(${args.accent})`, bg && `bg:${bg}`, anim && `anim:${anim}`, font && `font:${font}`].filter(Boolean).join(" "); | |
| push({ type: "action", text: `Restyled ${n} scene${n === 1 ? "" : "s"} ${bits}`.trim() }); | |
| return `restyled ${n} scenes ${bits}`.trim(); | |
| } | |
| case "place_asset": { | |
| const s2 = useStore.getState(); | |
| if (!s2.assets.length) return "error: the asset library is empty — the user must import media in My Assets first"; | |
| const a = resolveAsset(s2.assets, args.asset); | |
| if (!a) return `error: no asset matching "${args.asset}". Library: ${s2.assets.map((x, i) => `${i + 1}. ${x.label} [${x.kind || "video"}]`).join("; ")}`; | |
| const start = typeof args.startSec === "number" ? Math.max(0, Math.round(args.startSec * 1000)) : (args.atStart ? 0 : undefined); | |
| const durationMs = typeof args.durationSec === "number" ? Math.round(args.durationSec * 1000) : undefined; | |
| const id = s2.placeAsset(a.id, start, durationMs); | |
| if (!id) return `error: could not place asset ${a.id}`; | |
| const atMs = start ?? s2.playheadMs; | |
| push({ type: "action", text: `Placed “${a.label}” on V1${start != null ? ` at ${(atMs / 1000).toFixed(1)}s` : ""}` }); | |
| return `placed "${a.label}" as clip ${id} at ${(atMs / 1000).toFixed(1)}s`; | |
| } | |
| case "set_transition": { | |
| const s2 = useStore.getState(); | |
| const clear = String(args.type ?? "").toLowerCase().replace(/[^a-z]/g, "") === "none"; | |
| const type = clear ? "" : canonicalTransition(args.type); | |
| if (!clear && !type) return `error: unknown transition "${args.type}". Valid: ${TRANSITION_IDS.join(", ")} (or "none")`; | |
| const tgt = resolveTransitionTarget(s2, args); | |
| if (!tgt) return `error: no clip matching ${args.id ?? `${args.target ?? "motion"} #${args.index}`} — see the LIVE PROJECT STATE for ids`; | |
| const t = clear ? null : { type, durationMs: typeof args.durationSec === "number" ? Math.max(100, Math.round(args.durationSec * 1000)) : 500 }; | |
| s2.setTransition(tgt.id, t); | |
| push({ type: "action", text: t ? `${type} transition → “${tgt.label}”` : `Cleared transition on “${tgt.label}”` }); | |
| return t ? `set ${type} transition (${(t.durationMs / 1000).toFixed(1)}s) on ${tgt.lane} clip ${tgt.id}` : `cleared transition on ${tgt.id}`; | |
| } | |
| case "add_track": { | |
| const kind = args.kind === "audio" ? "audio" : "video"; | |
| const id = useStore.getState().addTrack(kind, typeof args.name === "string" ? args.name : undefined); | |
| const name = useStore.getState().tracks.find((t) => t.id === id)?.name || kind; | |
| push({ type: "action", text: `Added ${kind} track “${name}”` }); | |
| return `added ${kind} track ${id} (${name})`; | |
| } | |
| case "split_clip": { | |
| const n = useStore.getState().splitAtPlayhead(); | |
| push({ type: "action", text: n ? `Split ${n} clip${n === 1 ? "" : "s"} at the playhead` : "Nothing to split at the playhead" }); | |
| return `split ${n} clip(s)`; | |
| } | |
| case "design_scene": { | |
| const spec = args.spec && typeof args.spec === "object" ? args.spec : null; | |
| const is3d = !!(spec && spec.scene3d === true); | |
| const ok = !!spec && (is3d ? Array.isArray(spec.objects) && (spec.objects.length || (Array.isArray(spec.texts) && spec.texts.length)) : Array.isArray(spec.layers) && spec.layers.length); | |
| if (!ok) return "error: design_scene needs a spec — either a 2D spec with a non-empty \"layers\" array, OR a 3D spec with \"scene3d\":true and an \"objects\" array (see CUSTOM SCENES in the system prompt)."; | |
| // bake uploaded-asset image refs ("asset:Video 1") into concrete /media URLs so preview + export match (2D image layers) | |
| if (Array.isArray(spec.layers)) for (const L of spec.layers) { | |
| if (L && L.type === "image" && typeof L.src === "string" && L.src.startsWith("asset:")) { | |
| const a = resolveAsset(st.assets, L.src.slice(6)); | |
| const full = a && st.assets.find((x) => x.id === a.id); | |
| L.src = full ? "/media/" + full.src : ""; | |
| } | |
| } | |
| const count = is3d ? spec.objects.length : spec.layers.length; | |
| const unit = is3d ? "objects" : "layers"; | |
| const durationSec = typeof spec.duration === "number" && spec.duration > 0 ? Math.min(20, spec.duration) : (is3d ? 6 : 5); | |
| const theme = typeof args.theme === "string" && VALID_THEMES.has(args.theme) ? args.theme : "dark"; | |
| const accent = hexAccent(args.accent) || "#7C5CFF"; | |
| let startMs: number | undefined; | |
| let where = "appended"; | |
| if (typeof args.anchor === "string" && args.anchor.trim()) { const t = st.timeForPhrase(args.anchor); if (t != null) { startMs = t; where = `@${(t / 1000).toFixed(1)}s`; } } | |
| if (startMs == null && typeof args.atSec === "number" && args.atSec >= 0) { startMs = Math.round(args.atSec * 1000); where = `@${args.atSec}s`; } | |
| st.addGeneratedScene({ template: "custom", props: { spec }, label: args.label || "Custom scene", durationSec, theme, accent, bg: "solid", anim: "none" }, startMs); | |
| push({ type: "action", text: `Designed a custom ${is3d ? "3D " : ""}scene${args.label ? ` “${args.label}”` : ""} (${count} ${unit}${startMs != null ? `, ${where}` : ""})` }); | |
| return `created custom ${is3d ? "3D " : ""}scene "${args.label || "Custom scene"}" with ${count} ${unit} (${durationSec}s)`; | |
| } | |
| case "suggest": | |
| push({ type: "suggest", chips: args.chips || [] }); | |
| return "suggestions shown"; | |
| default: | |
| return `unknown tool ${name}`; | |
| } | |
| } | |
| export type ChatMsg = { role: "user" | "assistant"; content: string }; | |
| /** Runs the tool-calling loop against the chosen model, applying edits to the store. */ | |
| export async function runAgent(history: ChatMsg[], model: string, push: (e: AgentEvent) => void) { | |
| const msgs: any[] = [{ role: "system", content: SYSTEM_PROMPT() }, ...history]; | |
| for (let i = 0; i < 16; i++) { | |
| const res = await fetch("/api/chat", { | |
| method: "POST", | |
| headers: { "Content-Type": "application/json", ...aiHeaders() }, | |
| body: JSON.stringify({ model, messages: msgs, tools: AGENT_TOOLS, toolChoice: "auto", maxTokens: 1600 }), | |
| }); | |
| const d = await res.json(); | |
| if (d.error) { push({ type: "assistant", text: "⚠️ " + d.error }); return; } | |
| const msg = d.raw?.choices?.[0]?.message; | |
| const toolCalls = msg?.tool_calls; | |
| if (toolCalls?.length) { | |
| msgs.push(msg); | |
| for (const tc of toolCalls) { | |
| let args: any = {}; | |
| try { args = JSON.parse(tc.function.arguments || "{}"); } catch {} | |
| const result = await execTool(tc.function.name, args, push); | |
| msgs.push({ role: "tool", tool_call_id: tc.id, content: String(result) }); | |
| } | |
| continue; | |
| } | |
| const text = (d.content || "").trim(); | |
| if (text.includes("<function=")) { | |
| const textCalls = parseTextToolCalls(text); | |
| if (textCalls.length) { | |
| const cleaned = stripFunctionCalls(text); | |
| if (cleaned) push({ type: "assistant", text: cleaned }); | |
| msgs.push({ role: "assistant", content: text }); | |
| for (const t of textCalls) { let a: any = {}; try { a = JSON.parse(t.args || "{}"); } catch {} const result = await execTool(t.name, a, push); msgs.push({ role: "user", content: `(tool ${t.name} → ${result})` }); } | |
| continue; | |
| } | |
| } | |
| if (text) push({ type: "assistant", text }); | |
| else if (d.reasoning) push({ type: "assistant", text: d.reasoning.slice(0, 400) }); | |
| return; | |
| } | |
| push({ type: "assistant", text: "(stopped after several steps)" }); | |
| } | |
| export type BuildOutcome = "complete" | "stopped" | "timeout" | "capped" | "error"; | |
| export interface StreamCb { | |
| onStart: () => void; // a streamed assistant message begins | |
| onToken: (t: string) => void; // append a token to it | |
| onEnd: () => void; // it's complete | |
| onEvent: (e: AgentEvent) => void; // discrete events (action / outline / suggest) | |
| onReplaceLast?: (text: string) => void; // rewrite the last streamed bubble (e.g. strip <function> text) | |
| signal?: AbortSignal; // abort the whole run (the Stop button) | |
| onDone?: (outcome: BuildOutcome) => void; // how the run ended — lets the caller reconcile the build tracker | |
| } | |
| // if a streamed turn produces no new bytes for this long, treat the connection as hung and abort the turn | |
| // (the real culprit behind a build that "spins forever" on a flaky free-tier model connection) | |
| const TURN_IDLE_MS = 45000; | |
| /** Streaming variant: streams assistant text token-by-token, runs the tool loop. | |
| * Cancelable via cb.signal (Stop) and self-healing against hung connections via a per-turn idle | |
| * watchdog. Always reports how it ended through cb.onDone so the caller can reconcile the tracker. */ | |
| export async function runAgentStream(history: ChatMsg[], model: string, cb: StreamCb) { | |
| const msgs: any[] = [{ role: "system", content: SYSTEM_PROMPT() }, ...history]; | |
| const ext = cb.signal; | |
| const finish = (o: BuildOutcome) => { cb.onDone?.(o); }; | |
| for (let turn = 0; turn < 16; turn++) { | |
| if (ext?.aborted) { finish("stopped"); return; } | |
| // per-turn controller: trips on the user's Stop OR on the inactivity watchdog (a stalled stream) | |
| const turnCtrl = new AbortController(); | |
| let timedOut = false; | |
| const onExt = () => turnCtrl.abort(); | |
| ext?.addEventListener("abort", onExt, { once: true }); | |
| let idle: ReturnType<typeof setTimeout> | undefined; | |
| const bumpIdle = () => { if (idle) clearTimeout(idle); idle = setTimeout(() => { timedOut = true; turnCtrl.abort(); }, TURN_IDLE_MS); }; | |
| try { | |
| bumpIdle(); | |
| const res = await fetch("/api/chat/stream", { | |
| method: "POST", | |
| headers: { "Content-Type": "application/json", ...aiHeaders() }, | |
| body: JSON.stringify({ model, messages: msgs, tools: AGENT_TOOLS, toolChoice: "auto", maxTokens: 1600 }), | |
| signal: turnCtrl.signal, | |
| }); | |
| if (!res.ok || !res.body) { | |
| const t = await res.text().catch(() => String(res.status)); | |
| const rl = /rate.?limit|429|tokens per minute|TPM/i.test(t); | |
| cb.onEvent({ type: "assistant", text: rl | |
| ? "⚠️ Hit the model's rate limit (free Groq tier = 12k tokens/min). I auto-retry these, but it's still capped — wait a few seconds and tap **Resume**, or switch to a lighter model like **Llama 3.1 8B Instant** in the model picker to fit more in the budget." | |
| : "⚠️ " + t.slice(0, 240) }); | |
| finish("error"); return; | |
| } | |
| const reader = res.body.getReader(); | |
| const dec = new TextDecoder(); | |
| let buf = "", text = "", started = false, sawReasoning = false; | |
| const toolMap: Record<number, { id: string; name: string; args: string }> = {}; | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) break; | |
| bumpIdle(); // got bytes → the connection is alive; reset the watchdog | |
| buf += dec.decode(value, { stream: true }); | |
| let nl; | |
| while ((nl = buf.indexOf("\n")) >= 0) { | |
| const line = buf.slice(0, nl).trim(); | |
| buf = buf.slice(nl + 1); | |
| if (!line.startsWith("data:")) continue; | |
| const data = line.slice(5).trim(); | |
| if (data === "[DONE]" || !data) continue; | |
| let json: any; try { json = JSON.parse(data); } catch { continue; } | |
| const delta = json.choices?.[0]?.delta || {}; | |
| if (delta.content) { if (!started) { started = true; cb.onStart(); } text += delta.content; cb.onToken(delta.content); } | |
| if (delta.reasoning_content) sawReasoning = true; | |
| if (delta.tool_calls) { | |
| for (const tc of delta.tool_calls) { | |
| const i = tc.index ?? 0; | |
| toolMap[i] ||= { id: tc.id || `call_${i}`, name: "", args: "" }; | |
| if (tc.id) toolMap[i].id = tc.id; | |
| if (tc.function?.name) toolMap[i].name += tc.function.name; | |
| if (tc.function?.arguments) toolMap[i].args += tc.function.arguments; | |
| } | |
| } | |
| } | |
| } | |
| if (started) cb.onEnd(); | |
| const toolCalls = Object.values(toolMap).filter((t) => t.name); | |
| if (toolCalls.length) { | |
| msgs.push({ role: "assistant", content: text || null, tool_calls: toolCalls.map((t) => ({ id: t.id, type: "function", function: { name: t.name, arguments: t.args } })) }); | |
| for (const t of toolCalls) { | |
| if (ext?.aborted) { finish("stopped"); return; } // honor Stop between tool calls | |
| let args: any = {}; try { args = JSON.parse(t.args || "{}"); } catch {} | |
| const result = await execTool(t.name, args, cb.onEvent); | |
| msgs.push({ role: "tool", tool_call_id: t.id, content: String(result) }); | |
| } | |
| continue; | |
| } | |
| // fallback: model emitted tool calls as <function=…> TEXT rather than structured tool_calls | |
| if (text.includes("<function=")) { | |
| const textCalls = parseTextToolCalls(text); | |
| if (textCalls.length) { | |
| cb.onReplaceLast?.(stripFunctionCalls(text)); // hide the raw function text in the UI | |
| msgs.push({ role: "assistant", content: text }); | |
| for (const t of textCalls) { | |
| if (ext?.aborted) { finish("stopped"); return; } | |
| let args: any = {}; try { args = JSON.parse(t.args || "{}"); } catch {} | |
| const result = await execTool(t.name, args, cb.onEvent); | |
| msgs.push({ role: "user", content: `(tool ${t.name} → ${result})` }); | |
| } | |
| continue; | |
| } | |
| } | |
| if (!started && sawReasoning) { cb.onStart(); cb.onToken("(this reasoning model returned no final text — switch to a non-reasoning model like Llama 3.3 70B for tool use)"); cb.onEnd(); } | |
| finish("complete"); return; // the model answered with no further tool calls → the run is done | |
| } catch (e) { | |
| if (ext?.aborted && !timedOut) { finish("stopped"); return; } // user pressed Stop | |
| if (timedOut) { | |
| cb.onEvent({ type: "assistant", text: "⚠️ The model stalled on this step, so I stopped waiting. Tap **Resume** to pick the build back up." }); | |
| finish("timeout"); return; | |
| } | |
| cb.onEvent({ type: "assistant", text: "⚠️ Connection problem — " + String((e as Error)?.message || e).slice(0, 180) + ". Tap **Resume** to continue." }); | |
| finish("error"); return; | |
| } finally { | |
| if (idle) clearTimeout(idle); | |
| ext?.removeEventListener("abort", onExt); | |
| } | |
| } | |
| // hit the turn cap without the model wrapping up — surface it instead of silently leaving a spinner | |
| cb.onEvent({ type: "assistant", text: "Paused after several build steps. Tap **Resume** to keep building, or **Start over** from the chat." }); | |
| finish("capped"); | |
| } | |