import React, { useState, useCallback, useEffect, useRef } from 'react'; import { motion } from 'framer-motion'; import { useGeneration } from '@/context/GenerationContext'; import type { GenerationInputs, VideoProvider, GeneratedVideo, VeoSegment } from '@/types'; import { SparklesIcon, ArrowLeftIcon, ImageIcon } from './Icons'; import { generatePrompts, uploadImage, klingGenerate, klingExtend, waitForKlingVideo, generateVideoWithRetry, downloadVideo, getVideoDuration, generateThumbnails, replicateGenerate, waitForReplicateVideo, whisperAnalyzeAndExtract } from '@/utils/api'; interface GenerationFormProps { provider: VideoProvider; onBack: () => void; } const voiceTypes = ['Deep', 'Warm', 'Crisp', 'None']; const energyLevels = ['Low', 'Medium', 'High']; const cameraStyles = ['Standard', 'Handheld', 'Steadicam', 'FPV Drone']; const narrativeStyles = ['Standard', 'Documentary', 'Action', 'Introspective']; const aspectRatios = ['9:16', '16:9', '1:1']; // Generation modes type GenerationMode = 'extend' | 'frame-continuity'; export const GenerationForm: React.FC = ({ provider, onBack }) => { const { startGeneration, updateProgress, addVideo, setStep, setError, setRetryState, updateSegments, addTaskId, removeTaskId, state } = useGeneration(); const { retryState, generatedVideos, segments, isCancelling } = state; // Draft storage key const draftKey = `video-gen-draft-${provider}`; // Load draft on mount - initialize state from localStorage const loadDraft = useCallback(() => { try { const savedDraft = localStorage.getItem(draftKey); if (savedDraft) { const draft = JSON.parse(savedDraft); return draft; } } catch (error) { console.warn('Failed to load draft:', error); } return null; }, [draftKey]); const draft = loadDraft(); const [draftRestored, setDraftRestored] = useState(!!draft); const [formState, setFormState] = useState( draft?.formState || { script: '', style: '', voiceType: 'Deep', energyLevel: 'Medium', cameraStyle: 'Standard', narrativeStyle: 'Standard', seedValue: 12005, aspectRatio: '9:16', model: provider === 'kling' ? 'veo3_fast' : 'google/veo-3', } ); const [imageFile, setImageFile] = useState(null); const [imagePreview, setImagePreview] = useState(draft?.imagePreview || null); const [isDragging, setIsDragging] = useState(false); const [isGenerating, setIsGenerating] = useState(false); // Generation mode selection const [generationMode, setGenerationMode] = useState(draft?.generationMode || 'frame-continuity'); // Retry editing state const [retryDialogue, setRetryDialogue] = useState(''); const [retryEnvironment, setRetryEnvironment] = useState(''); const [retryAction, setRetryAction] = useState(''); // Initialize retry fields when error occurs useEffect(() => { if (retryState && segments[retryState.failedSegmentIndex]) { const seg = segments[retryState.failedSegmentIndex]; setRetryDialogue(seg.action_timeline?.dialogue || ''); setRetryEnvironment(seg.scene_continuity?.environment || ''); setRetryAction(seg.character_description?.current_state || ''); } }, [retryState, segments]); const handleRetrySubmit = () => { if (!retryState) return; const idx = retryState.failedSegmentIndex; const updatedSegments = [...segments]; // Update the segment with edited values if (updatedSegments[idx]) { updatedSegments[idx] = { ...updatedSegments[idx], action_timeline: { ...updatedSegments[idx].action_timeline, dialogue: retryDialogue }, scene_continuity: { ...updatedSegments[idx].scene_continuity, environment: retryEnvironment }, character_description: { ...updatedSegments[idx].character_description, current_state: retryAction } }; updateSegments(updatedSegments); } // Clear error and resume setRetryState(null); setStep('generating_video'); setIsGenerating(true); // Resume generation based on provider if (provider === 'kling') { if (generationMode === 'frame-continuity') { handleKlingFrameContinuityFlow(); } else { handleKlingExtendFlow(); } } else { handleReplicateGeneration(); } }; const handleCancelRetry = () => { setRetryState(null); setIsGenerating(false); }; // Show notification if draft was restored useEffect(() => { if (draftRestored) { console.log('📝 Draft restored from localStorage'); // Auto-hide notification after 5 seconds const timer = setTimeout(() => setDraftRestored(false), 5000); return () => clearTimeout(timer); } }, [draftRestored]); // Save draft whenever formState, imagePreview, or generationMode changes // Skip saving on initial mount to avoid overwriting with default values const isInitialMount = useRef(true); useEffect(() => { if (isInitialMount.current) { isInitialMount.current = false; return; } try { const draft = { formState, imagePreview, generationMode, savedAt: new Date().toISOString(), }; localStorage.setItem(draftKey, JSON.stringify(draft)); } catch (error) { console.warn('Failed to save draft:', error); } }, [formState, imagePreview, generationMode, draftKey]); // Clear draft function const clearDraft = useCallback(() => { try { localStorage.removeItem(draftKey); setDraftRestored(false); console.log('🗑️ Draft cleared'); } catch (error) { console.warn('Failed to clear draft:', error); } }, [draftKey]); // Calculate estimated segments const wordCount = formState.script.trim().split(/\s+/).filter(w => w).length; const estimatedSegments = wordCount > 0 ? Math.max(1, Math.min(Math.ceil(wordCount / 17), 10)) : 0; // Handle input changes const handleChange = (e: React.ChangeEvent) => { const { name, value } = e.target; setFormState(prev => ({ ...prev, [name]: value })); }; // Handle image upload const handleImageUpload = useCallback((file: File) => { if (file.type.startsWith('image/')) { setImageFile(file); const reader = new FileReader(); reader.onloadend = () => setImagePreview(reader.result as string); reader.readAsDataURL(file); } }, []); // Drag and drop handlers const handleDragOver = (e: React.DragEvent) => { e.preventDefault(); setIsDragging(true); }; const handleDragLeave = () => setIsDragging(false); const handleDrop = (e: React.DragEvent) => { e.preventDefault(); setIsDragging(false); const file = e.dataTransfer.files[0]; if (file) handleImageUpload(file); }; // Extract last frame from video blob const extractLastFrame = async (videoBlob: Blob): Promise => { return new Promise((resolve, reject) => { const video = document.createElement('video'); video.preload = 'metadata'; video.muted = true; video.src = URL.createObjectURL(videoBlob); video.onloadedmetadata = async () => { // Seek to near the end of the video const targetTime = Math.max(0, video.duration - 0.1); video.currentTime = targetTime; }; video.onseeked = () => { // Create canvas and draw current frame const canvas = document.createElement('canvas'); canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext('2d'); if (!ctx) { URL.revokeObjectURL(video.src); reject(new Error('Could not get canvas context')); return; } ctx.drawImage(video, 0, 0); // Convert to blob then to file canvas.toBlob((blob) => { URL.revokeObjectURL(video.src); if (!blob) { reject(new Error('Could not extract frame')); return; } const file = new File([blob], `frame-${Date.now()}.jpg`, { type: 'image/jpeg' }); resolve(file); }, 'image/jpeg', 0.95); }; video.onerror = () => { URL.revokeObjectURL(video.src); reject(new Error('Failed to load video for frame extraction')); }; }); }; // ============================================ // KIE GENERATION - FRAME CONTINUITY FLOW // ============================================ // This mirrors the Replicate flow from standalone_video_creator.py: // 1. Generate first video with original reference image // 2. Extract last frame using the whisper analysis from generated video // 3. Use that frame as reference for next segment // 4. Repeat for all segments const handleKlingFrameContinuityFlow = async () => { if (!imageFile || !formState.script.trim()) return; setIsGenerating(true); setError(null); try { // Step 1: Generate prompts using GPT-4o updateProgress('Analyzing script with GPT-4o...'); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); formData.append('image', imageFile); const payload = await generatePrompts(formData); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } const segments = payload.segments; updateProgress(`Generated ${segments.length} segments. Starting video generation...`); startGeneration(segments); // Track current reference image (starts with original) let currentImageFile = imageFile; const generatedVideos: GeneratedVideo[] = []; // Step 2: Generate videos segment by segment with frame continuity for (let i = 0; i < segments.length; i++) { const segment = segments[i]; const isLastSegment = i === segments.length - 1; updateProgress( `Generating video ${i + 1} of ${segments.length}...${i > 0 ? ' (using last frame from previous)' : ''}`, i, segments.length ); // Upload current reference image updateProgress(`Uploading reference image for segment ${i + 1}...`); const uploadResult = await uploadImage(currentImageFile); const hostedImageUrl = uploadResult.url; console.log(`🖼️ Segment ${i + 1} using image: ${i === 0 ? 'original' : 'last frame from previous'}`); // Generate video with current reference image updateProgress(`Submitting segment ${i + 1} to KIE Veo 3.1...`); const generateResult = await klingGenerate({ prompt: segment, imageUrls: [hostedImageUrl], model: 'veo3_fast', aspectRatio: formState.aspectRatio, generationType: 'FIRST_AND_LAST_FRAMES_2_VIDEO', seeds: formState.seedValue, voiceType: formState.voiceType, }); // Wait for completion updateProgress(`Processing video ${i + 1}... (this may take 1-2 minutes)`); const videoUrl = await waitForKlingVideo(generateResult.taskId); // Download video updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); // Get video duration const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); // Use Whisper to find optimal trim point, extract frame, and get transcription let trimPoint = duration; // Default to full duration let transcribedText = ''; // What Whisper actually heard if (!isLastSegment) { updateProgress(`Analyzing video ${i + 1} with Whisper for optimal continuity...`); try { // Get dialogue from segment for Whisper analysis const dialogue = segment.action_timeline?.dialogue || ''; const whisperResult = await whisperAnalyzeAndExtract({ video_url: videoUrl, dialogue: dialogue, buffer_time: 0.3, model_size: 'base' }); if (whisperResult.success && whisperResult.frame_base64) { // Convert base64 frame to File for next segment const base64Data = whisperResult.frame_base64.split(',')[1] || whisperResult.frame_base64; const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); for (let j = 0; j < byteCharacters.length; j++) { byteNumbers[j] = byteCharacters.charCodeAt(j); } const byteArray = new Uint8Array(byteNumbers); const frameBlob = new Blob([byteArray], { type: 'image/jpeg' }); currentImageFile = new File([frameBlob], `whisper-frame-${i + 1}.jpg`, { type: 'image/jpeg' }); // Store trim point for later merge if (whisperResult.trim_point) { trimPoint = whisperResult.trim_point; } // Store transcribed text for prompt refinement if (whisperResult.transcribed_text) { transcribedText = whisperResult.transcribed_text; console.log(`📝 Whisper transcription: "${transcribedText.substring(0, 100)}..."`); } console.log(`✅ Whisper: Last word at ${whisperResult.last_word_timestamp?.toFixed(2)}s, frame at ${whisperResult.frame_timestamp?.toFixed(2)}s, trim at ${trimPoint.toFixed(2)}s`); // REFINE NEXT SEGMENT PROMPT with frame + transcription const nextSegment = segments[i + 1]; if (nextSegment && currentImageFile) { updateProgress(`Refining segment ${i + 2} prompt with visual and audio context...`); try { const { refinePromptWithContext } = await import('@/utils/api'); const refined = await refinePromptWithContext( nextSegment, currentImageFile, transcribedText, dialogue ); // Update the next segment with refined prompt segments[i + 1] = refined.refined_prompt as typeof nextSegment; console.log(`✅ Refined segment ${i + 2} prompt for consistency`); } catch (refineError) { console.warn(`⚠️ Prompt refinement failed, using original:`, refineError); } } } else { // Fallback to simple last frame extraction console.log(`⚠️ Whisper failed (${whisperResult.error}), falling back to last frame extraction`); const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } } catch (frameError) { console.error(`⚠️ Whisper analysis failed, using fallback:`, frameError); try { const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } catch { // Continue with current image if all extraction fails } } } // Add to generated videos with trim metadata const generatedVideo: GeneratedVideo = { id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, trimPoint, // Store trim point for merge }; generatedVideos.push(generatedVideo); addVideo(generatedVideo); updateProgress(`Completed video ${i + 1} of ${segments.length}`, i + 1, segments.length); } // All done! clearDraft(); // Clear draft on successful generation clearDraft(); // Clear draft on successful generation setStep('completed'); updateProgress('All videos generated successfully!'); } catch (err) { console.error('Generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Generation failed'; // Enable retry mode setRetryState({ failedSegmentIndex: generatedVideos.length, // Current segment that failed error: errorMessage }); setStep('configuring'); // Go back to form, but with retry overlay } finally { setIsGenerating(false); } }; // ============================================ // KIE GENERATION - EXTEND API FLOW // ============================================ // Original flow using KIE's extend API const handleKlingExtendFlow = async () => { if (!imageFile || !formState.script.trim()) return; setIsGenerating(true); setError(null); try { // Step 1: Generate prompts using GPT-4o updateProgress('Analyzing script with GPT-4o...'); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); formData.append('image', imageFile); // Use existing segments if retrying, otherwise generate new ones let payload: { segments: VeoSegment[] }; if (retryState && segments.length > 0) { // Retry mode: use existing segments (they may have been edited) payload = { segments }; updateProgress(`Using existing ${segments.length} segments for retry...`); } else { // Normal mode: generate new segments payload = await generatePrompts(formData); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } updateProgress(`Generated ${payload.segments.length} segments. Starting video generation...`); startGeneration(payload.segments); } // Step 2: Upload reference image once updateProgress('Uploading reference image...'); const uploadResult = await uploadImage(imageFile); const hostedImageUrl = uploadResult.url; // Step 3: Generate videos (resume from where we left off if retrying) const startIndex = generatedVideos.length; let currentTaskId: string | null = null; let currentImageUrl = hostedImageUrl; // Start with original image // If resuming, extract last frame from previous video for continuity if (startIndex > 0 && generatedVideos[startIndex - 1]?.blobUrl) { updateProgress(`Extracting last frame from segment ${startIndex} for continuity...`); try { const lastVideoBlob = await fetch(generatedVideos[startIndex - 1].blobUrl!).then(r => r.blob()); const lastFrameFile = await extractLastFrame(lastVideoBlob); const frameUploadResult = await uploadImage(lastFrameFile); currentImageUrl = frameUploadResult.url; updateProgress(`Using frame from segment ${startIndex} for segment ${startIndex + 1}...`); } catch (frameError) { console.warn('Failed to extract frame, using original image:', frameError); // Continue with original image } } for (let i = startIndex; i < payload.segments.length; i++) { const segment = payload.segments[i]; updateProgress(`Generating video ${i + 1} of ${payload.segments.length}...`, i, payload.segments.length); // Generate video with automatic retry (retries once on failure) updateProgress(`Processing video ${i + 1}... (this may take 1-2 minutes)`); // Check if cancelled if (isCancelling) { throw new Error('Generation cancelled by user'); } const videoUrl = await generateVideoWithRetry(async () => { if (i === 0 || (i === startIndex && startIndex > 0)) { // First segment OR resuming after failure: use generate API with current image const generateResult = await klingGenerate({ prompt: segment, imageUrls: [currentImageUrl], model: 'veo3_fast', aspectRatio: formState.aspectRatio, generationType: 'FIRST_AND_LAST_FRAMES_2_VIDEO', seeds: formState.seedValue, voiceType: formState.voiceType, }); currentTaskId = generateResult.taskId; addTaskId(currentTaskId); return generateResult; } else { // Subsequent segments: use extend API const extendResult = await klingExtend( currentTaskId!, segment, formState.seedValue, formState.voiceType ); currentTaskId = extendResult.taskId; addTaskId(currentTaskId); return extendResult; } }, 300000, (attempt) => { updateProgress(`Retrying video ${i + 1}... (attempt ${attempt}/2)`); }); // Download and save updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); addVideo({ id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, }); updateProgress(`Completed video ${i + 1} of ${payload.segments.length}`, i + 1, payload.segments.length); // Remove task ID after completion if (currentTaskId) { removeTaskId(currentTaskId); } } clearDraft(); // Clear draft on successful generation setStep('completed'); updateProgress('All videos generated successfully!'); } catch (err) { console.error('Generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Generation failed'; // If cancelled, don't show retry option if (errorMessage.includes('cancelled') || isCancelling) { setError('Generation cancelled by user'); setStep('error'); } else { // Enable retry mode setRetryState({ failedSegmentIndex: generatedVideos.length, // Current segment that failed error: errorMessage }); setStep('configuring'); // Go back to form, but with retry overlay } } finally { setIsGenerating(false); // Clean up any remaining task IDs state.activeTaskIds.forEach(taskId => removeTaskId(taskId)); } }; // ============================================ // REPLICATE GENERATION - FRAME CONTINUITY FLOW // ============================================ // This mirrors the approach from standalone_video_creator.py: // 1. Generate prompts using GPT-4o // 2. For each segment, generate video with current reference image // 3. Extract last frame from generated video // 4. Use that frame as reference for next segment // 5. Result: Perfect visual continuity across all segments const handleReplicateGeneration = async () => { if (!formState.script.trim()) return; setIsGenerating(true); setError(null); try { // Step 1: Generate prompts using GPT-4o // Note: Replicate can work without an image, but for consistency we encourage one updateProgress('Analyzing script with GPT-4o...'); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); // If image provided, include it for GPT-4o analysis if (imageFile) { formData.append('image', imageFile); } else { // Create a placeholder image for GPT-4o (it needs one for analysis) // In production, you might want to handle this differently const placeholderBlob = new Blob(['placeholder'], { type: 'image/jpeg' }); formData.append('image', placeholderBlob, 'placeholder.jpg'); } const payload = await generatePrompts(formData); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } const segments = payload.segments; updateProgress(`Generated ${segments.length} segments. Starting Replicate generation...`); startGeneration(segments); // Track current reference image (starts with original if provided) let currentImageFile = imageFile; const generatedVideos: GeneratedVideo[] = []; // Step 2: Generate videos segment by segment with frame continuity for (let i = 0; i < segments.length; i++) { const segment = segments[i]; const isLastSegment = i === segments.length - 1; updateProgress( `Generating video ${i + 1} of ${segments.length} with Replicate...${i > 0 ? ' (using last frame)' : ''}`, i, segments.length ); // Convert structured segment to text prompt for Replicate // Replicate models typically expect text prompts const textPrompt = convertSegmentToTextPrompt(segment); console.log(`🎬 Segment ${i + 1} prompt:`, textPrompt.substring(0, 100) + '...'); // Upload current reference image if available let imageUrl: string | undefined; if (currentImageFile) { updateProgress(`Uploading reference image for segment ${i + 1}...`); const uploadResult = await uploadImage(currentImageFile); imageUrl = uploadResult.url; console.log(`🖼️ Segment ${i + 1} using image: ${i === 0 ? 'original' : 'last frame from previous'}`); } // Generate video with Replicate updateProgress(`Submitting segment ${i + 1} to Replicate...`); const generateResult = await replicateGenerate({ prompt: textPrompt, imageUrl: imageUrl, model: formState.model || 'google/veo-3', aspectRatio: formState.aspectRatio, }); // Wait for completion (polling) updateProgress(`Processing video ${i + 1}... (this may take 2-5 minutes)`); const videoUrl = await waitForReplicateVideo(generateResult.id); // Download video updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); // Get video duration and thumbnails const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); // Use Whisper to find optimal trim point, extract frame, and get transcription // This is more accurate than extracting the very last frame let trimPoint = duration; // Default to full duration let transcribedText = ''; // What Whisper actually heard if (!isLastSegment) { updateProgress(`Analyzing video ${i + 1} with Whisper for optimal continuity...`); try { // Get dialogue from segment for Whisper analysis const dialogue = segment.action_timeline?.dialogue || textPrompt; const whisperResult = await whisperAnalyzeAndExtract({ video_url: videoUrl, dialogue: dialogue, buffer_time: 0.3, model_size: 'base' }); if (whisperResult.success && whisperResult.frame_base64) { // Convert base64 frame to File for next segment const base64Data = whisperResult.frame_base64.split(',')[1] || whisperResult.frame_base64; const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); for (let j = 0; j < byteCharacters.length; j++) { byteNumbers[j] = byteCharacters.charCodeAt(j); } const byteArray = new Uint8Array(byteNumbers); const frameBlob = new Blob([byteArray], { type: 'image/jpeg' }); currentImageFile = new File([frameBlob], `whisper-frame-${i + 1}.jpg`, { type: 'image/jpeg' }); // Store trim point for later merge if (whisperResult.trim_point) { trimPoint = whisperResult.trim_point; } // Store transcribed text for prompt refinement if (whisperResult.transcribed_text) { transcribedText = whisperResult.transcribed_text; console.log(`📝 Whisper transcription: "${transcribedText.substring(0, 100)}..."`); } console.log(`✅ Whisper: Last word at ${whisperResult.last_word_timestamp?.toFixed(2)}s, frame at ${whisperResult.frame_timestamp?.toFixed(2)}s, trim at ${trimPoint.toFixed(2)}s`); // REFINE NEXT SEGMENT PROMPT with frame + transcription const nextSegment = segments[i + 1]; if (nextSegment && currentImageFile) { updateProgress(`Refining segment ${i + 2} prompt with visual and audio context...`); try { const { refinePromptWithContext } = await import('@/utils/api'); const refined = await refinePromptWithContext( nextSegment, currentImageFile, transcribedText, dialogue ); // Update the next segment with refined prompt segments[i + 1] = refined.refined_prompt as typeof nextSegment; console.log(`✅ Refined segment ${i + 2} prompt for consistency`); } catch (refineError) { console.warn(`⚠️ Prompt refinement failed, using original:`, refineError); } } } else { // Fallback to simple last frame extraction console.log(`⚠️ Whisper failed (${whisperResult.error}), falling back to last frame extraction`); const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } } catch (frameError) { console.error(`⚠️ Whisper analysis failed, using fallback:`, frameError); try { const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } catch { // Continue with current image if all extraction fails } } } // Add to generated videos with trim metadata const generatedVideo: GeneratedVideo = { id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, trimPoint, // Store trim point for merge }; generatedVideos.push(generatedVideo); addVideo(generatedVideo); updateProgress(`Completed video ${i + 1} of ${segments.length}`, i + 1, segments.length); } // All done! setStep('completed'); updateProgress('All videos generated successfully with Replicate!'); } catch (err) { console.error('Replicate generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Replicate generation failed'; // Enable retry mode setRetryState({ failedSegmentIndex: generatedVideos.length, // Current segment that failed error: errorMessage }); setStep('configuring'); // Go back to form, but with retry overlay } finally { setIsGenerating(false); } }; // Helper: Convert structured segment JSON to text prompt for Replicate // Replicate models typically expect plain text, not structured JSON const convertSegmentToTextPrompt = (segment: VeoSegment): string => { const parts: string[] = []; // Extract dialogue const dialogue = segment.action_timeline?.dialogue; if (dialogue) { parts.push(`"${dialogue}"`); } // Extract character description const character = segment.character_description; if (character?.current_state) { parts.push(`Character: ${character.current_state}`); } // Extract scene description const scene = segment.scene_continuity; if (scene?.environment) { parts.push(`Scene: ${scene.environment}`); } if (scene?.lighting_state) { parts.push(`Lighting: ${scene.lighting_state}`); } if (scene?.camera_position) { parts.push(`Camera: ${scene.camera_position}`); } if (scene?.camera_movement) { parts.push(`Movement: ${scene.camera_movement}`); } // Extract synchronized actions const syncedActions = segment.action_timeline?.synchronized_actions; if (syncedActions) { const actionsList = Object.entries(syncedActions) .filter(([, value]) => value) .map(([key, value]) => `${key}: ${value}`) .join('; '); if (actionsList) { parts.push(`Actions: ${actionsList}`); } } // Add instruction to not include captions/subtitles parts.push('Do not include any captions, subtitles, or text overlays in the video'); return parts.join('. '); }; // Main submit handler const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); if (provider === 'kling') { if (generationMode === 'frame-continuity') { handleKlingFrameContinuityFlow(); } else { handleKlingExtendFlow(); } } else { handleReplicateGeneration(); } }; const isValid = provider === 'kling' ? !!imageFile && formState.script.trim().length > 0 : formState.script.trim().length > 0; return ( {/* Header */}

{provider === 'kling' ? 'KIE API' : 'Replicate'} Video Generation

{provider === 'kling' ? 'Generate professional UGC videos with AI-powered segmentation' : 'Create unique videos with open-source models' }

{/* Retry Modal */} {retryState && (

Generation Failed

Error at segment {retryState.failedSegmentIndex + 1}: {retryState.error}

Edit Segment {retryState.failedSegmentIndex + 1} to fix the issue: