Spaces:
Sleeping
Sleeping
| import type { | |
| VideoStatusResponse, | |
| SegmentsPayload, | |
| HealthStatus, | |
| ExtractedFrame, | |
| LoginRequest, | |
| LoginResponse, | |
| AuthUser | |
| } from '@/types'; | |
| const API_BASE = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000'; | |
| // Get auth token from localStorage | |
| function getAuthToken(): string | null { | |
| return localStorage.getItem('auth_token'); | |
| } | |
| // Set auth token in localStorage | |
| export function setAuthToken(token: string): void { | |
| localStorage.setItem('auth_token', token); | |
| } | |
| // Remove auth token from localStorage | |
| export function removeAuthToken(): void { | |
| localStorage.removeItem('auth_token'); | |
| } | |
| // Generic API request handler | |
| async function apiRequest<T>( | |
| path: string, | |
| options: RequestInit = {} | |
| ): Promise<T> { | |
| const url = path.startsWith('http') ? path : `${API_BASE}${path}`; | |
| const token = getAuthToken(); | |
| // Normalize headers to a plain object | |
| const headers: Record<string, string> = {}; | |
| // Convert Headers object or array to plain object | |
| if (options.headers) { | |
| if (options.headers instanceof Headers) { | |
| options.headers.forEach((value, key) => { | |
| headers[key] = value; | |
| }); | |
| } else if (Array.isArray(options.headers)) { | |
| options.headers.forEach(([key, value]) => { | |
| headers[key] = value; | |
| }); | |
| } else { | |
| Object.assign(headers, options.headers); | |
| } | |
| } | |
| // Add auth token if available | |
| if (token) { | |
| headers['Authorization'] = `Bearer ${token}`; | |
| } | |
| const response = await fetch(url, { | |
| ...options, | |
| headers, | |
| }); | |
| if (!response.ok) { | |
| // If unauthorized, clear token | |
| if (response.status === 401) { | |
| removeAuthToken(); | |
| } | |
| let errorMessage = `Request failed with status ${response.status}`; | |
| // Try to extract error message from response | |
| const contentType = response.headers.get('content-type'); | |
| const isJson = contentType && contentType.includes('application/json'); | |
| try { | |
| if (isJson) { | |
| const errorData = await response.json(); | |
| // Try multiple common error message fields | |
| errorMessage = errorData.detail || | |
| errorData.message || | |
| errorData.error || | |
| (typeof errorData === 'string' ? errorData : errorMessage); | |
| } else { | |
| const text = await response.text(); | |
| if (text && text.trim()) { | |
| errorMessage = text; | |
| } else { | |
| // Fall back to default message based on status code | |
| if (response.status === 401) { | |
| errorMessage = 'Incorrect username or password.'; | |
| } else if (response.status === 403) { | |
| errorMessage = 'Access forbidden.'; | |
| } else if (response.status === 404) { | |
| errorMessage = 'Resource not found.'; | |
| } else if (response.status >= 500) { | |
| errorMessage = 'Server error. Please try again later.'; | |
| } | |
| } | |
| } | |
| } catch (parseError) { | |
| // If parsing fails, use status-based default | |
| if (response.status === 401) { | |
| errorMessage = 'Incorrect username or password.'; | |
| } else if (response.status === 403) { | |
| errorMessage = 'Access forbidden.'; | |
| } else if (response.status === 404) { | |
| errorMessage = 'Resource not found.'; | |
| } else if (response.status >= 500) { | |
| errorMessage = 'Server error. Please try again later.'; | |
| } | |
| } | |
| const error = new Error(errorMessage); | |
| console.error('API Error:', { | |
| status: response.status, | |
| statusText: response.statusText, | |
| message: errorMessage, | |
| url: url, | |
| contentType: contentType | |
| }); | |
| throw error; | |
| } | |
| return response.json(); | |
| } | |
| // Health check | |
| export async function checkHealth(): Promise<HealthStatus> { | |
| return apiRequest<HealthStatus>('/health'); | |
| } | |
| // ==================== AUTHENTICATION ==================== | |
| export async function login(credentials: LoginRequest): Promise<LoginResponse> { | |
| const response = await apiRequest<LoginResponse>('/api/auth/login', { | |
| method: 'POST', | |
| headers: { | |
| 'Content-Type': 'application/json', | |
| }, | |
| body: JSON.stringify(credentials), | |
| }); | |
| // Store token | |
| if (response.access_token) { | |
| setAuthToken(response.access_token); | |
| } | |
| return response; | |
| } | |
| export async function verifyAuth(): Promise<AuthUser> { | |
| try { | |
| const response = await apiRequest<AuthUser>('/api/auth/verify'); | |
| return response; | |
| } catch { | |
| removeAuthToken(); | |
| return { username: '', authenticated: false }; | |
| } | |
| } | |
| export async function getCurrentUser(): Promise<AuthUser> { | |
| try { | |
| const response = await apiRequest<AuthUser>('/api/auth/me'); | |
| return response; | |
| } catch { | |
| removeAuthToken(); | |
| return { username: '', authenticated: false }; | |
| } | |
| } | |
| export function logout(): void { | |
| removeAuthToken(); | |
| } | |
| // ==================== KLING/KIE API ==================== | |
| export interface KlingGenerateParams { | |
| prompt: string | object; | |
| imageUrls?: string[]; | |
| model?: string; | |
| aspectRatio?: string; | |
| generationType?: string; | |
| seeds?: number; | |
| voiceType?: string; | |
| } | |
| export interface KlingGenerateResponse { | |
| taskId: string; | |
| status: string; | |
| } | |
| export async function klingGenerate(params: KlingGenerateParams): Promise<KlingGenerateResponse> { | |
| return apiRequest<KlingGenerateResponse>('/api/veo/generate', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify(params), | |
| }); | |
| } | |
| export async function klingExtend(taskId: string, prompt: string | object, seeds?: number, voiceType?: string): Promise<KlingGenerateResponse> { | |
| return apiRequest<KlingGenerateResponse>('/api/veo/extend', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify({ taskId, prompt, seeds, voiceType }), | |
| }); | |
| } | |
| export async function klingGetStatus(taskId: string): Promise<VideoStatusResponse> { | |
| return apiRequest<VideoStatusResponse>(`/api/veo/status/${taskId}`); | |
| } | |
| export async function klingCancel(taskId: string): Promise<{ code: number; msg: string; taskId: string }> { | |
| return apiRequest<{ code: number; msg: string; taskId: string }>(`/api/veo/cancel/${taskId}`, { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| }); | |
| } | |
| export function createKlingEventSource(taskId: string): EventSource { | |
| return new EventSource(`${API_BASE}/api/veo/events/${taskId}`); | |
| } | |
| // ==================== REPLICATE API ==================== | |
| export interface ReplicateGenerateParams { | |
| prompt: string; | |
| imageUrl?: string; | |
| model?: string; | |
| duration?: number; | |
| aspectRatio?: string; | |
| } | |
| export interface ReplicateGenerateResponse { | |
| id: string; | |
| status: string; | |
| } | |
| export async function replicateGenerate(params: ReplicateGenerateParams): Promise<ReplicateGenerateResponse> { | |
| return apiRequest<ReplicateGenerateResponse>('/api/replicate/generate', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify(params), | |
| }); | |
| } | |
| export async function replicateGetStatus(predictionId: string): Promise<VideoStatusResponse> { | |
| return apiRequest<VideoStatusResponse>(`/api/replicate/status/${predictionId}`); | |
| } | |
| // Wait for Replicate video completion by polling | |
| export async function waitForReplicateVideo( | |
| predictionId: string, | |
| timeoutMs: number = 600000, // 10 minutes | |
| pollIntervalMs: number = 15000 // Poll every 15 seconds (video gen takes 2-5 min) | |
| ): Promise<string> { | |
| const startTime = Date.now(); | |
| while (Date.now() - startTime < timeoutMs) { | |
| const status = await replicateGetStatus(predictionId); | |
| if (status.status === 'succeeded' && status.url) { | |
| return status.url; | |
| } else if (status.status === 'failed') { | |
| throw new Error(status.error || 'Replicate video generation failed'); | |
| } | |
| // Wait before next poll | |
| await new Promise(resolve => setTimeout(resolve, pollIntervalMs)); | |
| } | |
| throw new Error('Replicate video generation timed out'); | |
| } | |
| // ==================== PROMPT GENERATION ==================== | |
| export async function generatePrompts(formData: FormData): Promise<SegmentsPayload> { | |
| const response = await fetch(`${API_BASE}/api/generate-prompts`, { | |
| method: 'POST', | |
| body: formData, | |
| }); | |
| if (!response.ok) { | |
| let errorMessage = 'Failed to generate prompts'; | |
| try { | |
| const errorData = await response.json(); | |
| errorMessage = errorData.detail || errorMessage; | |
| } catch { | |
| // Ignore | |
| } | |
| throw new Error(errorMessage); | |
| } | |
| return response.json(); | |
| } | |
| export async function refinePromptContinuity( | |
| segmentPrompt: object, | |
| lastFrameBlob: Blob | |
| ): Promise<{ refined_prompt: object }> { | |
| const formData = new FormData(); | |
| formData.append('segmentPrompt', JSON.stringify(segmentPrompt)); | |
| formData.append('lastFrame', lastFrameBlob, 'last-frame.jpg'); | |
| return apiRequest<{ refined_prompt: object }>('/api/refine-prompt-continuity', { | |
| method: 'POST', | |
| body: formData, | |
| }); | |
| } | |
| // ==================== IMAGE UPLOAD ==================== | |
| export async function uploadImage(file: File): Promise<{ url: string; filename: string }> { | |
| const formData = new FormData(); | |
| formData.append('file', file); | |
| return apiRequest<{ url: string; filename: string }>('/api/upload-image', { | |
| method: 'POST', | |
| body: formData, | |
| }); | |
| } | |
| // ==================== FRAME EXTRACTION ==================== | |
| export interface ExtractFramesParams { | |
| video_url: string; | |
| script?: string; | |
| buffer_time?: number; | |
| num_frames?: number; | |
| model_size?: string; | |
| } | |
| export interface ExtractFramesResponse { | |
| frames: ExtractedFrame[]; | |
| } | |
| export async function extractFrames(params: ExtractFramesParams): Promise<ExtractFramesResponse> { | |
| return apiRequest<ExtractFramesResponse>('/api/extract-frames', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify(params), | |
| }); | |
| } | |
| // ==================== WHISPER ANALYSIS ==================== | |
| export interface WhisperAnalyzeParams { | |
| video_url: string; | |
| dialogue: string; | |
| buffer_time?: number; | |
| model_size?: string; | |
| } | |
| export interface WhisperAnalyzeResponse { | |
| success: boolean; | |
| last_word_timestamp: number | null; | |
| trim_point: number | null; | |
| frame_timestamp: number | null; | |
| frame_base64: string | null; | |
| video_duration: number; | |
| transcribed_text: string | null; // What Whisper actually heard - for prompt refinement | |
| error: string | null; | |
| } | |
| /** | |
| * Analyze video with Whisper to find last spoken word and extract frame. | |
| * This is the optimized flow that combines Whisper analysis and frame extraction. | |
| */ | |
| export async function whisperAnalyzeAndExtract(params: WhisperAnalyzeParams): Promise<WhisperAnalyzeResponse> { | |
| return apiRequest<WhisperAnalyzeResponse>('/api/whisper/analyze-and-extract', { | |
| method: 'POST', | |
| headers: { 'Content-Type': 'application/json' }, | |
| body: JSON.stringify(params), | |
| }); | |
| } | |
| /** | |
| * Refine a segment prompt using the frame AND transcription from the previous segment. | |
| * This ensures perfect visual and audio continuity. | |
| */ | |
| export async function refinePromptWithContext( | |
| segmentPrompt: object, | |
| frameFile: File, | |
| transcribedDialogue: string, | |
| expectedDialogue: string | |
| ): Promise<{ refined_prompt: object; original_prompt: object }> { | |
| const formData = new FormData(); | |
| formData.append('segmentPrompt', JSON.stringify(segmentPrompt)); | |
| formData.append('lastFrame', frameFile); | |
| formData.append('transcribedDialogue', transcribedDialogue); | |
| formData.append('expectedDialogue', expectedDialogue); | |
| const response = await fetch(`${API_BASE}/api/refine-prompt-continuity`, { | |
| method: 'POST', | |
| body: formData, | |
| }); | |
| if (!response.ok) { | |
| throw new Error(`Failed to refine prompt: ${response.status}`); | |
| } | |
| return response.json(); | |
| } | |
| /** | |
| * Check if Whisper is available on the backend | |
| */ | |
| export async function checkWhisperStatus(): Promise<{ available: boolean; message: string }> { | |
| return apiRequest<{ available: boolean; message: string }>('/api/whisper/status'); | |
| } | |
| // ==================== VIDEO DOWNLOAD ==================== | |
| export async function downloadVideo(url: string): Promise<Blob> { | |
| const response = await fetch(`${API_BASE}/api/veo/download?url=${encodeURIComponent(url)}`); | |
| if (!response.ok) { | |
| throw new Error(`Failed to download video: ${response.status}`); | |
| } | |
| return response.blob(); | |
| } | |
| // ==================== UTILITIES ==================== | |
| export async function getVideoDuration(file: File): Promise<number> { | |
| return new Promise((resolve, reject) => { | |
| const video = document.createElement('video'); | |
| video.preload = 'metadata'; | |
| video.src = URL.createObjectURL(file); | |
| video.onloadedmetadata = () => { | |
| URL.revokeObjectURL(video.src); | |
| resolve(video.duration); | |
| }; | |
| video.onerror = () => { | |
| URL.revokeObjectURL(video.src); | |
| reject(new Error('Failed to load video metadata')); | |
| }; | |
| }); | |
| } | |
| export async function generateThumbnails(file: File, count: number = 5): Promise<string[]> { | |
| return new Promise((resolve, reject) => { | |
| const video = document.createElement('video'); | |
| video.preload = 'metadata'; | |
| video.src = URL.createObjectURL(file); | |
| video.muted = true; | |
| video.onloadedmetadata = async () => { | |
| const duration = video.duration; | |
| const thumbnails: string[] = []; | |
| const canvas = document.createElement('canvas'); | |
| const ctx = canvas.getContext('2d'); | |
| if (!ctx) { | |
| URL.revokeObjectURL(video.src); | |
| reject(new Error('Could not get canvas context')); | |
| return; | |
| } | |
| // Use video's actual dimensions for proper aspect ratio | |
| // Scale down while maintaining aspect ratio (max 400px on longest side) | |
| const maxSize = 400; | |
| const videoWidth = video.videoWidth || 1080; | |
| const videoHeight = video.videoHeight || 1920; | |
| const scale = Math.min(maxSize / videoWidth, maxSize / videoHeight); | |
| canvas.width = Math.round(videoWidth * scale); | |
| canvas.height = Math.round(videoHeight * scale); | |
| for (let i = 0; i < count; i++) { | |
| const time = (duration / count) * i; | |
| video.currentTime = time; | |
| await new Promise<void>((res) => { | |
| video.onseeked = () => res(); | |
| }); | |
| ctx.drawImage(video, 0, 0, canvas.width, canvas.height); | |
| thumbnails.push(canvas.toDataURL('image/jpeg', 0.85)); | |
| } | |
| URL.revokeObjectURL(video.src); | |
| resolve(thumbnails); | |
| }; | |
| video.onerror = () => { | |
| URL.revokeObjectURL(video.src); | |
| reject(new Error('Failed to load video')); | |
| }; | |
| }); | |
| } | |
| // Wait for video completion using SSE | |
| export function waitForKlingVideo(taskId: string, timeoutMs: number = 300000): Promise<string> { | |
| return new Promise((resolve, reject) => { | |
| const eventSource = createKlingEventSource(taskId); | |
| const startTime = Date.now(); | |
| const timeout = setTimeout(() => { | |
| eventSource.close(); | |
| reject(new Error('Video generation timed out')); | |
| }, timeoutMs); | |
| eventSource.onmessage = (event) => { | |
| try { | |
| const data = JSON.parse(event.data); | |
| if (data.status === 'succeeded' && data.url) { | |
| clearTimeout(timeout); | |
| eventSource.close(); | |
| resolve(data.url); | |
| } else if (data.status === 'cancelled') { | |
| clearTimeout(timeout); | |
| eventSource.close(); | |
| reject(new Error('Video generation cancelled by user')); | |
| } else if (data.status === 'failed' || data.code !== undefined && data.code !== 200) { | |
| clearTimeout(timeout); | |
| eventSource.close(); | |
| const errorMsg = data.error || data.msg || `Video generation failed (code: ${data.code || 'unknown'})`; | |
| reject(new Error(errorMsg)); | |
| } | |
| } catch (err) { | |
| console.error('Failed to parse SSE data:', err); | |
| // If we can't parse, don't reject immediately - might be a partial message | |
| } | |
| }; | |
| eventSource.onerror = () => { | |
| // If connection error and we've been waiting a while, reject | |
| const elapsed = Date.now() - startTime; | |
| if (elapsed > 5000) { // Wait at least 5 seconds before rejecting on connection error | |
| clearTimeout(timeout); | |
| eventSource.close(); | |
| reject(new Error('SSE connection error - video generation may have failed')); | |
| } | |
| }; | |
| }); | |
| } | |
| // Generate video with automatic retry (retries once on failure) | |
| export async function generateVideoWithRetry( | |
| generateFn: () => Promise<{ taskId: string }>, | |
| timeoutMs: number = 300000, | |
| onRetry?: (attempt: number) => void | |
| ): Promise<string> { | |
| let lastError: Error | null = null; | |
| for (let attempt = 0; attempt < 2; attempt++) { | |
| try { | |
| if (attempt > 0) { | |
| console.log(`🔄 Retrying video generation (attempt ${attempt + 1}/2)...`); | |
| if (onRetry) { | |
| onRetry(attempt + 1); | |
| } | |
| } | |
| const result = await generateFn(); | |
| const videoUrl = await waitForKlingVideo(result.taskId, timeoutMs); | |
| return videoUrl; | |
| } catch (error) { | |
| lastError = error instanceof Error ? error : new Error(String(error)); | |
| console.error(`❌ Video generation attempt ${attempt + 1} failed:`, lastError.message); | |
| // If this was the first attempt, retry once | |
| if (attempt === 0) { | |
| console.log('⏳ Waiting 2 seconds before retry...'); | |
| await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2 seconds before retry | |
| continue; | |
| } | |
| // If both attempts failed, throw the error | |
| throw lastError; | |
| } | |
| } | |
| // This should never be reached, but TypeScript needs it | |
| throw lastError || new Error('Video generation failed'); | |
| } | |
| // ==================== VIDEO MERGE/EXPORT ==================== | |
| export interface ClipMetadata { | |
| index: number; | |
| startTime: number; | |
| endTime: number; | |
| type: 'video' | 'image'; | |
| duration?: number; | |
| } | |
| // Merge multiple video files into a single video | |
| export async function mergeVideos( | |
| videoBlobs: Blob[], | |
| clipMetadata: ClipMetadata[] | |
| ): Promise<Blob> { | |
| const formData = new FormData(); | |
| // Add clip metadata as JSON | |
| formData.append('clips_data', JSON.stringify(clipMetadata)); | |
| // Add video files | |
| videoBlobs.forEach((blob, index) => { | |
| formData.append('files', blob, `video_${index}.mp4`); | |
| }); | |
| const response = await fetch(`${API_BASE}/api/export/merge`, { | |
| method: 'POST', | |
| body: formData, | |
| }); | |
| if (!response.ok) { | |
| let errorMessage = 'Failed to merge videos'; | |
| try { | |
| const errorData = await response.json(); | |
| errorMessage = errorData.detail || errorMessage; | |
| } catch { | |
| // Ignore JSON parse errors | |
| } | |
| throw new Error(errorMessage); | |
| } | |
| return response.blob(); | |
| } | |