sushilideaclan01's picture
Add video cancellation feature and update API integration
66e744c
import type {
VideoStatusResponse,
SegmentsPayload,
HealthStatus,
ExtractedFrame,
LoginRequest,
LoginResponse,
AuthUser
} from '@/types';
const API_BASE = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000';
// Get auth token from localStorage
function getAuthToken(): string | null {
return localStorage.getItem('auth_token');
}
// Set auth token in localStorage
export function setAuthToken(token: string): void {
localStorage.setItem('auth_token', token);
}
// Remove auth token from localStorage
export function removeAuthToken(): void {
localStorage.removeItem('auth_token');
}
// Generic API request handler
async function apiRequest<T>(
path: string,
options: RequestInit = {}
): Promise<T> {
const url = path.startsWith('http') ? path : `${API_BASE}${path}`;
const token = getAuthToken();
// Normalize headers to a plain object
const headers: Record<string, string> = {};
// Convert Headers object or array to plain object
if (options.headers) {
if (options.headers instanceof Headers) {
options.headers.forEach((value, key) => {
headers[key] = value;
});
} else if (Array.isArray(options.headers)) {
options.headers.forEach(([key, value]) => {
headers[key] = value;
});
} else {
Object.assign(headers, options.headers);
}
}
// Add auth token if available
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
const response = await fetch(url, {
...options,
headers,
});
if (!response.ok) {
// If unauthorized, clear token
if (response.status === 401) {
removeAuthToken();
}
let errorMessage = `Request failed with status ${response.status}`;
// Try to extract error message from response
const contentType = response.headers.get('content-type');
const isJson = contentType && contentType.includes('application/json');
try {
if (isJson) {
const errorData = await response.json();
// Try multiple common error message fields
errorMessage = errorData.detail ||
errorData.message ||
errorData.error ||
(typeof errorData === 'string' ? errorData : errorMessage);
} else {
const text = await response.text();
if (text && text.trim()) {
errorMessage = text;
} else {
// Fall back to default message based on status code
if (response.status === 401) {
errorMessage = 'Incorrect username or password.';
} else if (response.status === 403) {
errorMessage = 'Access forbidden.';
} else if (response.status === 404) {
errorMessage = 'Resource not found.';
} else if (response.status >= 500) {
errorMessage = 'Server error. Please try again later.';
}
}
}
} catch (parseError) {
// If parsing fails, use status-based default
if (response.status === 401) {
errorMessage = 'Incorrect username or password.';
} else if (response.status === 403) {
errorMessage = 'Access forbidden.';
} else if (response.status === 404) {
errorMessage = 'Resource not found.';
} else if (response.status >= 500) {
errorMessage = 'Server error. Please try again later.';
}
}
const error = new Error(errorMessage);
console.error('API Error:', {
status: response.status,
statusText: response.statusText,
message: errorMessage,
url: url,
contentType: contentType
});
throw error;
}
return response.json();
}
// Health check
export async function checkHealth(): Promise<HealthStatus> {
return apiRequest<HealthStatus>('/health');
}
// ==================== AUTHENTICATION ====================
export async function login(credentials: LoginRequest): Promise<LoginResponse> {
const response = await apiRequest<LoginResponse>('/api/auth/login', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(credentials),
});
// Store token
if (response.access_token) {
setAuthToken(response.access_token);
}
return response;
}
export async function verifyAuth(): Promise<AuthUser> {
try {
const response = await apiRequest<AuthUser>('/api/auth/verify');
return response;
} catch {
removeAuthToken();
return { username: '', authenticated: false };
}
}
export async function getCurrentUser(): Promise<AuthUser> {
try {
const response = await apiRequest<AuthUser>('/api/auth/me');
return response;
} catch {
removeAuthToken();
return { username: '', authenticated: false };
}
}
export function logout(): void {
removeAuthToken();
}
// ==================== KLING/KIE API ====================
export interface KlingGenerateParams {
prompt: string | object;
imageUrls?: string[];
model?: string;
aspectRatio?: string;
generationType?: string;
seeds?: number;
voiceType?: string;
}
export interface KlingGenerateResponse {
taskId: string;
status: string;
}
export async function klingGenerate(params: KlingGenerateParams): Promise<KlingGenerateResponse> {
return apiRequest<KlingGenerateResponse>('/api/veo/generate', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
}
export async function klingExtend(taskId: string, prompt: string | object, seeds?: number, voiceType?: string): Promise<KlingGenerateResponse> {
return apiRequest<KlingGenerateResponse>('/api/veo/extend', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ taskId, prompt, seeds, voiceType }),
});
}
export async function klingGetStatus(taskId: string): Promise<VideoStatusResponse> {
return apiRequest<VideoStatusResponse>(`/api/veo/status/${taskId}`);
}
export async function klingCancel(taskId: string): Promise<{ code: number; msg: string; taskId: string }> {
return apiRequest<{ code: number; msg: string; taskId: string }>(`/api/veo/cancel/${taskId}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
});
}
export function createKlingEventSource(taskId: string): EventSource {
return new EventSource(`${API_BASE}/api/veo/events/${taskId}`);
}
// ==================== REPLICATE API ====================
export interface ReplicateGenerateParams {
prompt: string;
imageUrl?: string;
model?: string;
duration?: number;
aspectRatio?: string;
}
export interface ReplicateGenerateResponse {
id: string;
status: string;
}
export async function replicateGenerate(params: ReplicateGenerateParams): Promise<ReplicateGenerateResponse> {
return apiRequest<ReplicateGenerateResponse>('/api/replicate/generate', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
}
export async function replicateGetStatus(predictionId: string): Promise<VideoStatusResponse> {
return apiRequest<VideoStatusResponse>(`/api/replicate/status/${predictionId}`);
}
// Wait for Replicate video completion by polling
export async function waitForReplicateVideo(
predictionId: string,
timeoutMs: number = 600000, // 10 minutes
pollIntervalMs: number = 15000 // Poll every 15 seconds (video gen takes 2-5 min)
): Promise<string> {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
const status = await replicateGetStatus(predictionId);
if (status.status === 'succeeded' && status.url) {
return status.url;
} else if (status.status === 'failed') {
throw new Error(status.error || 'Replicate video generation failed');
}
// Wait before next poll
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
}
throw new Error('Replicate video generation timed out');
}
// ==================== PROMPT GENERATION ====================
export async function generatePrompts(formData: FormData): Promise<SegmentsPayload> {
const response = await fetch(`${API_BASE}/api/generate-prompts`, {
method: 'POST',
body: formData,
});
if (!response.ok) {
let errorMessage = 'Failed to generate prompts';
try {
const errorData = await response.json();
errorMessage = errorData.detail || errorMessage;
} catch {
// Ignore
}
throw new Error(errorMessage);
}
return response.json();
}
export async function refinePromptContinuity(
segmentPrompt: object,
lastFrameBlob: Blob
): Promise<{ refined_prompt: object }> {
const formData = new FormData();
formData.append('segmentPrompt', JSON.stringify(segmentPrompt));
formData.append('lastFrame', lastFrameBlob, 'last-frame.jpg');
return apiRequest<{ refined_prompt: object }>('/api/refine-prompt-continuity', {
method: 'POST',
body: formData,
});
}
// ==================== IMAGE UPLOAD ====================
export async function uploadImage(file: File): Promise<{ url: string; filename: string }> {
const formData = new FormData();
formData.append('file', file);
return apiRequest<{ url: string; filename: string }>('/api/upload-image', {
method: 'POST',
body: formData,
});
}
// ==================== FRAME EXTRACTION ====================
export interface ExtractFramesParams {
video_url: string;
script?: string;
buffer_time?: number;
num_frames?: number;
model_size?: string;
}
export interface ExtractFramesResponse {
frames: ExtractedFrame[];
}
export async function extractFrames(params: ExtractFramesParams): Promise<ExtractFramesResponse> {
return apiRequest<ExtractFramesResponse>('/api/extract-frames', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
}
// ==================== WHISPER ANALYSIS ====================
export interface WhisperAnalyzeParams {
video_url: string;
dialogue: string;
buffer_time?: number;
model_size?: string;
}
export interface WhisperAnalyzeResponse {
success: boolean;
last_word_timestamp: number | null;
trim_point: number | null;
frame_timestamp: number | null;
frame_base64: string | null;
video_duration: number;
transcribed_text: string | null; // What Whisper actually heard - for prompt refinement
error: string | null;
}
/**
* Analyze video with Whisper to find last spoken word and extract frame.
* This is the optimized flow that combines Whisper analysis and frame extraction.
*/
export async function whisperAnalyzeAndExtract(params: WhisperAnalyzeParams): Promise<WhisperAnalyzeResponse> {
return apiRequest<WhisperAnalyzeResponse>('/api/whisper/analyze-and-extract', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
}
/**
* Refine a segment prompt using the frame AND transcription from the previous segment.
* This ensures perfect visual and audio continuity.
*/
export async function refinePromptWithContext(
segmentPrompt: object,
frameFile: File,
transcribedDialogue: string,
expectedDialogue: string
): Promise<{ refined_prompt: object; original_prompt: object }> {
const formData = new FormData();
formData.append('segmentPrompt', JSON.stringify(segmentPrompt));
formData.append('lastFrame', frameFile);
formData.append('transcribedDialogue', transcribedDialogue);
formData.append('expectedDialogue', expectedDialogue);
const response = await fetch(`${API_BASE}/api/refine-prompt-continuity`, {
method: 'POST',
body: formData,
});
if (!response.ok) {
throw new Error(`Failed to refine prompt: ${response.status}`);
}
return response.json();
}
/**
* Check if Whisper is available on the backend
*/
export async function checkWhisperStatus(): Promise<{ available: boolean; message: string }> {
return apiRequest<{ available: boolean; message: string }>('/api/whisper/status');
}
// ==================== VIDEO DOWNLOAD ====================
export async function downloadVideo(url: string): Promise<Blob> {
const response = await fetch(`${API_BASE}/api/veo/download?url=${encodeURIComponent(url)}`);
if (!response.ok) {
throw new Error(`Failed to download video: ${response.status}`);
}
return response.blob();
}
// ==================== UTILITIES ====================
export async function getVideoDuration(file: File): Promise<number> {
return new Promise((resolve, reject) => {
const video = document.createElement('video');
video.preload = 'metadata';
video.src = URL.createObjectURL(file);
video.onloadedmetadata = () => {
URL.revokeObjectURL(video.src);
resolve(video.duration);
};
video.onerror = () => {
URL.revokeObjectURL(video.src);
reject(new Error('Failed to load video metadata'));
};
});
}
export async function generateThumbnails(file: File, count: number = 5): Promise<string[]> {
return new Promise((resolve, reject) => {
const video = document.createElement('video');
video.preload = 'metadata';
video.src = URL.createObjectURL(file);
video.muted = true;
video.onloadedmetadata = async () => {
const duration = video.duration;
const thumbnails: string[] = [];
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
if (!ctx) {
URL.revokeObjectURL(video.src);
reject(new Error('Could not get canvas context'));
return;
}
// Use video's actual dimensions for proper aspect ratio
// Scale down while maintaining aspect ratio (max 400px on longest side)
const maxSize = 400;
const videoWidth = video.videoWidth || 1080;
const videoHeight = video.videoHeight || 1920;
const scale = Math.min(maxSize / videoWidth, maxSize / videoHeight);
canvas.width = Math.round(videoWidth * scale);
canvas.height = Math.round(videoHeight * scale);
for (let i = 0; i < count; i++) {
const time = (duration / count) * i;
video.currentTime = time;
await new Promise<void>((res) => {
video.onseeked = () => res();
});
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
thumbnails.push(canvas.toDataURL('image/jpeg', 0.85));
}
URL.revokeObjectURL(video.src);
resolve(thumbnails);
};
video.onerror = () => {
URL.revokeObjectURL(video.src);
reject(new Error('Failed to load video'));
};
});
}
// Wait for video completion using SSE
export function waitForKlingVideo(taskId: string, timeoutMs: number = 300000): Promise<string> {
return new Promise((resolve, reject) => {
const eventSource = createKlingEventSource(taskId);
const startTime = Date.now();
const timeout = setTimeout(() => {
eventSource.close();
reject(new Error('Video generation timed out'));
}, timeoutMs);
eventSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.status === 'succeeded' && data.url) {
clearTimeout(timeout);
eventSource.close();
resolve(data.url);
} else if (data.status === 'cancelled') {
clearTimeout(timeout);
eventSource.close();
reject(new Error('Video generation cancelled by user'));
} else if (data.status === 'failed' || data.code !== undefined && data.code !== 200) {
clearTimeout(timeout);
eventSource.close();
const errorMsg = data.error || data.msg || `Video generation failed (code: ${data.code || 'unknown'})`;
reject(new Error(errorMsg));
}
} catch (err) {
console.error('Failed to parse SSE data:', err);
// If we can't parse, don't reject immediately - might be a partial message
}
};
eventSource.onerror = () => {
// If connection error and we've been waiting a while, reject
const elapsed = Date.now() - startTime;
if (elapsed > 5000) { // Wait at least 5 seconds before rejecting on connection error
clearTimeout(timeout);
eventSource.close();
reject(new Error('SSE connection error - video generation may have failed'));
}
};
});
}
// Generate video with automatic retry (retries once on failure)
export async function generateVideoWithRetry(
generateFn: () => Promise<{ taskId: string }>,
timeoutMs: number = 300000,
onRetry?: (attempt: number) => void
): Promise<string> {
let lastError: Error | null = null;
for (let attempt = 0; attempt < 2; attempt++) {
try {
if (attempt > 0) {
console.log(`🔄 Retrying video generation (attempt ${attempt + 1}/2)...`);
if (onRetry) {
onRetry(attempt + 1);
}
}
const result = await generateFn();
const videoUrl = await waitForKlingVideo(result.taskId, timeoutMs);
return videoUrl;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
console.error(`❌ Video generation attempt ${attempt + 1} failed:`, lastError.message);
// If this was the first attempt, retry once
if (attempt === 0) {
console.log('⏳ Waiting 2 seconds before retry...');
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2 seconds before retry
continue;
}
// If both attempts failed, throw the error
throw lastError;
}
}
// This should never be reached, but TypeScript needs it
throw lastError || new Error('Video generation failed');
}
// ==================== VIDEO MERGE/EXPORT ====================
export interface ClipMetadata {
index: number;
startTime: number;
endTime: number;
type: 'video' | 'image';
duration?: number;
}
// Merge multiple video files into a single video
export async function mergeVideos(
videoBlobs: Blob[],
clipMetadata: ClipMetadata[]
): Promise<Blob> {
const formData = new FormData();
// Add clip metadata as JSON
formData.append('clips_data', JSON.stringify(clipMetadata));
// Add video files
videoBlobs.forEach((blob, index) => {
formData.append('files', blob, `video_${index}.mp4`);
});
const response = await fetch(`${API_BASE}/api/export/merge`, {
method: 'POST',
body: formData,
});
if (!response.ok) {
let errorMessage = 'Failed to merge videos';
try {
const errorData = await response.json();
errorMessage = errorData.detail || errorMessage;
} catch {
// Ignore JSON parse errors
}
throw new Error(errorMessage);
}
return response.blob();
}