diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..2a89419fe5cab463233e2936bcb0165efd77efee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +services/ktts/assets/edu_note.wav filter=lfs diff=lfs merge=lfs -text +services/ktts/assets/fun_fact.wav filter=lfs diff=lfs merge=lfs -text +services/ktts/assets/thanks.wav filter=lfs diff=lfs merge=lfs -text +services/ktts/gradio_demo.png filter=lfs diff=lfs merge=lfs -text diff --git a/services/ffmpeg_automation/.gitattributes b/services/ffmpeg_automation/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/services/ffmpeg_automation/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/services/ffmpeg_automation/Dockerfile b/services/ffmpeg_automation/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1af343a680c5c3b807e52bd572022c27c513a84b --- /dev/null +++ b/services/ffmpeg_automation/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.10-slim + +# Install system binaries and fonts +RUN apt-get update && apt-get install -y \ + ffmpeg \ + wget \ + libmagic1 \ + fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Create temp dir and set permissions for HF user +RUN mkdir -p /app/temp && chmod 777 /app/temp + +COPY requirements.txt . +RUN pip install --no-cache-dir -U yt-dlp && \ + pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Set permissions for the entire app dir to avoid runtime write issues +RUN chmod -R 777 /app + +ENV PYTHONUNBUFFERED=1 +ENV TEMP_DIR=/app/temp + +EXPOSE 7860 + +CMD ["python", "app.py"] diff --git a/services/ffmpeg_automation/README.md b/services/ffmpeg_automation/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0e9afc5071d509ee92659534ccdc5fc6fa5df00c --- /dev/null +++ b/services/ffmpeg_automation/README.md @@ -0,0 +1,217 @@ +--- +title: Ffmpeg +emoji: 👀 +colorFrom: green +colorTo: yellow +sdk: docker +pinned: false +license: mit +--- + +# Basyx FFmpeg Automation Hub + +FastAPI + Gradio app for common FFmpeg media tasks. It can run directly in +Docker/Hugging Face Spaces and exposes both a browser UI and HTTP API. + +## Features + +- Task-specific Gradio controls for video, audio, subtitle, image, and social presets. +- File and URL inputs, including `yt-dlp` downloads. +- Input validation with `python-magic`, upload size limits, and temp-file cleanup. +- Synchronous execution at `/execute/{task_id}`. +- Background jobs at `/jobs/{task_id}` with `/status/{job_id}` and `/download/{job_id}`. +- Recent output history at `/history` with per-item download URLs. +- Configurable FFmpeg options: CRF, preset, resolution, audio bitrate, trim times, + aspect ratio, GIF settings, watermark settings, speed, frame rate, and text styling. +- Persistent SQLite job/history state. +- FFmpeg execution timeout, production readiness checks, and URL download limits. +- Faceless short-video automation for quote cards, story slides, image narration, + b-roll narration, and vertical montage generation. +- TikTok/Reels mini-series packaging with numbered episodes, recap cards, and + ZIP exports containing publish-order manifests. + +## Tasks + +Call `GET /tasks` to list the current task registry with file requirements. + +Current tasks: + +- `normalize` +- `extract_audio` +- `resize_916` +- `add_subtitles` +- `burn_lyrics` +- `text_overlay` +- `merge_music` +- `thumbnail` +- `watermark` +- `compress` +- `batch_compress` +- `make_gif` +- `tiktok_lyrics` +- `tiktok_pro_reframer` +- `reels_blur_fit` +- `reels_safe_caption` +- `reels_hook_title` +- `reels_progress_bar` +- `reels_loop` +- `reels_subtitle_safe` +- `reels_reaction_stack` +- `reels_audio_duck` +- `faceless_quote_card` +- `faceless_story_pages` +- `faceless_image_narration` +- `faceless_video_narration` +- `faceless_broll_montage` +- `series_split_pack` +- `series_episode_badge` +- `series_batch_pack` +- `series_recap_card` +- `concat` +- `slideshow` +- `trim` +- `crop_aspect` +- `waveform` +- `extract_frames` +- `add_intro_outro` +- `speed` +- `remove_audio` +- `replace_audio` + +## API Examples + +Run a task immediately and download the returned file: + +```bash +curl -X POST \ + -F "files=@input.mp4" \ + -F "resolution=720p" \ + -F "crf=28" \ + http://localhost:7860/execute/compress \ + --output compressed.mp4 +``` + +Run a background job: + +```bash +curl -X POST \ + -F "files=@input.mp4" \ + -F "text=Launch caption" \ + http://localhost:7860/jobs/text_overlay +``` + +Check the job: + +```bash +curl http://localhost:7860/status/ +``` + +Download a completed background job: + +```bash +curl http://localhost:7860/download/ --output result.mp4 +``` + +Use a URL input: + +```bash +curl -X POST \ + -F "url=https://example.com/video.mp4" \ + http://localhost:7860/execute/thumbnail \ + --output thumbnail.jpg +``` + +## n8n Inputs + +Use `POST /n8n/execute/{task_id}` for immediate file output or +`POST /n8n/jobs/{task_id}` for background jobs. These endpoints accept common +n8n HTTP Request node payload styles: + +- multipart form-data with any binary field name +- form-data URL fields: `url`, `urls`, `file_url`, `source_url`, `download_url` +- JSON base64 files in `files`, `binary`, or `data` +- JSON URL lists +- raw binary body for single-file tasks, with optional `X-Filename` header + +Multipart binary from n8n: + +```bash +curl -X POST \ + -F "myBinary=@input.mp4" \ + -F "text=Episode 1" \ + http://localhost:7860/n8n/execute/series_episode_badge \ + --output episode.mp4 +``` + +JSON base64: + +```json +{ + "text": "Episode title", + "files": [ + { + "fileName": "input.mp4", + "mimeType": "video/mp4", + "data": "" + } + ] +} +``` + +JSON URLs: + +```json +{ + "urls": ["https://example.com/input.mp4"], + "text": "Mini series title", + "duration": "60" +} +``` + +Raw binary: + +```bash +curl -X POST \ + -H "Content-Type: application/octet-stream" \ + -H "X-Filename: input.mp4" \ + --data-binary @input.mp4 \ + http://localhost:7860/n8n/execute/reels_blur_fit \ + --output output.mp4 +``` + +## Configuration + +Environment variables: + +- `TEMP_DIR`: working directory for uploads and outputs. Default: `temp`. +- `STATE_DB_PATH`: SQLite path for job/history state. Default: `TEMP_DIR/state.sqlite3`. +- `FONT_PATH`: font used by FFmpeg `drawtext`. Default: + `/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf`. +- `MAX_UPLOAD_MB`: max upload size per file. Default: `500`. +- `MAX_URL_DOWNLOAD_MB`: max URL download size. Default: same as `MAX_UPLOAD_MB`. +- `ALLOW_URL_INPUTS`: enable/disable URL downloads. Default: `true`. +- `FFMPEG_TIMEOUT_SECONDS`: max runtime for an FFmpeg command. Default: `1800`. +- `FILE_TTL_SECONDS`: temp file lifetime. Default: `3600`. +- `HISTORY_LIMIT`: number of recent output records to keep. Default: `50`. + +## Production Checks + +- `GET /healthz`: process health and dependency check details. +- `GET /readyz`: returns `503` if required runtime dependencies are missing. + +Required runtime dependencies: + +- `ffmpeg` +- `ffprobe` +- writable `TEMP_DIR` +- writable SQLite state database +- readable `FONT_PATH` + +## Local Run + +```bash +pip install -r requirements.txt +python app.py +``` + +The app listens on `http://localhost:7860`. diff --git a/services/ffmpeg_automation/app.py b/services/ffmpeg_automation/app.py new file mode 100644 index 0000000000000000000000000000000000000000..f299821fd67f642d0ae6aa8e66dfa6e1f9342d0b --- /dev/null +++ b/services/ffmpeg_automation/app.py @@ -0,0 +1,2215 @@ +import base64 +import binascii +import os +import shutil +import sqlite3 +import subprocess +import textwrap +import threading +import time +import uuid +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional + +import gradio as gr +import magic +import uvicorn +import yt_dlp +from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import FileResponse + +app = FastAPI(title="Basyx FFmpeg Automation Hub") + +TEMP_DIR = os.getenv("TEMP_DIR", "temp") +FONT_PATH = os.getenv( + "FONT_PATH", "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf" +) +MAX_UPLOAD_MB = int(os.getenv("MAX_UPLOAD_MB", "500")) +MAX_UPLOAD_BYTES = MAX_UPLOAD_MB * 1024 * 1024 +FILE_TTL_SECONDS = int(os.getenv("FILE_TTL_SECONDS", "3600")) +HISTORY_LIMIT = int(os.getenv("HISTORY_LIMIT", "50")) +FFMPEG_TIMEOUT_SECONDS = int(os.getenv("FFMPEG_TIMEOUT_SECONDS", "1800")) +MAX_URL_DOWNLOAD_MB = int(os.getenv("MAX_URL_DOWNLOAD_MB", str(MAX_UPLOAD_MB))) +MAX_URL_DOWNLOAD_BYTES = MAX_URL_DOWNLOAD_MB * 1024 * 1024 +ALLOW_URL_INPUTS = os.getenv("ALLOW_URL_INPUTS", "true").lower() in {"1", "true", "yes"} +STATE_DB_PATH = os.getenv("STATE_DB_PATH", os.path.join(TEMP_DIR, "state.sqlite3")) +OPTION_FIELDS = [ + "text", + "start_time", + "end_time", + "duration", + "aspect_ratio", + "resolution", + "crf", + "preset", + "audio_bitrate", + "volume", + "position", + "opacity", + "fps", + "width", + "speed", + "timestamp", + "image_duration", + "frame_rate", + "font_size", + "wave_color", +] + +os.makedirs(TEMP_DIR, exist_ok=True) + + +@dataclass(frozen=True) +class TaskDefinition: + label: str + description: str + category: str + output_ext: str + min_files: int + max_files: Optional[int] + accepted_types: List[str] + builder: Optional[Callable[[List[str], Dict[str, str], str], List[str]]] = None + runner: Optional[Callable[[List[str], Dict[str, str], str], str]] = None + file_types: Optional[List[List[str]]] = None + + +JOBS: Dict[str, Dict[str, object]] = {} +HISTORY: List[Dict[str, object]] = [] +STATE_LOCK = threading.Lock() + + +def db_connect(): + connection = sqlite3.connect(STATE_DB_PATH, timeout=30) + connection.row_factory = sqlite3.Row + return connection + + +def init_state_store() -> None: + with db_connect() as connection: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + status TEXT NOT NULL, + message TEXT NOT NULL, + output TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + """ + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS history ( + history_id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + task_id TEXT NOT NULL, + task TEXT NOT NULL, + output TEXT NOT NULL, + filename TEXT NOT NULL, + size INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) + """ + ) + connection.execute( + "CREATE INDEX IF NOT EXISTS idx_history_created_at ON history(created_at DESC)" + ) + + +def load_state_store() -> None: + with STATE_LOCK, db_connect() as connection: + JOBS.clear() + for row in connection.execute("SELECT * FROM jobs ORDER BY created_at DESC"): + JOBS[row["job_id"]] = dict(row) + HISTORY.clear() + for row in connection.execute( + "SELECT * FROM history ORDER BY created_at DESC LIMIT ?", (HISTORY_LIMIT,) + ): + item = dict(row) + item["download_url"] = f"/history/{item['history_id']}/download" + HISTORY.append(item) + + +def persist_job(job: Dict[str, object]) -> None: + now = int(time.time()) + with db_connect() as connection: + connection.execute( + """ + INSERT INTO jobs (job_id, task_id, status, message, output, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id) DO UPDATE SET + status=excluded.status, + message=excluded.message, + output=excluded.output, + updated_at=excluded.updated_at + """, + ( + str(job["job_id"]), + str(job["task_id"]), + str(job["status"]), + str(job["message"]), + str(job.get("output") or ""), + int(job.get("created_at") or now), + now, + ), + ) + + +def persist_history(item: Dict[str, object]) -> None: + with db_connect() as connection: + connection.execute( + """ + INSERT OR REPLACE INTO history + (history_id, job_id, task_id, task, output, filename, size, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + str(item["history_id"]), + str(item["job_id"]), + str(item["task_id"]), + str(item["task"]), + str(item["output"]), + str(item["filename"]), + int(item["size"]), + int(item["created_at"]), + ), + ) + stale_ids = [ + row["history_id"] + for row in connection.execute( + """ + SELECT history_id FROM history + ORDER BY created_at DESC + LIMIT -1 OFFSET ? + """, + (HISTORY_LIMIT,), + ) + ] + if stale_ids: + connection.executemany( + "DELETE FROM history WHERE history_id = ?", + [(history_id,) for history_id in stale_ids], + ) + + +init_state_store() +load_state_store() + + +def ffmpeg_escape(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace(":", "\\:") + .replace("'", "\\'") + .replace("\n", "\\n") + .replace("%", "\\%") + ) + + +def filter_path(path: str) -> str: + return ffmpeg_escape(os.path.abspath(path)) + + +def ffconcat_path(path: str) -> str: + return os.path.abspath(path).replace("\\", "\\\\").replace("'", "\\'") + + +def drawtext_file_path(path: str) -> str: + return filter_path(path) + + +def remove_file_quietly(path: str) -> None: + try: + os.remove(path) + except FileNotFoundError: + return + except OSError: + return + + +def clamp_int(value: str, default: int, minimum: int, maximum: int) -> int: + try: + parsed = int(float(value)) + except (TypeError, ValueError): + parsed = default + return max(minimum, min(maximum, parsed)) + + +def clamp_float(value: str, default: float, minimum: float, maximum: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + parsed = default + return max(minimum, min(maximum, parsed)) + + +def dimensions(preset: str) -> tuple[int, int]: + presets = { + "9:16": (1080, 1920), + "16:9": (1920, 1080), + "1:1": (1080, 1080), + "4:5": (1080, 1350), + } + return presets.get(preset, presets["9:16"]) + + +def scaled_crop_filter(preset: str) -> str: + width, height = dimensions(preset) + return ( + f"scale={width}:{height}:force_original_aspect_ratio=increase," + f"crop={width}:{height}" + ) + + +def output_scale_filter(size: str) -> str: + sizes = {"480p": 480, "720p": 720, "1080p": 1080} + height = sizes.get(size, 720) + return f"scale=-2:{height}" + + +def write_wrapped_text_file(text: str, width: int = 24, max_lines: Optional[int] = None) -> str: + cleaned = " ".join((text or "").split()) + if not cleaned: + cleaned = "Your faceless video" + lines = textwrap.wrap(cleaned, width=width, break_long_words=False) + if max_lines: + lines = lines[:max_lines] + path = os.path.abspath(os.path.join(TEMP_DIR, f"text_{uuid.uuid4().hex[:8]}.txt")) + with open(path, "w", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + return path + + +def split_story_pages(text: str, words_per_page: int = 22) -> List[str]: + words = (text or "").split() + if not words: + words = ["Add", "story", "text", "to", "generate", "faceless", "video", "slides."] + pages = [ + " ".join(words[index : index + words_per_page]) + for index in range(0, len(words), words_per_page) + ] + return pages[:20] + + +def production_checks() -> Dict[str, Dict[str, object]]: + checks = { + "ffmpeg": {"ok": shutil.which("ffmpeg") is not None}, + "ffprobe": {"ok": shutil.which("ffprobe") is not None}, + "font": {"ok": os.path.exists(FONT_PATH), "path": FONT_PATH}, + "temp_dir": { + "ok": os.path.isdir(TEMP_DIR) and os.access(TEMP_DIR, os.W_OK), + "path": TEMP_DIR, + }, + "state_db": { + "ok": os.path.exists(STATE_DB_PATH) and os.access(STATE_DB_PATH, os.W_OK), + "path": STATE_DB_PATH, + }, + } + checks["url_inputs"] = {"ok": True, "enabled": ALLOW_URL_INPUTS} + return checks + + +def production_ready() -> bool: + required = ["ffmpeg", "ffprobe", "font", "temp_dir", "state_db"] + checks = production_checks() + return all(bool(checks[name]["ok"]) for name in required) + + +def add_video_quality(cmd: List[str], opts: Dict[str, str]) -> List[str]: + crf = str(clamp_int(opts.get("crf", "23"), 23, 12, 35)) + preset = opts.get("preset") or "fast" + if preset not in {"ultrafast", "veryfast", "fast", "medium", "slow"}: + preset = "fast" + return cmd + ["-c:v", "libx264", "-preset", preset, "-crf", crf, "-c:a", "aac"] + + +def run_command(cmd: List[str]) -> subprocess.CompletedProcess: + try: + return subprocess.run( + cmd, + check=True, + capture_output=True, + text=True, + timeout=FFMPEG_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + raise HTTPException( + status_code=504, + detail=f"FFmpeg timed out after {FFMPEG_TIMEOUT_SECONDS} seconds.", + ) from exc + + +def media_duration_seconds(path: str) -> float: + result = run_command( + [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + path, + ] + ) + try: + duration = float(result.stdout.strip()) + except ValueError as exc: + raise HTTPException(status_code=500, detail="Could not read media duration.") from exc + if duration <= 0: + raise HTTPException(status_code=500, detail="Media duration is not usable.") + return duration + + +def build_normalize(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + cmd = ["ffmpeg", "-i", paths[0]] + return add_video_quality(cmd, opts) + ["-y", out] + + +def build_extract_audio(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + bitrate = str(clamp_int(opts.get("audio_bitrate", "192"), 192, 64, 320)) + return [ + "ffmpeg", + "-i", + paths[0], + "-vn", + "-acodec", + "libmp3lame", + "-ar", + "44100", + "-ac", + "2", + "-b:a", + f"{bitrate}k", + "-y", + out, + ] + + +def build_resize(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + vf = scaled_crop_filter(opts.get("aspect_ratio", "9:16")) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_subtitles(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + subtitle_filter = f"subtitles='{filter_path(paths[1])}'" + return ["ffmpeg", "-i", paths[0], "-vf", subtitle_filter, "-c:a", "copy", "-y", out] + + +def build_burn_lyrics(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + style = ( + "Fontname=DejaVu Sans,Fontsize=24,PrimaryColour=&H00FFFF," + "OutlineColour=&H000000,BorderStyle=3,Outline=1,Shadow=1,Alignment=2" + ) + vf = f"subtitles='{filter_path(paths[1])}':force_style='{style}'" + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_text_overlay(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text = ffmpeg_escape(opts.get("text", "")) + font_size = clamp_int(opts.get("font_size", "60"), 60, 18, 160) + vf = ( + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{text}':" + f"fontcolor=white:fontsize={font_size}:box=1:boxcolor=black@0.45:" + "boxborderw=16:x=(w-text_w)/2:y=h-200" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_merge_music(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + volume = clamp_float(opts.get("volume", "0.3"), 0.3, 0.0, 2.0) + return [ + "ffmpeg", + "-i", + paths[0], + "-i", + paths[1], + "-filter_complex", + f"[1:a]volume={volume}[a1]", + "-map", + "0:v", + "-map", + "[a1]", + "-c:v", + "copy", + "-shortest", + "-y", + out, + ] + + +def build_thumbnail(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + timestamp = opts.get("timestamp") or "00:00:02" + return ["ffmpeg", "-i", paths[0], "-ss", timestamp, "-vframes", "1", "-y", out] + + +def build_watermark(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + positions = { + "bottom-right": "W-w-20:H-h-20", + "bottom-left": "20:H-h-20", + "top-right": "W-w-20:20", + "top-left": "20:20", + "center": "(W-w)/2:(H-h)/2", + } + position = positions.get(opts.get("position"), positions["bottom-right"]) + opacity = clamp_float(opts.get("opacity", "1"), 1, 0.1, 1.0) + overlay = f"[1:v]format=rgba,colorchannelmixer=aa={opacity}[wm];[0:v][wm]overlay={position}" + return ["ffmpeg", "-i", paths[0], "-i", paths[1], "-filter_complex", overlay, "-y", out] + + +def build_compress(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + vf = output_scale_filter(opts.get("resolution", "720p")) + cmd = ["ffmpeg", "-i", paths[0], "-vf", vf] + return add_video_quality(cmd, opts) + ["-y", out] + + +def build_gif(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = str(clamp_int(opts.get("duration", "5"), 5, 1, 60)) + fps = str(clamp_int(opts.get("fps", "10"), 10, 5, 30)) + width = str(clamp_int(opts.get("width", "480"), 480, 240, 1080)) + vf = f"fps={fps},scale={width}:-1:flags=lanczos" + return ["ffmpeg", "-i", paths[0], "-t", duration, "-vf", vf, "-y", out] + + +def build_tiktok_lyrics(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text = ffmpeg_escape(opts.get("text", "")) + duration = str(clamp_int(opts.get("duration", "15"), 15, 1, 180)) + vf = ( + scaled_crop_filter("9:16") + + "," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{text}':" + "fontcolor=white:fontsize=50:box=1:boxcolor=black@0.5:" + "boxborderw=20:line_spacing=15:x=(w-text_w)/2:y=(h-text_h)/2" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-t", duration, "-y", out] + + +def build_tiktok_pro(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text = ffmpeg_escape(opts.get("text", "")) + vf = ( + scaled_crop_filter("9:16") + + "," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{text}':" + "fontcolor=white:fontsize=48:box=1:boxcolor=black@0.4:" + "boxborderw=14:x=(w-text_w)/2:y=h-200," + "eq=contrast=1.15:brightness=0.04" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_reels_blur_fit(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + filter_complex = ( + "[0:v]scale=1080:1920:force_original_aspect_ratio=increase," + "crop=1080:1920,gblur=sigma=24,eq=brightness=-0.08[bg];" + "[0:v]scale=1080:1920:force_original_aspect_ratio=decrease[fg];" + "[bg][fg]overlay=(W-w)/2:(H-h)/2,setsar=1[v]" + ) + return [ + "ffmpeg", + "-i", + paths[0], + "-filter_complex", + filter_complex, + "-map", + "[v]", + "-map", + "0:a?", + "-c:v", + "libx264", + "-preset", + opts.get("preset") or "fast", + "-crf", + str(clamp_int(opts.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-y", + out, + ] + + +def build_reels_safe_caption(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text = ffmpeg_escape(opts.get("text", "")) + font_size = clamp_int(opts.get("font_size", "58"), 58, 24, 140) + vf = ( + scaled_crop_filter("9:16") + + "," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{text}':" + f"fontcolor=white:fontsize={font_size}:line_spacing=10:" + "box=1:boxcolor=black@0.55:boxborderw=22:" + "x=(w-text_w)/2:y=h-430" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_reels_hook_title(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text = ffmpeg_escape(opts.get("text", "")) + font_size = clamp_int(opts.get("font_size", "64"), 64, 28, 150) + vf = ( + scaled_crop_filter("9:16") + + ",drawbox=x=0:y=95:w=iw:h=210:color=black@0.62:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{text}':" + f"fontcolor=white:fontsize={font_size}:line_spacing=8:" + "x=(w-text_w)/2:y=145" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_reels_progress_bar(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = clamp_float(opts.get("duration", "30"), 30, 1, 180) + vf = ( + scaled_crop_filter("9:16") + + ",drawbox=x=0:y=0:w=iw:h=10:color=black@0.35:t=fill," + + f"drawbox=x=0:y=0:w='min(iw,iw*t/{duration:.3f})':h=10:" + "color=white@0.95:t=fill" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-t", f"{duration:.3f}", "-y", out] + + +def build_reels_loop(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = clamp_float(opts.get("duration", "15"), 15, 1, 180) + return [ + "ffmpeg", + "-stream_loop", + "-1", + "-i", + paths[0], + "-t", + f"{duration:.3f}", + "-vf", + scaled_crop_filter("9:16"), + "-c:v", + "libx264", + "-preset", + opts.get("preset") or "fast", + "-crf", + str(clamp_int(opts.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-y", + out, + ] + + +def build_reels_subtitle_safe(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + vf = ( + scaled_crop_filter("9:16") + + ",subtitles='" + + filter_path(paths[1]) + + "':force_style='Fontname=DejaVu Sans,Fontsize=28," + + "PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000," + + "BorderStyle=3,Outline=1,Shadow=0,Alignment=2,MarginV=250'" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_reels_reaction_stack(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + filter_complex = ( + "[0:v]scale=1080:960:force_original_aspect_ratio=increase," + "crop=1080:960,setsar=1[top];" + "[1:v]scale=1080:960:force_original_aspect_ratio=increase," + "crop=1080:960,setsar=1[bottom];" + "[top][bottom]vstack=inputs=2[v]" + ) + return [ + "ffmpeg", + "-i", + paths[0], + "-i", + paths[1], + "-filter_complex", + filter_complex, + "-map", + "[v]", + "-map", + "0:a?", + "-shortest", + "-c:v", + "libx264", + "-preset", + opts.get("preset") or "fast", + "-crf", + str(clamp_int(opts.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-y", + out, + ] + + +def build_reels_audio_duck(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + music_volume = clamp_float(opts.get("volume", "0.18"), 0.18, 0.0, 1.0) + filter_complex = ( + f"[1:a]volume={music_volume}[music];" + "[0:a][music]amix=inputs=2:duration=first:dropout_transition=2[a]" + ) + return [ + "ffmpeg", + "-i", + paths[0], + "-i", + paths[1], + "-filter_complex", + filter_complex, + "-map", + "0:v", + "-map", + "[a]", + "-c:v", + "copy", + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def build_faceless_quote_card(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = clamp_float(opts.get("duration", "12"), 12, 3, 180) + font_size = clamp_int(opts.get("font_size", "68"), 68, 28, 140) + text_path = write_wrapped_text_file(opts.get("text", ""), width=22, max_lines=9) + vf = ( + "drawbox=x=0:y=0:w=iw:h=ih:color=0x111827@1:t=fill," + "drawbox=x=70:y=190:w=940:h=1540:color=0x0f766e@0.22:t=fill," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':" + f"textfile='{drawtext_file_path(text_path)}':fontcolor=white:" + f"fontsize={font_size}:line_spacing=14:x=(w-text_w)/2:y=(h-text_h)/2" + ) + return [ + "ffmpeg", + "-f", + "lavfi", + "-i", + f"color=c=0x111827:s=1080x1920:r=30:d={duration:.3f}", + "-f", + "lavfi", + "-i", + "anullsrc=channel_layout=stereo:sample_rate=44100", + "-vf", + vf, + "-map", + "0:v", + "-map", + "1:a", + "-t", + f"{duration:.3f}", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def build_faceless_image_narration(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text_path = write_wrapped_text_file(opts.get("text", ""), width=24, max_lines=6) + font_size = clamp_int(opts.get("font_size", "54"), 54, 24, 110) + vf = ( + "scale=1200:-1,zoompan=z='min(zoom+0.0009,1.12)':" + "d=1800:s=1080x1920:fps=30," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':" + f"textfile='{drawtext_file_path(text_path)}':fontcolor=white:" + f"fontsize={font_size}:line_spacing=10:box=1:boxcolor=black@0.52:" + "boxborderw=22:x=(w-text_w)/2:y=h-520" + ) + return [ + "ffmpeg", + "-loop", + "1", + "-i", + paths[0], + "-i", + paths[1], + "-vf", + vf, + "-map", + "0:v", + "-map", + "1:a", + "-c:v", + "libx264", + "-preset", + opts.get("preset") or "fast", + "-crf", + str(clamp_int(opts.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-shortest", + "-pix_fmt", + "yuv420p", + "-y", + out, + ] + + +def build_faceless_video_narration(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + text_path = write_wrapped_text_file(opts.get("text", ""), width=24, max_lines=6) + font_size = clamp_int(opts.get("font_size", "52"), 52, 24, 110) + filter_complex = ( + "[0:v]scale=1080:1920:force_original_aspect_ratio=increase," + "crop=1080:1920,setsar=1," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':" + f"textfile='{drawtext_file_path(text_path)}':fontcolor=white:" + f"fontsize={font_size}:line_spacing=10:box=1:boxcolor=black@0.5:" + "boxborderw=20:x=(w-text_w)/2:y=h-500[v]" + ) + return [ + "ffmpeg", + "-i", + paths[0], + "-i", + paths[1], + "-filter_complex", + filter_complex, + "-map", + "[v]", + "-map", + "1:a", + "-c:v", + "libx264", + "-preset", + opts.get("preset") or "fast", + "-crf", + str(clamp_int(opts.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def run_faceless_story_pages(paths: List[str], options: Dict[str, str], out: str) -> str: + pages = split_story_pages(options.get("text", ""), words_per_page=24) + seconds = clamp_float(options.get("image_duration", "3"), 3, 1, 12) + work_dir = os.path.join(TEMP_DIR, f"story_{uuid.uuid4().hex[:8]}") + os.makedirs(work_dir, exist_ok=True) + list_path = os.path.join(work_dir, "concat.txt") + try: + with open(list_path, "w", encoding="utf-8") as concat_file: + for index, page in enumerate(pages, start=1): + clip_path = os.path.join(work_dir, f"page_{index:03d}.mp4") + text_path = write_wrapped_text_file(page, width=24, max_lines=8) + vf = ( + "drawbox=x=0:y=0:w=iw:h=ih:color=0x0f172a@1:t=fill," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':" + f"textfile='{drawtext_file_path(text_path)}':fontcolor=white:" + "fontsize=62:line_spacing=12:x=(w-text_w)/2:y=(h-text_h)/2," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{index}/{len(pages)}':" + "fontcolor=white@0.62:fontsize=34:x=(w-text_w)/2:y=h-150" + ) + cmd = [ + "ffmpeg", + "-f", + "lavfi", + "-i", + f"color=c=0x0f172a:s=1080x1920:r=30:d={seconds:.3f}", + "-f", + "lavfi", + "-i", + "anullsrc=channel_layout=stereo:sample_rate=44100", + "-vf", + vf, + "-map", + "0:v", + "-map", + "1:a", + "-t", + f"{seconds:.3f}", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-y", + clip_path, + ] + run_command(cmd) + concat_file.write(f"file '{ffconcat_path(clip_path)}'\n") + run_command(["ffmpeg", "-f", "concat", "-safe", "0", "-i", list_path, "-c", "copy", "-y", out]) + return out + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def run_faceless_broll_montage(paths: List[str], options: Dict[str, str], out: str) -> str: + clip_seconds = clamp_float(options.get("image_duration", "2.5"), 2.5, 1, 8) + max_clips = clamp_int(options.get("duration", "12"), 12, 1, 60) + work_dir = os.path.join(TEMP_DIR, f"broll_{uuid.uuid4().hex[:8]}") + os.makedirs(work_dir, exist_ok=True) + list_path = os.path.join(work_dir, "concat.txt") + try: + with open(list_path, "w", encoding="utf-8") as concat_file: + for index, path in enumerate(paths[:max_clips], start=1): + clip_path = os.path.join(work_dir, f"clip_{index:03d}.mp4") + cmd = [ + "ffmpeg", + "-i", + path, + "-t", + f"{clip_seconds:.3f}", + "-vf", + scaled_crop_filter("9:16") + ",setsar=1", + "-an", + "-c:v", + "libx264", + "-preset", + options.get("preset") or "fast", + "-crf", + str(clamp_int(options.get("crf", "23"), 23, 12, 35)), + "-pix_fmt", + "yuv420p", + "-y", + clip_path, + ] + run_command(cmd) + concat_file.write(f"file '{ffconcat_path(clip_path)}'\n") + run_command(["ffmpeg", "-f", "concat", "-safe", "0", "-i", list_path, "-c", "copy", "-y", out]) + return out + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def build_series_episode_badge(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + title = ffmpeg_escape(opts.get("text", "Mini Series")) + font_size = clamp_int(opts.get("font_size", "48"), 48, 24, 110) + vf = ( + scaled_crop_filter("9:16") + + ",drawbox=x=36:y=86:w=360:h=92:color=black@0.68:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{title}':" + f"fontcolor=white:fontsize={font_size}:x=60:y=106," + + "drawbox=x=36:y=h-238:w=1008:h=116:color=black@0.44:t=fill" + ) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_series_recap_card(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = clamp_float(opts.get("duration", "5"), 5, 2, 30) + text_path = write_wrapped_text_file(opts.get("text", "Previously in this series"), width=22, max_lines=7) + vf = ( + "drawbox=x=0:y=0:w=iw:h=ih:color=0x111827@1:t=fill," + "drawbox=x=70:y=260:w=940:h=1180:color=0x7c2d12@0.32:t=fill," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='PREVIOUSLY':" + "fontcolor=white@0.72:fontsize=46:x=(w-text_w)/2:y=350," + f"drawtext=fontfile='{filter_path(FONT_PATH)}':textfile='{drawtext_file_path(text_path)}':" + "fontcolor=white:fontsize=68:line_spacing=14:x=(w-text_w)/2:y=(h-text_h)/2" + ) + return [ + "ffmpeg", + "-f", + "lavfi", + "-i", + f"color=c=0x111827:s=1080x1920:r=30:d={duration:.3f}", + "-f", + "lavfi", + "-i", + "anullsrc=channel_layout=stereo:sample_rate=44100", + "-vf", + vf, + "-map", + "0:v", + "-map", + "1:a", + "-t", + f"{duration:.3f}", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def run_series_split_pack(paths: List[str], options: Dict[str, str], out: str) -> str: + segment_seconds = clamp_float(options.get("duration", "60"), 60, 15, 180) + title = options.get("text", "Mini Series") + duration = media_duration_seconds(paths[0]) + episode_count = min(100, int((duration + segment_seconds - 0.001) // segment_seconds)) + work_dir = os.path.join(TEMP_DIR, f"series_{uuid.uuid4().hex[:8]}") + os.makedirs(work_dir, exist_ok=True) + manifest_path = os.path.join(work_dir, "manifest.txt") + try: + with open(manifest_path, "w", encoding="utf-8") as manifest: + manifest.write(f"series_title={title}\n") + manifest.write(f"episode_count={episode_count}\n") + manifest.write(f"episode_seconds={segment_seconds:.3f}\n\n") + for episode in range(1, episode_count + 1): + start = (episode - 1) * segment_seconds + remaining = max(0.1, duration - start) + clip_duration = min(segment_seconds, remaining) + episode_out = os.path.join(work_dir, f"episode_{episode:03d}_of_{episode_count:03d}.mp4") + label = ffmpeg_escape(f"PART {episode}/{episode_count}") + caption = ffmpeg_escape(title) + vf = ( + scaled_crop_filter("9:16") + + ",drawbox=x=36:y=86:w=390:h=92:color=black@0.68:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{label}':" + "fontcolor=white:fontsize=46:x=58:y=108," + + "drawbox=x=36:y=h-238:w=1008:h=116:color=black@0.48:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{caption}':" + "fontcolor=white:fontsize=42:x=(w-text_w)/2:y=h-205" + ) + cmd = [ + "ffmpeg", + "-ss", + f"{start:.3f}", + "-i", + paths[0], + "-t", + f"{clip_duration:.3f}", + "-vf", + vf, + "-c:v", + "libx264", + "-preset", + options.get("preset") or "fast", + "-crf", + str(clamp_int(options.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-y", + episode_out, + ] + run_command(cmd) + manifest.write(f"{episode_out}\n") + shutil.make_archive(out[:-4], "zip", work_dir) + return out + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def run_series_batch_pack(paths: List[str], options: Dict[str, str], out: str) -> str: + title = options.get("text", "Mini Series") + work_dir = os.path.join(TEMP_DIR, f"series_batch_{uuid.uuid4().hex[:8]}") + os.makedirs(work_dir, exist_ok=True) + manifest_path = os.path.join(work_dir, "manifest.txt") + try: + with open(manifest_path, "w", encoding="utf-8") as manifest: + manifest.write(f"series_title={title}\n") + manifest.write(f"episode_count={len(paths)}\n\n") + for episode, path in enumerate(paths, start=1): + episode_out = os.path.join(work_dir, f"episode_{episode:03d}_of_{len(paths):03d}.mp4") + label = ffmpeg_escape(f"PART {episode}/{len(paths)}") + caption = ffmpeg_escape(title) + vf = ( + scaled_crop_filter("9:16") + + ",drawbox=x=36:y=86:w=390:h=92:color=black@0.68:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{label}':" + "fontcolor=white:fontsize=46:x=58:y=108," + + "drawbox=x=36:y=h-238:w=1008:h=116:color=black@0.48:t=fill," + + f"drawtext=fontfile='{filter_path(FONT_PATH)}':text='{caption}':" + "fontcolor=white:fontsize=42:x=(w-text_w)/2:y=h-205" + ) + cmd = [ + "ffmpeg", + "-i", + path, + "-vf", + vf, + "-c:v", + "libx264", + "-preset", + options.get("preset") or "fast", + "-crf", + str(clamp_int(options.get("crf", "23"), 23, 12, 35)), + "-c:a", + "aac", + "-y", + episode_out, + ] + run_command(cmd) + manifest.write(f"{episode_out}\n") + shutil.make_archive(out[:-4], "zip", work_dir) + return out + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +def build_concat(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + job_id = uuid.uuid4().hex[:8] + list_path = os.path.join(TEMP_DIR, f"list_{job_id}.txt") + with open(list_path, "w", encoding="utf-8") as handle: + for path in paths: + quoted = ffconcat_path(path) + handle.write(f"file '{quoted}'\n") + return ["ffmpeg", "-f", "concat", "-safe", "0", "-i", list_path, "-c", "copy", "-y", out] + + +def build_slideshow(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + duration = str(clamp_int(opts.get("image_duration", "3"), 3, 1, 15)) + job_id = uuid.uuid4().hex[:8] + list_path = os.path.join(TEMP_DIR, f"slides_{job_id}.txt") + with open(list_path, "w", encoding="utf-8") as handle: + for path in paths: + quoted = ffconcat_path(path) + handle.write(f"file '{quoted}'\nduration {duration}\n") + quoted = ffconcat_path(paths[-1]) + handle.write(f"file '{quoted}'\n") + return [ + "ffmpeg", + "-f", + "concat", + "-safe", + "0", + "-i", + list_path, + "-vsync", + "vfr", + "-pix_fmt", + "yuv420p", + "-y", + out, + ] + + +def build_trim(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + start = opts.get("start_time") or "00:00:00" + end = opts.get("end_time") or "" + cmd = ["ffmpeg", "-ss", start, "-i", paths[0]] + if end: + cmd += ["-to", end] + return cmd + ["-c", "copy", "-y", out] + + +def build_crop(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + vf = scaled_crop_filter(opts.get("aspect_ratio", "1:1")) + return ["ffmpeg", "-i", paths[0], "-vf", vf, "-c:a", "copy", "-y", out] + + +def build_waveform(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + color = opts.get("wave_color") or "00e5ff" + if not all(ch in "0123456789abcdefABCDEF" for ch in color) or len(color) != 6: + color = "00e5ff" + return [ + "ffmpeg", + "-i", + paths[0], + "-filter_complex", + f"[0:a]showwaves=s=1280x720:mode=line:colors=#{color}[v]", + "-map", + "[v]", + "-map", + "0:a", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def build_extract_frames(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + fps = str(clamp_float(opts.get("frame_rate", "1"), 1, 0.1, 30)) + pattern = out.replace(".zip", "_%04d.jpg") + return ["ffmpeg", "-i", paths[0], "-vf", f"fps={fps}", "-y", pattern] + + +def build_add_intro_outro(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + clips = paths[:] + return build_concat(clips, opts, out) + + +def build_speed(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + speed = clamp_float(opts.get("speed", "1.25"), 1.25, 0.25, 4.0) + setpts = 1 / speed + atempo_parts = [] + remaining = speed + while remaining > 2.0: + atempo_parts.append("atempo=2.0") + remaining /= 2.0 + while remaining < 0.5: + atempo_parts.append("atempo=0.5") + remaining /= 0.5 + atempo_parts.append(f"atempo={remaining:.4f}") + filter_complex = f"[0:v]setpts={setpts:.6f}*PTS[v];[0:a]{','.join(atempo_parts)}[a]" + return [ + "ffmpeg", + "-i", + paths[0], + "-filter_complex", + filter_complex, + "-map", + "[v]", + "-map", + "[a]", + "-y", + out, + ] + + +def build_remove_audio(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + return ["ffmpeg", "-i", paths[0], "-c:v", "copy", "-an", "-y", out] + + +def build_replace_audio(paths: List[str], opts: Dict[str, str], out: str) -> List[str]: + return [ + "ffmpeg", + "-i", + paths[0], + "-i", + paths[1], + "-map", + "0:v", + "-map", + "1:a", + "-c:v", + "copy", + "-c:a", + "aac", + "-shortest", + "-y", + out, + ] + + +def run_batch_compress(paths: List[str], options: Dict[str, str], out: str) -> str: + archive_dir = os.path.join(TEMP_DIR, f"batch_{uuid.uuid4().hex[:8]}") + os.makedirs(archive_dir, exist_ok=True) + try: + for index, path in enumerate(paths, start=1): + name, _ext = os.path.splitext(os.path.basename(path)) + item_out = os.path.join(archive_dir, f"{index:02d}_{name}_compressed.mp4") + cmd = build_compress([path], options, item_out) + run_command(cmd) + shutil.make_archive(out[:-4], "zip", archive_dir) + return out + finally: + shutil.rmtree(archive_dir, ignore_errors=True) + + +TASKS: Dict[str, TaskDefinition] = { + "normalize": TaskDefinition("Normalize MP4", "Convert to H.264/AAC MP4.", "Video", "mp4", 1, 1, ["video"], build_normalize), + "extract_audio": TaskDefinition("Extract Audio", "Export MP3 audio from a video.", "Audio", "mp3", 1, 1, ["video", "audio"], build_extract_audio), + "resize_916": TaskDefinition("Resize 9:16", "Crop video for vertical platforms.", "Social", "mp4", 1, 1, ["video"], build_resize), + "add_subtitles": TaskDefinition("Add Subtitles", "Burn an SRT/ASS subtitle file into video.", "Subtitles", "mp4", 2, 2, ["video", "text"], build_subtitles, file_types=[["video"], ["text"]]), + "burn_lyrics": TaskDefinition("Burn Lyrics", "Burn styled lyric subtitles into video.", "Subtitles", "mp4", 2, 2, ["video", "text"], build_burn_lyrics, file_types=[["video"], ["text"]]), + "text_overlay": TaskDefinition("Text Overlay", "Add caption text to video.", "Social", "mp4", 1, 1, ["video"], build_text_overlay), + "merge_music": TaskDefinition("Merge Music", "Add background music to video.", "Audio", "mp4", 2, 2, ["video", "audio"], build_merge_music, file_types=[["video"], ["audio"]]), + "thumbnail": TaskDefinition("Thumbnail", "Export one JPG frame.", "Image", "jpg", 1, 1, ["video"], build_thumbnail), + "watermark": TaskDefinition("Watermark", "Overlay a logo/image on video.", "Branding", "mp4", 2, 2, ["video", "image"], build_watermark, file_types=[["video"], ["image"]]), + "compress": TaskDefinition("Compress", "Reduce video size using CRF and scale.", "Video", "mp4", 1, 1, ["video"], build_compress), + "batch_compress": TaskDefinition("Batch Compress", "Compress multiple videos and return a ZIP.", "Video", "zip", 1, None, ["video"], runner=run_batch_compress), + "make_gif": TaskDefinition("Make GIF", "Create a short GIF clip.", "Image", "gif", 1, 1, ["video"], build_gif), + "tiktok_lyrics": TaskDefinition("TikTok Lyrics", "Vertical lyric clip with boxed center text.", "Social", "mp4", 1, 1, ["video"], build_tiktok_lyrics), + "tiktok_pro_reframer": TaskDefinition("TikTok Pro Reframer", "Vertical crop, text, and contrast pass.", "Social", "mp4", 1, 1, ["video"], build_tiktok_pro), + "reels_blur_fit": TaskDefinition("Reels Blur Fit", "Fit any video into 9:16 with a blurred background.", "Social", "mp4", 1, 1, ["video"], build_reels_blur_fit), + "reels_safe_caption": TaskDefinition("Reels Safe Caption", "Add lower-third caption text inside Reels and Shorts safe zones.", "Social", "mp4", 1, 1, ["video"], build_reels_safe_caption), + "reels_hook_title": TaskDefinition("Reels Hook Title", "Add a bold top hook/title banner to a vertical clip.", "Social", "mp4", 1, 1, ["video"], build_reels_hook_title), + "reels_progress_bar": TaskDefinition("Reels Progress Bar", "Add a top progress bar and optional duration trim.", "Social", "mp4", 1, 1, ["video"], build_reels_progress_bar), + "reels_loop": TaskDefinition("Reels Loop", "Loop a clip to a target vertical Shorts/Reels duration.", "Social", "mp4", 1, 1, ["video"], build_reels_loop), + "reels_subtitle_safe": TaskDefinition("Reels Subtitle Safe", "Burn subtitles with mobile-safe lower margins.", "Social", "mp4", 2, 2, ["video", "text"], build_reels_subtitle_safe, file_types=[["video"], ["text"]]), + "reels_reaction_stack": TaskDefinition("Reels Reaction Stack", "Stack two videos vertically for reaction-style Shorts.", "Social", "mp4", 2, 2, ["video"], build_reels_reaction_stack), + "reels_audio_duck": TaskDefinition("Reels Audio Duck", "Mix background music under the original video audio.", "Social", "mp4", 2, 2, ["video", "audio"], build_reels_audio_duck, file_types=[["video"], ["audio"]]), + "faceless_quote_card": TaskDefinition("Faceless Quote Card", "Generate a vertical text-only quote short from caption text.", "Faceless", "mp4", 0, 0, [], build_faceless_quote_card), + "faceless_story_pages": TaskDefinition("Faceless Story Pages", "Split long text into timed vertical story slides.", "Faceless", "mp4", 0, 0, [], runner=run_faceless_story_pages), + "faceless_image_narration": TaskDefinition("Faceless Image Narration", "Create a Ken Burns image short with narration audio and caption text.", "Faceless", "mp4", 2, 2, ["image", "audio"], build_faceless_image_narration, file_types=[["image"], ["audio"]]), + "faceless_video_narration": TaskDefinition("Faceless Video Narration", "Create a vertical b-roll short with narration audio and caption text.", "Faceless", "mp4", 2, 2, ["video", "audio"], build_faceless_video_narration, file_types=[["video"], ["audio"]]), + "faceless_broll_montage": TaskDefinition("Faceless B-roll Montage", "Create a silent vertical montage from multiple b-roll videos.", "Faceless", "mp4", 2, None, ["video"], runner=run_faceless_broll_montage), + "series_split_pack": TaskDefinition("Mini Series Split Pack", "Split one long video into numbered vertical TikTok/Reels episodes and return a ZIP.", "Series", "zip", 1, 1, ["video"], runner=run_series_split_pack), + "series_episode_badge": TaskDefinition("Mini Series Episode Badge", "Add a series title and episode-safe badge area to one vertical clip.", "Series", "mp4", 1, 1, ["video"], build_series_episode_badge), + "series_batch_pack": TaskDefinition("Mini Series Batch Pack", "Number multiple clips as a TikTok/Reels mini-series and return a ZIP.", "Series", "zip", 2, None, ["video"], runner=run_series_batch_pack), + "series_recap_card": TaskDefinition("Mini Series Recap Card", "Generate a short text-only recap card for the next episode.", "Series", "mp4", 0, 0, [], build_series_recap_card), + "concat": TaskDefinition("Concat Videos", "Join clips in upload order.", "Video", "mp4", 2, None, ["video"], build_concat), + "slideshow": TaskDefinition("Slideshow", "Create a video from images.", "Image", "mp4", 2, None, ["image"], build_slideshow), + "trim": TaskDefinition("Trim", "Cut a clip by start/end time.", "Video", "mp4", 1, 1, ["video", "audio"], build_trim), + "crop_aspect": TaskDefinition("Crop Aspect", "Crop to 1:1, 4:5, 16:9, or 9:16.", "Video", "mp4", 1, 1, ["video"], build_crop), + "waveform": TaskDefinition("Waveform Video", "Create a waveform video from audio.", "Audio", "mp4", 1, 1, ["audio", "video"], build_waveform), + "extract_frames": TaskDefinition("Extract Frames", "Export JPG frames and return a ZIP.", "Image", "zip", 1, 1, ["video"], build_extract_frames), + "add_intro_outro": TaskDefinition("Add Intro/Outro", "Join intro, main clip, and outro.", "Video", "mp4", 2, 3, ["video"], build_add_intro_outro), + "speed": TaskDefinition("Speed Change", "Speed up or slow down video and audio.", "Video", "mp4", 1, 1, ["video"], build_speed), + "remove_audio": TaskDefinition("Remove Audio", "Export video without audio.", "Audio", "mp4", 1, 1, ["video"], build_remove_audio), + "replace_audio": TaskDefinition("Replace Audio", "Replace video audio with another file.", "Audio", "mp4", 2, 2, ["video", "audio"], build_replace_audio, file_types=[["video"], ["audio"]]), +} + + +def cleanup_worker(): + while True: + now = time.time() + expired_outputs = [] + for name in os.listdir(TEMP_DIR): + path = os.path.join(TEMP_DIR, name) + try: + if os.path.abspath(path) == os.path.abspath(STATE_DB_PATH): + continue + if os.path.isfile(path) and now - os.path.getmtime(path) > FILE_TTL_SECONDS: + expired_outputs.append(os.path.abspath(path)) + remove_file_quietly(path) + except OSError: + continue + if expired_outputs: + with db_connect() as connection: + connection.executemany( + "DELETE FROM history WHERE output = ?", + [(path,) for path in expired_outputs], + ) + load_state_store() + time.sleep(600) + + +threading.Thread(target=cleanup_worker, daemon=True).start() + + +def file_category(path: str) -> str: + try: + mime_type = magic.from_file(path, mime=True) or "" + except Exception: + mime_type = "" + if mime_type.startswith("video/"): + return "video" + if mime_type.startswith("audio/"): + return "audio" + if mime_type.startswith("image/"): + return "image" + if mime_type in {"text/plain", "application/x-subrip"} or path.lower().endswith( + (".srt", ".ass", ".vtt") + ): + return "text" + return "unknown" + + +def validate_files(task_id: str, paths: List[str]) -> None: + task = get_task(task_id) + if len(paths) < task.min_files: + raise HTTPException( + status_code=400, + detail=f"{task.label} requires at least {task.min_files} file(s).", + ) + if task.max_files is not None and len(paths) > task.max_files: + raise HTTPException( + status_code=400, + detail=f"{task.label} accepts at most {task.max_files} file(s).", + ) + for index, path in enumerate(paths): + if not os.path.exists(path): + raise HTTPException(status_code=400, detail=f"Input not found: {path}") + if os.path.getsize(path) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=413, + detail=f"{os.path.basename(path)} exceeds {MAX_UPLOAD_MB} MB.", + ) + category = file_category(path) + expected_types = ( + task.file_types[index] + if task.file_types and index < len(task.file_types) + else task.accepted_types + ) + if category not in expected_types: + raise HTTPException( + status_code=400, + detail=( + f"{os.path.basename(path)} is {category}; " + f"{task.label} expects {', '.join(expected_types)}." + ), + ) + + +def get_task(task_id: str) -> TaskDefinition: + task = TASKS.get(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Unknown task: {task_id}") + return task + + +def output_path(task_id: str, ext: str, job_id: Optional[str] = None) -> str: + safe_job_id = job_id or uuid.uuid4().hex[:8] + return os.path.abspath(os.path.join(TEMP_DIR, f"{task_id}_{safe_job_id}.{ext}")) + + +def zip_frames(pattern_prefix: str, zip_path: str) -> str: + frame_dir = os.path.dirname(pattern_prefix) + prefix = os.path.basename(pattern_prefix).split("_%04d")[0] + archive_base = zip_path[:-4] + frame_paths = [ + os.path.join(frame_dir, name) + for name in os.listdir(frame_dir) + if name.startswith(prefix) and name.endswith(".jpg") + ] + if not frame_paths: + raise HTTPException(status_code=500, detail="No frames were generated.") + temp_archive_dir = os.path.join(TEMP_DIR, f"frames_{uuid.uuid4().hex[:8]}") + os.makedirs(temp_archive_dir, exist_ok=True) + try: + for frame_path in frame_paths: + shutil.copy2(frame_path, os.path.join(temp_archive_dir, os.path.basename(frame_path))) + shutil.make_archive(archive_base, "zip", temp_archive_dir) + finally: + shutil.rmtree(temp_archive_dir, ignore_errors=True) + return zip_path + + +def run_ffmpeg(task_id: str, paths: List[str], options: Optional[Dict[str, str]] = None) -> str: + task = get_task(task_id) + opts = options or {} + validate_files(task_id, paths) + job_id = uuid.uuid4().hex[:8] + out = output_path(task_id, task.output_ext, job_id) + if task.runner: + try: + return task.runner(paths, opts, out) + except subprocess.CalledProcessError as exc: + stderr = exc.stderr.strip()[-2000:] or "FFmpeg failed without stderr." + raise HTTPException(status_code=500, detail=stderr) + if not task.builder: + raise HTTPException(status_code=500, detail=f"{task_id} is not executable.") + cmd = task.builder(paths, opts, out) + try: + run_command(cmd) + if task_id == "extract_frames": + out = zip_frames(out.replace(".zip", "_%04d.jpg"), out) + return out + except subprocess.CalledProcessError as exc: + stderr = exc.stderr.strip()[-2000:] or "FFmpeg failed without stderr." + raise HTTPException(status_code=500, detail=stderr) + + +def record_history(job_id: str, task_id: str, output: str) -> None: + task = get_task(task_id) + history_id = uuid.uuid4().hex[:12] + item = { + "history_id": history_id, + "job_id": job_id, + "task_id": task_id, + "task": task.label, + "output": output, + "filename": os.path.basename(output), + "size": os.path.getsize(output) if os.path.exists(output) else 0, + "created_at": int(time.time()), + "download_url": f"/history/{history_id}/download", + } + with STATE_LOCK: + HISTORY.insert(0, item) + del HISTORY[HISTORY_LIMIT:] + persist_history(item) + + +def update_job(job_id: str, **updates: object) -> None: + with STATE_LOCK: + job = JOBS.get(job_id) + if not job: + return + job.update(updates) + snapshot = dict(job) + persist_job(snapshot) + + +def execute_job(job_id: str, task_id: str, paths: List[str], options: Dict[str, str]) -> None: + update_job(job_id, status="running", message="Running FFmpeg") + try: + output = run_ffmpeg(task_id, paths, options) + record_history(job_id, task_id, output) + update_job(job_id, status="complete", message="Complete", output=output) + except HTTPException as exc: + update_job(job_id, status="failed", message=exc.detail) + except Exception as exc: + update_job(job_id, status="failed", message=str(exc)) + + +async def save_uploads(files: Optional[List[UploadFile]]) -> List[str]: + saved = [] + if not files: + return saved + for upload in files: + filename = os.path.basename(upload.filename or "upload.bin") + path = os.path.join(TEMP_DIR, f"{uuid.uuid4().hex}_{filename}") + size = 0 + with open(path, "wb") as handle: + while chunk := await upload.read(1024 * 1024): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + handle.close() + remove_file_quietly(path) + raise HTTPException( + status_code=413, + detail=f"{filename} exceeds {MAX_UPLOAD_MB} MB.", + ) + handle.write(chunk) + saved.append(path) + return saved + + +def safe_upload_name(filename: str, fallback: str = "upload.bin") -> str: + cleaned = os.path.basename(filename or fallback).strip() + return cleaned or fallback + + +def save_bytes_file(content: bytes, filename: str) -> str: + if len(content) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=413, + detail=f"{safe_upload_name(filename)} exceeds {MAX_UPLOAD_MB} MB.", + ) + path = os.path.abspath( + os.path.join(TEMP_DIR, f"{uuid.uuid4().hex}_{safe_upload_name(filename)}") + ) + with open(path, "wb") as handle: + handle.write(content) + return path + + +def decode_base64_content(value: str) -> bytes: + if "," in value and value.lstrip().startswith("data:"): + value = value.split(",", 1)[1] + try: + return base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as exc: + raise HTTPException(status_code=400, detail="Invalid base64 file content.") from exc + + +def collect_base64_files(payload: object) -> List[str]: + saved = [] + + def walk(value: object, inherited_name: str = "n8n-upload.bin") -> None: + if isinstance(value, list): + for item in value: + walk(item, inherited_name) + return + if not isinstance(value, dict): + return + + filename = str( + value.get("fileName") + or value.get("filename") + or value.get("name") + or inherited_name + ) + content = ( + value.get("content_base64") + or value.get("base64") + or value.get("content") + or value.get("data") + ) + if isinstance(content, str) and ( + value.get("mimeType") + or value.get("mime_type") + or value.get("fileName") + or value.get("filename") + or value.get("base64") + or value.get("content_base64") + ): + saved.append(save_bytes_file(decode_base64_content(content), filename)) + return + + for key, nested in value.items(): + walk(nested, str(key)) + + walk(payload) + return saved + + +def split_url_values(value: object) -> List[str]: + if not value: + return list() + if isinstance(value, list): + urls = [] + for item in value: + urls.extend(split_url_values(item)) + return urls + if not isinstance(value, str): + return list() + chunks = value.replace(",", "\n").splitlines() + return [chunk.strip() for chunk in chunks if chunk.strip()] + + +def options_from_mapping(values: Dict[str, object]) -> Dict[str, str]: + nested_options = values.get("options") + merged = dict(values) + if isinstance(nested_options, dict): + merged.update(nested_options) + return { + field: "" if merged.get(field) is None else str(merged.get(field, "")) + for field in OPTION_FIELDS + } + + +async def collect_n8n_inputs(request: Request) -> tuple[List[str], Dict[str, str]]: + content_type = request.headers.get("content-type", "").lower() + saved: List[str] = [] + values: Dict[str, object] = {} + + if "multipart/form-data" in content_type: + form = await request.form() + uploads = [] + urls = [] + for key, value in form.multi_items(): + if hasattr(value, "filename") and hasattr(value, "read"): + uploads.append(value) + else: + values[key] = value + if key in {"url", "urls", "file_url", "source_url", "download_url"}: + urls.extend(split_url_values(value)) + saved.extend(await save_uploads(uploads)) + for url in urls: + saved.append(download_url(url)) + return saved, options_from_mapping(values) + + if "application/json" in content_type: + payload = await request.json() + if not isinstance(payload, dict): + raise HTTPException(status_code=400, detail="JSON body must be an object.") + values.update(payload) + for key in ("url", "urls", "file_url", "source_url", "download_url"): + for url in split_url_values(payload.get(key)): + saved.append(download_url(url)) + saved.extend(collect_base64_files(payload.get("files", []))) + saved.extend(collect_base64_files(payload.get("binary", {}))) + saved.extend(collect_base64_files(payload.get("data", {}))) + if not saved: + saved.extend(collect_base64_files(payload)) + return saved, options_from_mapping(values) + + body = await request.body() + if body: + filename = ( + request.headers.get("x-filename") + or request.headers.get("x-file-name") + or "n8n-upload.bin" + ) + saved.append(save_bytes_file(body, filename)) + return saved, options_from_mapping(dict(request.query_params)) + + +def download_url(url: str) -> str: + if not ALLOW_URL_INPUTS: + raise HTTPException(status_code=403, detail="URL inputs are disabled.") + outtmpl = os.path.join(TEMP_DIR, f"{uuid.uuid4().hex}.%(ext)s") + ydl_opts = { + "outtmpl": outtmpl, + "format": "bestvideo+bestaudio/best", + "max_filesize": MAX_URL_DOWNLOAD_BYTES, + "noplaylist": True, + "quiet": True, + "no_warnings": True, + } + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(url, download=True) + path = ydl.prepare_filename(info) + if not os.path.exists(path): + raise HTTPException(status_code=502, detail="URL download did not produce a file.") + if os.path.getsize(path) > MAX_URL_DOWNLOAD_BYTES: + remove_file_quietly(path) + raise HTTPException( + status_code=413, + detail=f"Downloaded media exceeds {MAX_URL_DOWNLOAD_MB} MB.", + ) + return path + + +def options_from_form( + text: str, + start_time: str, + end_time: str, + duration: str, + aspect_ratio: str, + resolution: str, + crf: str, + preset: str, + audio_bitrate: str, + volume: str, + position: str, + opacity: str, + fps: str, + width: str, + speed: str, + timestamp: str, + image_duration: str, + frame_rate: str, + font_size: str, + wave_color: str, +) -> Dict[str, str]: + return { + "text": text or "", + "start_time": start_time or "", + "end_time": end_time or "", + "duration": duration or "", + "aspect_ratio": aspect_ratio or "", + "resolution": resolution or "", + "crf": crf or "", + "preset": preset or "", + "audio_bitrate": audio_bitrate or "", + "volume": volume or "", + "position": position or "", + "opacity": opacity or "", + "fps": fps or "", + "width": width or "", + "speed": speed or "", + "timestamp": timestamp or "", + "image_duration": image_duration or "", + "frame_rate": frame_rate or "", + "font_size": font_size or "", + "wave_color": wave_color or "", + } + + +@app.get("/healthz") +def healthz(): + return {"status": "ok", "checks": production_checks()} + + +@app.get("/readyz") +def readyz(): + checks = production_checks() + if not production_ready(): + raise HTTPException(status_code=503, detail={"status": "not_ready", "checks": checks}) + return {"status": "ready", "checks": checks} + + +@app.get("/tasks") +def list_tasks(): + return { + task_id: { + "label": task.label, + "description": task.description, + "category": task.category, + "output_ext": task.output_ext, + "min_files": task.min_files, + "max_files": task.max_files, + "accepted_types": task.accepted_types, + "file_types": task.file_types, + } + for task_id, task in TASKS.items() + } + + +@app.post("/n8n/execute/{task_id}") +async def n8n_execute(task_id: str, request: Request): + task = get_task(task_id) + saved, options = await collect_n8n_inputs(request) + if not saved and task.min_files > 0: + raise HTTPException(status_code=400, detail="No n8n binary, base64 file, raw body, or URL input provided.") + output = run_ffmpeg(task_id, saved, options) + record_history(uuid.uuid4().hex[:8], task_id, output) + return FileResponse(output, filename=os.path.basename(output)) + + +@app.post("/n8n/jobs/{task_id}") +async def n8n_create_job(task_id: str, request: Request, background_tasks: BackgroundTasks): + task = get_task(task_id) + saved, options = await collect_n8n_inputs(request) + if not saved and task.min_files > 0: + raise HTTPException(status_code=400, detail="No n8n binary, base64 file, raw body, or URL input provided.") + job_id = uuid.uuid4().hex[:12] + job = { + "job_id": job_id, + "task_id": task_id, + "status": "queued", + "message": "Queued", + "output": None, + "created_at": int(time.time()), + } + with STATE_LOCK: + JOBS[job_id] = job + persist_job(job) + background_tasks.add_task(execute_job, job_id, task_id, saved, options) + return {"job_id": job_id, "status_url": f"/status/{job_id}"} + + +@app.post("/execute/{task_id}") +async def api_call( + task_id: str, + files: List[UploadFile] = File(None), + url: Optional[str] = Form(None), + text: str = Form(""), + start_time: str = Form(""), + end_time: str = Form(""), + duration: str = Form(""), + aspect_ratio: str = Form(""), + resolution: str = Form(""), + crf: str = Form(""), + preset: str = Form(""), + audio_bitrate: str = Form(""), + volume: str = Form(""), + position: str = Form(""), + opacity: str = Form(""), + fps: str = Form(""), + width: str = Form(""), + speed: str = Form(""), + timestamp: str = Form(""), + image_duration: str = Form(""), + frame_rate: str = Form(""), + font_size: str = Form(""), + wave_color: str = Form(""), +): + task = get_task(task_id) + saved = [] + if url: + saved.append(download_url(url)) + saved.extend(await save_uploads(files)) + if not saved and task.min_files > 0: + raise HTTPException(status_code=400, detail="No input provided") + options = options_from_form( + text, + start_time, + end_time, + duration, + aspect_ratio, + resolution, + crf, + preset, + audio_bitrate, + volume, + position, + opacity, + fps, + width, + speed, + timestamp, + image_duration, + frame_rate, + font_size, + wave_color, + ) + output = run_ffmpeg(task_id, saved, options) + record_history(uuid.uuid4().hex[:8], task_id, output) + return FileResponse(output, filename=os.path.basename(output)) + + +@app.post("/jobs/{task_id}") +async def create_job( + task_id: str, + background_tasks: BackgroundTasks, + files: List[UploadFile] = File(None), + url: Optional[str] = Form(None), + text: str = Form(""), + start_time: str = Form(""), + end_time: str = Form(""), + duration: str = Form(""), + aspect_ratio: str = Form(""), + resolution: str = Form(""), + crf: str = Form(""), + preset: str = Form(""), + audio_bitrate: str = Form(""), + volume: str = Form(""), + position: str = Form(""), + opacity: str = Form(""), + fps: str = Form(""), + width: str = Form(""), + speed: str = Form(""), + timestamp: str = Form(""), + image_duration: str = Form(""), + frame_rate: str = Form(""), + font_size: str = Form(""), + wave_color: str = Form(""), +): + task = get_task(task_id) + saved = [] + if url: + saved.append(download_url(url)) + saved.extend(await save_uploads(files)) + if not saved and task.min_files > 0: + raise HTTPException(status_code=400, detail="No input provided") + options = options_from_form( + text, + start_time, + end_time, + duration, + aspect_ratio, + resolution, + crf, + preset, + audio_bitrate, + volume, + position, + opacity, + fps, + width, + speed, + timestamp, + image_duration, + frame_rate, + font_size, + wave_color, + ) + job_id = uuid.uuid4().hex[:12] + job = { + "job_id": job_id, + "task_id": task_id, + "status": "queued", + "message": "Queued", + "output": None, + "created_at": int(time.time()), + } + with STATE_LOCK: + JOBS[job_id] = job + persist_job(job) + background_tasks.add_task(execute_job, job_id, task_id, saved, options) + return {"job_id": job_id, "status_url": f"/status/{job_id}"} + + +@app.get("/status/{job_id}") +def job_status(job_id: str): + with STATE_LOCK: + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + data = dict(job) + if data.get("output"): + data["download_url"] = f"/download/{job_id}" + return data + + +@app.get("/download/{job_id}") +def download_job(job_id: str): + with STATE_LOCK: + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + output = job.get("output") + if not output or not os.path.exists(str(output)): + raise HTTPException(status_code=404, detail="Output not available") + return FileResponse(str(output), filename=os.path.basename(str(output))) + + +@app.get("/history") +def history(): + with STATE_LOCK: + return [ + {key: value for key, value in item.items() if key != "output"} + for item in HISTORY + ] + + +@app.get("/history/{history_id}/download") +def download_history_item(history_id: str): + with STATE_LOCK: + item = next( + (entry for entry in HISTORY if entry.get("history_id") == history_id), None + ) + if not item: + raise HTTPException(status_code=404, detail="History item not found") + output = str(item["output"]) + if not os.path.exists(output): + raise HTTPException(status_code=404, detail="Output expired") + return FileResponse(output, filename=os.path.basename(output)) + + +def task_choices() -> List[str]: + return [f"{task.label} ({task_id})" for task_id, task in TASKS.items()] + + +def selected_task_id(choice: str) -> str: + if not choice: + return "normalize" + if "(" in choice and choice.endswith(")"): + return choice.rsplit("(", 1)[1][:-1] + return choice if choice in TASKS else "normalize" + + +def describe_task(choice: str) -> str: + task_id = selected_task_id(choice) + task = get_task(task_id) + max_files = "unlimited" if task.max_files is None else str(task.max_files) + return ( + f"{task.description}\n\n" + f"Files: {task.min_files}-{max_files}. " + f"Accepted: {', '.join(task.accepted_types)}. " + f"Output: .{task.output_ext}" + ) + + +def option_visibility(choice: str): + task_id = selected_task_id(choice) + text_tasks = { + "text_overlay", + "tiktok_lyrics", + "tiktok_pro_reframer", + "reels_safe_caption", + "reels_hook_title", + "faceless_quote_card", + "faceless_story_pages", + "faceless_image_narration", + "faceless_video_narration", + "series_split_pack", + "series_episode_badge", + "series_batch_pack", + "series_recap_card", + } + duration_tasks = { + "make_gif", + "tiktok_lyrics", + "reels_progress_bar", + "reels_loop", + "faceless_quote_card", + "faceless_broll_montage", + "series_split_pack", + "series_recap_card", + } + quality_tasks = { + "normalize", + "compress", + "batch_compress", + "reels_blur_fit", + "reels_loop", + "reels_reaction_stack", + "faceless_image_narration", + "faceless_video_narration", + "faceless_broll_montage", + "series_split_pack", + "series_batch_pack", + } + audio_tasks = {"extract_audio", "merge_music", "reels_audio_duck"} + font_tasks = { + "text_overlay", + "tiktok_lyrics", + "tiktok_pro_reframer", + "reels_safe_caption", + "reels_hook_title", + "faceless_quote_card", + "faceless_story_pages", + "faceless_image_narration", + "faceless_video_narration", + "series_episode_badge", + "series_recap_card", + } + visible = { + "text": task_id in text_tasks, + "trim": task_id == "trim", + "duration": task_id in duration_tasks, + "timestamp": task_id == "thumbnail", + "aspect": task_id in {"resize_916", "crop_aspect"}, + "resolution": task_id == "compress", + "quality": task_id in quality_tasks, + "audio": task_id in audio_tasks, + "watermark": task_id == "watermark", + "gif": task_id == "make_gif", + "speed": task_id == "speed", + "slideshow": task_id in {"slideshow", "faceless_story_pages", "faceless_broll_montage"}, + "frames": task_id == "extract_frames", + "font": task_id in font_tasks, + "wave": task_id == "waveform", + } + return [ + describe_task(choice), + gr.update(visible=visible["text"]), + gr.update(visible=visible["trim"]), + gr.update(visible=visible["duration"]), + gr.update(visible=visible["timestamp"]), + gr.update(visible=visible["aspect"]), + gr.update(visible=visible["resolution"]), + gr.update(visible=visible["quality"]), + gr.update(visible=visible["audio"]), + gr.update(visible=visible["watermark"]), + gr.update(visible=visible["gif"]), + gr.update(visible=visible["speed"]), + gr.update(visible=visible["slideshow"]), + gr.update(visible=visible["frames"]), + gr.update(visible=visible["font"]), + gr.update(visible=visible["wave"]), + ] + + +def ui_handler( + files, + url, + task_choice, + text, + start_time, + end_time, + duration, + aspect_ratio, + resolution, + crf, + preset, + audio_bitrate, + volume, + position, + opacity, + fps, + width, + speed, + timestamp, + image_duration, + frame_rate, + font_size, + wave_color, +): + task_id = selected_task_id(task_choice) + task = get_task(task_id) + paths = [] + if url: + paths.append(download_url(url)) + if files: + paths.extend([file.name for file in files]) + if not paths and task.min_files > 0: + raise gr.Error("Upload files or provide a URL.") + options = options_from_form( + text, + start_time, + end_time, + duration, + aspect_ratio, + resolution, + crf, + preset, + audio_bitrate, + volume, + position, + opacity, + fps, + width, + speed, + timestamp, + image_duration, + frame_rate, + font_size, + wave_color, + ) + try: + output = run_ffmpeg(task_id, paths, options) + record_history(uuid.uuid4().hex[:8], task_id, output) + return output, history_table() + except HTTPException as exc: + raise gr.Error(str(exc.detail)) + + +def history_table(): + with STATE_LOCK: + rows = [ + [ + item["task"], + item["filename"], + item["size"], + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(item["created_at"])), + ] + for item in HISTORY[:10] + ] + return rows + + +with gr.Blocks(title="Basyx FFmpeg Automation Hub") as ui: + gr.Markdown("# Basyx FFmpeg Automation Hub") + gr.Markdown("Upload media, choose a task, tune the options, and download the output.") + with gr.Row(): + with gr.Column(scale=1): + task_input = gr.Dropdown( + choices=task_choices(), + value=task_choices()[0], + label="Task", + ) + task_info = gr.Textbox( + label="Task requirements", value=describe_task(task_choices()[0]), lines=4 + ) + file_input = gr.File(label="Files", file_count="multiple") + url_input = gr.Textbox(label="Media URL") + with gr.Column(scale=1): + text_input = gr.Textbox(label="Text / Caption / Lyrics", lines=3, visible=False) + with gr.Row(visible=False) as trim_row: + start_input = gr.Textbox(label="Start", value="00:00:00") + end_input = gr.Textbox(label="End") + with gr.Row(visible=False) as duration_row: + duration_input = gr.Number(label="Duration seconds", value=5, precision=0) + with gr.Row(visible=False) as timestamp_row: + timestamp_input = gr.Textbox(label="Thumbnail timestamp", value="00:00:02") + with gr.Row(visible=False) as aspect_row: + aspect_input = gr.Dropdown(["9:16", "16:9", "1:1", "4:5"], label="Aspect", value="9:16") + with gr.Row(visible=False) as resolution_row: + resolution_input = gr.Dropdown(["480p", "720p", "1080p"], label="Resolution", value="720p") + with gr.Row(visible=True) as quality_row: + crf_input = gr.Slider(12, 35, value=23, step=1, label="CRF") + preset_input = gr.Dropdown( + ["ultrafast", "veryfast", "fast", "medium", "slow"], + label="Preset", + value="fast", + ) + with gr.Row(visible=False) as audio_row: + audio_bitrate_input = gr.Slider(64, 320, value=192, step=16, label="Audio kbps") + volume_input = gr.Slider(0, 2, value=0.3, step=0.05, label="Music volume") + with gr.Row(visible=False) as watermark_row: + position_input = gr.Dropdown( + ["bottom-right", "bottom-left", "top-right", "top-left", "center"], + label="Watermark position", + value="bottom-right", + ) + opacity_input = gr.Slider(0.1, 1, value=1, step=0.05, label="Watermark opacity") + with gr.Row(visible=False) as gif_row: + fps_input = gr.Slider(5, 30, value=10, step=1, label="GIF FPS") + width_input = gr.Slider(240, 1080, value=480, step=20, label="GIF width") + with gr.Row(visible=False) as speed_row: + speed_input = gr.Slider(0.25, 4, value=1.25, step=0.05, label="Speed") + with gr.Row(visible=False) as slideshow_row: + image_duration_input = gr.Slider(1, 15, value=3, step=1, label="Slide seconds") + with gr.Row(visible=False) as frames_row: + frame_rate_input = gr.Slider(0.1, 30, value=1, step=0.1, label="Frame FPS") + with gr.Row(visible=False) as font_row: + font_size_input = gr.Slider(18, 160, value=60, step=1, label="Font size") + wave_color_input = gr.Textbox(label="Wave color hex", value="00e5ff", visible=False) + run_button = gr.Button("Run") + output_file = gr.File(label="Output") + history_output = gr.Dataframe( + headers=["Task", "Filename", "Bytes", "Created"], + datatype=["str", "str", "number", "str"], + label="Recent outputs", + value=history_table, + ) + + task_input.change( + option_visibility, + inputs=task_input, + outputs=[ + task_info, + text_input, + trim_row, + duration_row, + timestamp_row, + aspect_row, + resolution_row, + quality_row, + audio_row, + watermark_row, + gif_row, + speed_row, + slideshow_row, + frames_row, + font_row, + wave_color_input, + ], + ) + run_button.click( + ui_handler, + inputs=[ + file_input, + url_input, + task_input, + text_input, + start_input, + end_input, + duration_input, + aspect_input, + resolution_input, + crf_input, + preset_input, + audio_bitrate_input, + volume_input, + position_input, + opacity_input, + fps_input, + width_input, + speed_input, + timestamp_input, + image_duration_input, + frame_rate_input, + font_size_input, + wave_color_input, + ], + outputs=[output_file, history_output], + ) + +app = gr.mount_gradio_app(app, ui, path="/") + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=7860) diff --git a/services/ffmpeg_automation/requirements.txt b/services/ffmpeg_automation/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..6582b936158fedf26b19f9a1055bf18e8aaae27f --- /dev/null +++ b/services/ffmpeg_automation/requirements.txt @@ -0,0 +1,6 @@ +fastapi +uvicorn +gradio +python-multipart +python-magic +yt-dlp diff --git a/services/ktts/.gitattributes b/services/ktts/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/services/ktts/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/services/ktts/.gitignore b/services/ktts/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..15201acc113da01edf6fa2fb2708b2e9076b6bc5 --- /dev/null +++ b/services/ktts/.gitignore @@ -0,0 +1,171 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# PyPI configuration file +.pypirc diff --git a/services/ktts/Dockerfile b/services/ktts/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0344f7bc625e311cd7e17de8dc95d0bfbb92c35a --- /dev/null +++ b/services/ktts/Dockerfile @@ -0,0 +1,22 @@ +# Base image with Python and essential libraries +FROM python:3.8-slim + +# Install system-level dependencies +RUN apt-get update && \ + apt-get install -y espeak && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /app + +# Copy your files into the container +COPY . . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Expose the port +EXPOSE 7860 + +# Run your app +CMD ["python", "app.py"] \ No newline at end of file diff --git a/services/ktts/README.md b/services/ktts/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b537818fae5bdf9161a7114893dc88ded8ec7d6a --- /dev/null +++ b/services/ktts/README.md @@ -0,0 +1,9 @@ +--- +title: "Kokoro-82M TTS ONNX" +emoji: "💡" +colorFrom: "blue" +colorTo: "green" +sdk: "docker" +app_file: app.py +pinned: false +--- diff --git a/services/ktts/README_git.md b/services/ktts/README_git.md new file mode 100644 index 0000000000000000000000000000000000000000..545b58d6e2903abc3d829b1bbc832545bb41d9b8 --- /dev/null +++ b/services/ktts/README_git.md @@ -0,0 +1,94 @@ +# Kokoro-82M ONNX Runtime Inference + +![Downloads](https://img.shields.io/github/downloads/yakhyo/kokoro-82m-onnx/total) +[![GitHub Repo stars](https://img.shields.io/github/stars/yakhyo/kokoro-82m-onnx)](https://github.com/yakhyo/kokoro-82m-onnx/stargazers) +[![GitHub Repository](https://img.shields.io/badge/GitHub-Repository-blue?logo=github)](https://github.com/yakhyo/kokoro-82m-onnx) + +This repository contains minimal code and resources for inference using the **Kokoro-82M** model. The repository supports inference using **ONNX Runtime**. + + + + + + + + + + +
Machine learning models rely on large datasets and complex algorithms to identify patterns and make predictions.Did you know that honey never spoils? Archaeologists have found pots of honey in ancient Egyptian tombs that are over 3,000 years old and still edible!
+ + + +
+ +## Features + +- **ONNX Runtime Inference**: Kokoro-82M (v0_19) Minimal ONNX Runtime Inference code. It supports `en-us` and `en-gb`. + +--- + +## Installation + +1. Clone the repository: + + ```bash + git clone https://github.com/yakhyo/kokoro-82m.git + cd kokoro-82m + ``` + +2. Install dependencies: + + ```bash + pip install -r requirements.txt + ``` + +3. Install `espeak` for text-to-speech functionality: + Linux: + ```bash + apt-get install espeak -y + ``` + +--- + +## Usage + +### Download ONNX Model + +[click to download](https://github.com/yakhyo/kokoro-82m/releases/download/v0.0.1/kokoro-v0_19.onnx) + +### Jupyter Notebook Inference Example + +Run inference using the jupyter notebook: + +[example.ipynb](example.ipynb) + +### CLI Inference + +Specify input text and model weights in `inference.py` then run: + +```bash +python inference.py +``` + +### Gradio App + +Run below start Gradio App +```bash +python app.py +``` +
+ 0: + try: + style_vector_path = str(BASE_DIR / "voices" / style_vector) + model_path_full = str(BASE_DIR / "weights" / model_path) + cache_key = (model_path_full, style_vector_path) + + if tokenizer_cache is None: + tokenizer_cache = Tokenizer() + + if cache_key not in kokoro_cache: + model_status = "loading" + model_error = None + kokoro_cache[cache_key] = Kokoro(model_path_full, style_vector_path, tokenizer=tokenizer_cache, lang='en-us') + model_status = "ready" + + inference = kokoro_cache[cache_key] + + audio, sample_rate = inference.generate_audio(text, speed=speed) + + with tempfile.NamedTemporaryFile(suffix=f".{output_file_format}", delete=False) as temp_file: + sf.write(temp_file.name, audio, sample_rate) + temp_file_path = temp_file.name + + return temp_file_path + + except Exception as e: + model_status = "failed" + model_error = str(e) + raise gr.Error(f"An error occurred during TTS inference: {str(e)}") + else: + raise gr.Error("Input text cannot be empty.") + +style_vector_choices = get_style_vector_choices() +onnx_models_choices = get_onnx_models() + +sample_outputs = [ + ("Educational Note", "Machine learning models rely on large datasets and complex algorithms to identify patterns and make predictions.", str(BASE_DIR / "assets" / "edu_note.wav")), + ("Fun Fact", "Did you know that honey never spoils? Archaeologists have found pots of honey in ancient Egyptian tombs that are over 3,000 years old and still edible!", str(BASE_DIR / "assets" / "fun_fact.wav")), + ("Thanks", "Thank you for listening to this audio. It was generated by the Kokoro TTS model.", str(BASE_DIR / "assets" / "thanks.wav")) +] + +example_texts = [ + ["Machine learning models rely on large datasets and complex algorithms to identify patterns and make predictions."], + ["Did you know that honey never spoils? Archaeologists have found pots of honey in ancient Egyptian tombs that are over 3,000 years old and still edible!"], + ["Thank you for listening to this audio. It was generated by the Kokoro TTS model."] +] + +# --- GRADIO INTERFACE (UNCHANGED) --- + +with gr.Blocks() as demo: + gr.Markdown("##
Kokoro TTS ONNX Inference | [GitHub Link](https://github.com/yakhyo/kokoro-onnx)
") + with gr.Row(variant="panel"): + model_path = gr.Dropdown(choices=onnx_models_choices, label="ONNX Model Path", value=onnx_models_choices[0]) + style_vector = gr.Dropdown(choices=style_vector_choices, label="Style Vector", value=style_vector_choices[0]) + output_file_format = gr.Dropdown(choices=["wav", "mp3"], label="Output Format", value="wav") + speed = gr.Slider(minimum=0.5, maximum=2.0, value=1.0, step=0.1, label="Speed") + + text = gr.Textbox(label="Input Text", placeholder="Enter text to convert to speech.") + btn = gr.Button("Generate Speech") + output_audio = gr.Audio(label="Generated Audio", type="filepath") + + btn.click(fn=local_tts, inputs=[text, model_path, style_vector, output_file_format, speed], outputs=output_audio) + + gr.Examples(examples=example_texts, inputs=[text], label="Click an example to populate the input text") + gr.Markdown("### Sample Texts and Audio") + for topic, sample_text, sample_audio in sample_outputs: + with gr.Row(): + gr.Textbox(value=sample_text, label=topic, interactive=False) + gr.Audio(value=sample_audio, label="Example Audio", type="filepath", interactive=False) + +# --- FASTAPI WRAPPER & STARTUP --- + +app = FastAPI() + +@app.post("/v1/audio/speech") +async def api_speech(request: Request): + """ + OpenAI-compatible /v1/audio/speech endpoint for automation. + Expects JSON: {"input": "text", "voice": "voice_file.pt", "model": "model_file.onnx"} + """ + data = await request.json() + input_text = data.get("input", "") + m_path = data.get("model", onnx_models_choices[0]) + s_vec = data.get("voice", style_vector_choices[0]) + spd = float(data.get("speed", 1.0)) + + file_path = local_tts(input_text, m_path, s_vec, speed=spd) + return FileResponse(file_path, media_type="audio/wav") + +# Mount Gradio into the FastAPI app +app = gr.mount_gradio_app(app, demo, path="/") + +if __name__ == "__main__": + # Runs on port 7860 as expected by the Dockerfile/Hugging Face + uvicorn.run(app, host="0.0.0.0", port=7860) diff --git a/services/ktts/assets/edu_note.wav b/services/ktts/assets/edu_note.wav new file mode 100644 index 0000000000000000000000000000000000000000..e5bbd527fba7db4a0bbfa3bf4116699c6eefb3c3 --- /dev/null +++ b/services/ktts/assets/edu_note.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880362273fe0348d8e73e9cf99cbe58573ccaac1e51628e92d4a362c8199d399 +size 416444 diff --git a/services/ktts/assets/fun_fact.wav b/services/ktts/assets/fun_fact.wav new file mode 100644 index 0000000000000000000000000000000000000000..58949399a97d7511b438fbce1855f1888e4d63bd --- /dev/null +++ b/services/ktts/assets/fun_fact.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:245b8db3fc90671b02baa477ce0c2d08754b225ae95a853687191ad126de562e +size 496844 diff --git a/services/ktts/assets/thanks.wav b/services/ktts/assets/thanks.wav new file mode 100644 index 0000000000000000000000000000000000000000..8ff68850b72654f7a9187fff3bfb6f66dd07bc5d --- /dev/null +++ b/services/ktts/assets/thanks.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04843d86bff241ff659cf6c6931641797411dbf3b1532eee83914058bbc5a64 +size 288044 diff --git a/services/ktts/example.ipynb b/services/ktts/example.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2ba06efda075bd5d285502be812e6ac52a4d3b94 --- /dev/null +++ b/services/ktts/example.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Kokoro-82M ONNX Runtime Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import required libraries\n", + "import numpy as np\n", + "from IPython.display import display, Audio\n", + "\n", + "# Import the Kokoro model class\n", + "from models import Tokenizer, Kokoro" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "model_path = \"weights/kokoro-v0_19.onnx\"\n", + "output_filename = \"output.wav\"\n", + "style_vector_path = \"voices/af_bella.pt\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Tokenizer and Kokoro model" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = Tokenizer()\n", + "\n", + "kokoro = Kokoro(model_path, style_vector_path, tokenizer=tokenizer, lang='en-us')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "text = (\n", + " \"This approach ensures the entire text is processed without exceeding the token limit and outputs seamless audio for the full input. \"\n", + " \"Let me know if you need further assistance!\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate audio" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "audio, sample_rate = kokoro.generate_audio(text, speed=1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Display the output on jupyter" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(Audio(data=audio, rate=24000, autoplay=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Save model the output" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Audio saved to output.wav\n" + ] + } + ], + "source": [ + "import soundfile as sf\n", + "\n", + "sf.write(output_filename, audio, sample_rate)\n", + "print(f\"Audio saved to {output_filename}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "torch", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/services/ktts/gradio_demo.png b/services/ktts/gradio_demo.png new file mode 100644 index 0000000000000000000000000000000000000000..98167cd468b4fdb435c2b80c5a2a3825037da31a --- /dev/null +++ b/services/ktts/gradio_demo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f48f88f75dd3fe3c6130acc0ca8a981f6331a14dda21eb821d10500916f7883 +size 202899 diff --git a/services/ktts/inference.py b/services/ktts/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..d7d21ff86db42befc2a5c777d8576c538703695f --- /dev/null +++ b/services/ktts/inference.py @@ -0,0 +1,25 @@ +import soundfile as sf + +from models import Tokenizer, Kokoro + + +def main(): + model_path = "weights/kokoro-v0_19.onnx" + style_vector_path = "voices/af.pt" + output_filename = "test_out.wav" + tokenizer = Tokenizer() + + text = ( + "This approach ensures the entire text is processed without exceeding the token limit and outputs seamless audio for the full input. Let me know if you need further assistance!" + ) + + inference = Kokoro(model_path, style_vector_path, tokenizer=tokenizer, lang='en-us') + audio, sample_rate = inference.generate_audio(text, speed=1.0) + + # Save the audio to a file + sf.write(output_filename, audio, sample_rate) + print(f"Audio saved to {output_filename}") + + +if __name__ == "__main__": + main() diff --git a/services/ktts/models/__init__.py b/services/ktts/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..43e5b9954a240746979472d96e9c517ea5a3a71f --- /dev/null +++ b/services/ktts/models/__init__.py @@ -0,0 +1,2 @@ +from .kokoro import Kokoro +from .tokenizer import Tokenizer diff --git a/services/ktts/models/kokoro.py b/services/ktts/models/kokoro.py new file mode 100644 index 0000000000000000000000000000000000000000..34b5dd61126263ef3ee837265847b4b2e0a516bd --- /dev/null +++ b/services/ktts/models/kokoro.py @@ -0,0 +1,125 @@ +import torch +import numpy as np +import onnxruntime as ort + +TOKEN_LIMIT = 510 +SAMPLE_RATE = 24_000 + + +class Kokoro: + def __init__(self, model_path: str, style_vector_path: str, tokenizer, lang: str = 'en-us') -> None: + """ + Initializes the ONNXInference class. + + Args: + model_path (str): Path to the ONNX model file. + style_vector_path (str): Path to the style vector file. + lang (str): Language code for the tokenizer. + """ + self.sess = ort.InferenceSession(model_path) + self.style_vector_path = style_vector_path + self.tokenizer = tokenizer + self.lang = lang + + def preprocess(self, text): + """ + Converts input text to tokenized numerical IDs and loads the style vector. + + Args: + text (str): Input text to preprocess. + + Returns: + tuple: Tokenized input and corresponding style vector. + """ + # Convert text to phonemes and tokenize + phonemes = self.tokenizer.phonemize(text, lang=self.lang) + tokenized_phonemes = self.tokenizer.tokenize(phonemes) + + if not tokenized_phonemes: + raise ValueError("No tokens found after tokenization") + + style_vector = torch.load(self.style_vector_path, weights_only=True) + + if len(tokenized_phonemes) > TOKEN_LIMIT: + token_chunks = self.split_into_chunks(tokenized_phonemes) + + tokens_list = [] + styles_list = [] + + for chunk in token_chunks: + token_chunk = [[0, *chunk, 0]] + style_chunk = style_vector[len(chunk)].numpy() + + tokens_list.append(token_chunk) + styles_list.append(style_chunk) + + return tokens_list, styles_list + + style_vector = style_vector[len(tokenized_phonemes)].numpy() + tokenized_phonemes = [[0, *tokenized_phonemes, 0]] + + return tokenized_phonemes, style_vector + + @staticmethod + def split_into_chunks(tokens): + """ + Splits a list of tokens into chunks of size TOKEN_LIMIT. + + Args: + tokens (list): List of tokens to split. + + Returns: + list: List of token chunks. + """ + tokens_chunks = [] + for i in range(0, len(tokens), TOKEN_LIMIT): + tokens_chunks.append(tokens[i:i+TOKEN_LIMIT]) + return tokens_chunks + + def infer(self, tokens, style_vector, speed=1.0): + """ + Runs inference using the ONNX model. + + Args: + tokens (list): Tokenized input for the model. + style_vector (numpy.ndarray): Style vector for the model. + speed (float): Speed parameter for inference. + + Returns: + numpy.ndarray: Generated audio data. + """ + # Perform inference + audio = self.sess.run( + None, + { + 'tokens': tokens, + 'style': style_vector, + 'speed': np.array([speed], dtype=np.float32), + } + )[0] + return audio + + def generate_audio(self, text, speed=1.0): + """ + Full pipeline: preprocess, infer, and save the generated audio. + + Args: + text (str): Input text to generate audio from. + speed (float): Speed parameter for inference. + """ + # Preprocess text + tokenized_data, styles_data = self.preprocess(text) + + audio_segments = [] + if len(tokenized_data) > 1: # list of token chunks + for token_chunk, style_chunk in zip(tokenized_data, styles_data): + audio = self.infer(token_chunk, style_chunk, speed=speed) + audio_segments.append(audio) + else: # single token less than input limit + # Run inference + audio = self.infer(tokenized_data, styles_data, speed=speed) + audio_segments.append(audio) + + full_audio = np.concatenate(audio_segments) + + return full_audio, SAMPLE_RATE diff --git a/services/ktts/models/tokenizer.py b/services/ktts/models/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..878e50b9111570b5778f082d8b3a2c99c4f2546f --- /dev/null +++ b/services/ktts/models/tokenizer.py @@ -0,0 +1,238 @@ +import re +from phonemizer import backend +from typing import List + + +class Tokenizer: + def __init__(self): + self.VOCAB = self._get_vocab() + self.phonemizers = { + 'en-us': backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True), + 'en-gb': backend.EspeakBackend(language='en-gb', preserve_punctuation=True, with_stress=True), + } + + @staticmethod + def _get_vocab(): + """ + Generates a mapping of symbols to integer indices for tokenization. + + Returns: + dict: A dictionary where keys are symbols and values are unique integer indices. + """ + # Define the symbols + _pad = "$" + _punctuation = ';:,.!?¡¿—…"«»“” ' + _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + _letters_ipa = ( + "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" + ) + symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa) + + # Create a dictionary mapping each symbol to its index + return {symbol: index for index, symbol in enumerate(symbols)} + + @staticmethod + def split_num(num: re.Match) -> str: + """ + Processes numeric strings, formatting them as time, years, or other representations. + + Args: + num (re.Match): A regex match object representing the numeric string. + + Returns: + str: A formatted string based on the numeric input. + """ + num = num.group() + + # Handle time (e.g., "12:30") + if ':' in num: + hours, minutes = map(int, num.split(':')) + if minutes == 0: + return f"{hours} o'clock" + elif minutes < 10: + return f'{hours} oh {minutes}' + return f'{hours} {minutes}' + + # Handle years or general numeric cases + year = int(num[:4]) + if year < 1100 or year % 1000 < 10: + return num + + left, right = num[:2], int(num[2:4]) + suffix = 's' if num.endswith('s') else '' + + # Format years + if 100 <= year % 1000 <= 999: + if right == 0: + return f'{left} hundred{suffix}' + elif right < 10: + return f'{left} oh {right}{suffix}' + return f'{left} {right}{suffix}' + + @staticmethod + def flip_money(match: re.Match) -> str: + """ + Converts monetary values to a textual representation. + + Args: + m (re.Match): A regex match object representing the monetary value. + + Returns: + str: A formatted string describing the monetary value. + """ + m = m.group() + currency = 'dollar' if m[0] == '$' else 'pound' + + # Handle whole amounts (e.g., "$10", "£20") + if '.' not in m: + singular = '' if m[1:] == '1' else 's' + return f'{m[1:]} {currency}{singular}' + + # Handle amounts with decimals (e.g., "$10.50", "£5.25") + whole, cents = m[1:].split('.') + singular = '' if whole == '1' else 's' + cents = int(cents.ljust(2, '0')) # Ensure 2 decimal places + coins = f"cent{'' if cents == 1 else 's'}" if m[0] == '$' else ('penny' if cents == 1 else 'pence') + return f'{whole} {currency}{singular} and {cents} {coins}' + + @staticmethod + def point_num(match): + whole, fractional = match.group().split('.') + return ' point '.join([whole, ' '.join(fractional)]) + + def normalize_text(self, text: str) -> str: + """ + Normalizes input text by replacing special characters, punctuation, and applying custom transformations. + + Args: + text (str): Input text to normalize. + + Returns: + str: Normalized text. + """ + # Replace specific characters with standardized versions + replacements = { + chr(8216): "'", # Left single quotation mark + chr(8217): "'", # Right single quotation mark + '«': chr(8220), # Left double angle quotation mark to left double quotation mark + '»': chr(8221), # Right double angle quotation mark to right double quotation mark + chr(8220): '"', # Left double quotation mark + chr(8221): '"', # Right double quotation mark + '(': '«', # Replace parentheses with angle quotation marks + ')': '»' + } + for old, new in replacements.items(): + text = text.replace(old, new) + + # Replace punctuation and add spaces + punctuation_replacements = { + '、': ',', + '。': '.', + '！': '!', + '，': ',', + '：': ':', + '；': ';', + '？': '?', + } + for old, new in punctuation_replacements.items(): + text = text.replace(old, new + ' ') + + # Apply regex-based replacements + text = re.sub(r'[^\S\n]', ' ', text) + text = re.sub(r' +', ' ', text) + text = re.sub(r'(?<=\n) +(?=\n)', '', text) + + # Expand abbreviations and handle special cases + abbreviation_patterns = [ + (r'\bD[Rr]\.(?= [A-Z])', 'Doctor'), + (r'\b(?:Mr\.|MR\.(?= [A-Z]))', 'Mister'), + (r'\b(?:Ms\.|MS\.(?= [A-Z]))', 'Miss'), + (r'\b(?:Mrs\.|MRS\.(?= [A-Z]))', 'Mrs'), + (r'\betc\.(?! [A-Z])', 'etc'), + (r'(?i)\b(y)eah?\b', r"\1e'a"), + ] + for pattern, replacement in abbreviation_patterns: + text = re.sub(pattern, replacement, text) + + # Handle numbers and monetary values + text = re.sub(r'\d*\.\d+|\b\d{4}s?\b|(? List[int]: + """ + Tokenizes a given string into a list of indices based on VOCAB. + + Args: + text (str): Input string to tokenize. + + Returns: + list: A list of integer indices corresponding to the characters in the input string. + """ + return [self.VOCAB[x] for x in phonemes if x in self.VOCAB] + + def phonemize(self, text: str, lang: str = 'en-us', normalize: bool = True) -> str: + """ + Converts text to phonemes using the specified language phonemizer and applies normalization. + + Args: + text (str): Input text to be phonemized. + lang (str): Language identifier ('en-us' or 'en-gb') for selecting the phonemizer. + normalize (bool): Whether to normalize the text before phonemization. + + Returns: + str: A processed string of phonemes. + """ + # Normalize text if required + if normalize: + text = self.normalize_text(text) + + # Generate phonemes using the specified phonemizer + if lang not in self.phonemizers: + print(f"Language '{lang}' not supported. Defaulting to 'en-us'.") + lang = 'en-us' + + phonemes = self.phonemizers[lang].phonemize([text]) + phonemes = phonemes[0] if phonemes else '' + + # Apply custom phoneme replacements + replacements = { + 'kəkˈoːɹoʊ': 'kˈoʊkəɹoʊ', + 'kəkˈɔːɹəʊ': 'kˈəʊkəɹəʊ', + 'ʲ': 'j', + 'r': 'ɹ', + 'x': 'k', + 'ɬ': 'l', + } + for old, new in replacements.items(): + phonemes = phonemes.replace(old, new) + + # Apply regex-based replacements + phonemes = re.sub(r'(?<=[a-zɹː])(?=hˈʌndɹɪd)', ' ', phonemes) + phonemes = re.sub(r' z(?=[;:,.!?¡¿—…"«»“” ]|$)', 'z', phonemes) + + # Additional language-specific rules + if lang == 'a': + phonemes = re.sub(r'(?<=nˈaɪn)ti(?!ː)', 'di', phonemes) + + # Filter out characters not in VOCAB + phonemes = ''.join(filter(lambda p: p in self.VOCAB, phonemes)) + + return phonemes.strip() diff --git a/services/ktts/requirements.txt b/services/ktts/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d0e161253bb979f4670439d364340ee6b4344c6 --- /dev/null +++ b/services/ktts/requirements.txt @@ -0,0 +1,9 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +torch +torchvision +gradio +phonemizer +soundfile +onnxruntime +fastapi +uvicorn diff --git a/services/ktts/voices/af.pt b/services/ktts/voices/af.pt new file mode 100644 index 0000000000000000000000000000000000000000..a67cad519413efaf099c768ed3ba6ed7bac6bfb4 --- /dev/null +++ b/services/ktts/voices/af.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d6457d8d31306c7c4e7afc5040b446e6cb556b40fd80d07fbdbc8dc75d300c +size 131 diff --git a/services/ktts/voices/af_bella.pt b/services/ktts/voices/af_bella.pt new file mode 100644 index 0000000000000000000000000000000000000000..9221cbb248feafe5b575b3a4c2297f7da9f10a23 --- /dev/null +++ b/services/ktts/voices/af_bella.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04339167efe7fe6dedf2eedfcdd357bbe3f4451619fcb67c0feea0708da0bb31 +size 131 diff --git a/services/ktts/voices/af_nicole.pt b/services/ktts/voices/af_nicole.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a2cc59923702171df2e4fe7720930b7e08fe93b --- /dev/null +++ b/services/ktts/voices/af_nicole.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca02cd41d325f5445b14805a3ec6985cae55a21cec91635f8c2038a5b288383 +size 131 diff --git a/services/ktts/voices/af_sarah.pt b/services/ktts/voices/af_sarah.pt new file mode 100644 index 0000000000000000000000000000000000000000..01586c4caa5ca7db52b54c0cb721452d7d32f66e --- /dev/null +++ b/services/ktts/voices/af_sarah.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a6f3c43e3645896c7f0d57968f23d5ef5860b764bd4abb6d75076a1e7fda2e9 +size 131 diff --git a/services/ktts/voices/af_sky.pt b/services/ktts/voices/af_sky.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c39e9bcd1590004f8761adc0bb103e0df338051 --- /dev/null +++ b/services/ktts/voices/af_sky.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab87dbbf5b86022f6c01aed43cb4bcd221fcb95241c1d1e67e27dde2978a59c5 +size 131 diff --git a/services/ktts/voices/am_adam.pt b/services/ktts/voices/am_adam.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3300d7399123b5bc5f16e44bbba64840a6ba4cf --- /dev/null +++ b/services/ktts/voices/am_adam.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd49625a3f7fba1b2ca0cd84b817bc0531fdc94bd256953c3636e8afb09df2f4 +size 131 diff --git a/services/ktts/voices/am_michael.pt b/services/ktts/voices/am_michael.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf80006e6e345586272f5881908a5f2972e4e664 --- /dev/null +++ b/services/ktts/voices/am_michael.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba4bc7701cefde36070eaf65d20a689dae85282efc79c8d5770933a56c2ff76 +size 131 diff --git a/services/ktts/voices/bf_emma.pt b/services/ktts/voices/bf_emma.pt new file mode 100644 index 0000000000000000000000000000000000000000..3379731d55b1234c1f4594b8065ac398894d7775 --- /dev/null +++ b/services/ktts/voices/bf_emma.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c84df49c195eb467a7a2e3f5f5e25624808f751db021fd5776495e1b9bb585 +size 131 diff --git a/services/ktts/voices/bf_isabella.pt b/services/ktts/voices/bf_isabella.pt new file mode 100644 index 0000000000000000000000000000000000000000..861e4bbf104fdee879c39bdd952d3ba24ccc6617 --- /dev/null +++ b/services/ktts/voices/bf_isabella.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc23980551bb88178169bbdcfe3f10d69fc8b8ded9fdb0ec1123a3294b95266d +size 131 diff --git a/services/ktts/voices/bm_george.pt b/services/ktts/voices/bm_george.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b2bc7a75b3ddbb8cd7e11aec17f6358dbee8239 --- /dev/null +++ b/services/ktts/voices/bm_george.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27dff07c3aaf98f311475d2d3212f377325ab49b0ff566100424a3301252bbb2 +size 131 diff --git a/services/ktts/voices/bm_lewis.pt b/services/ktts/voices/bm_lewis.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2e59aa976fd20aae5bcee04ea0081198a849599 --- /dev/null +++ b/services/ktts/voices/bm_lewis.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d076bfb89ada363657fea4d40d67f4174929dc771fdc245547397714fd33dc +size 131 diff --git a/services/ktts/weights/.gitkeep b/services/ktts/weights/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/services/ktts/weights/kokoro-quant.onnx b/services/ktts/weights/kokoro-quant.onnx new file mode 100644 index 0000000000000000000000000000000000000000..a34a4bec1c44b37756b2ef7622da8ce8ddc53380 --- /dev/null +++ b/services/ktts/weights/kokoro-quant.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fa3c2d4e9dabfb2929b555c425c1c56f8333d46c47de8668de365a3d158de83 +size 134 diff --git a/services/ktts/weights/kokoro-v0_19.onnx b/services/ktts/weights/kokoro-v0_19.onnx new file mode 100644 index 0000000000000000000000000000000000000000..f17da6d7150a4560e9b0b8e1cff29947448908dc --- /dev/null +++ b/services/ktts/weights/kokoro-v0_19.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bd36e46e7e6e1f118d2ae1a8ff807a24c06cdae309f3c4f34230ad9eb37652 +size 134 diff --git a/services/musicgen/.gitattributes b/services/musicgen/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/services/musicgen/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/services/musicgen/Dockerfile b/services/musicgen/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0be2d3b0afab7cb2be548088b792399abbf75a6d --- /dev/null +++ b/services/musicgen/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.10-slim + +WORKDIR /app + +# System deps +RUN apt-get update && apt-get install -y \ + git \ + ffmpeg \ + libsndfile1 \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --upgrade pip +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 7860 + +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/services/musicgen/README.md b/services/musicgen/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b201d6550bd0c936ebf932f0d85d01a2a5e6b8bd --- /dev/null +++ b/services/musicgen/README.md @@ -0,0 +1,10 @@ +--- +title: MusicGen +emoji: 📚 +colorFrom: indigo +colorTo: blue +sdk: docker +pinned: false +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/services/musicgen/app.py b/services/musicgen/app.py new file mode 100644 index 0000000000000000000000000000000000000000..627ff1fa225610d03063b3c98dbef8b992010b0d --- /dev/null +++ b/services/musicgen/app.py @@ -0,0 +1,74 @@ +from fastapi import FastAPI +from fastapi.responses import FileResponse +import gradio as gr +from generate import generate_music + +# ====================== +# FASTAPI BACKEND +# ====================== + +api = FastAPI(title="AI Background Music Generator") + +@api.get("/health") +def health(): + return {"status": "running"} + +@api.post("/generate") +async def api_generate(prompt: str, duration: int = 10): + file_path = generate_music(prompt, duration) + + return FileResponse( + path=file_path, + media_type="audio/wav", + filename="music.wav" + ) + +# ====================== +# GRADIO UI +# ====================== + +def ui_generate(prompt, duration): + audio = generate_music(prompt, duration) + return audio + + +with gr.Blocks( + title="AI Background Music Generator", + theme=gr.themes.Soft() +) as demo: + + gr.Markdown(""" +# 🎵 AI Background Music Generator +Generate royalty-free background music instantly. +""") + + prompt = gr.Textbox( + label="Music Description", + placeholder="Soft cinematic nasheed style..." + ) + + duration = gr.Slider( + minimum=5, + maximum=60, + value=10, + step=1, + label="Duration (seconds)" + ) + + btn = gr.Button("Generate") + + output = gr.Audio(type="filepath") + + btn.click( + fn=ui_generate, + inputs=[prompt, duration], + outputs=output + ) + + +# ====================== +# IMPORTANT FIX +# ====================== +# Mount GRADIO as ROOT APP + +app = gr.mount_gradio_app(api, demo, path="/") \ No newline at end of file diff --git a/services/musicgen/generate.py b/services/musicgen/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..2d88d3914d172240e49c69908c623bc4defbe571 --- /dev/null +++ b/services/musicgen/generate.py @@ -0,0 +1,58 @@ +import torch +from transformers import AutoProcessor, MusicgenForConditionalGeneration +import soundfile as sf +import uuid + +MODEL_NAME = "facebook/musicgen-small" + +processor = None +model = None +device = "cuda" if torch.cuda.is_available() else "cpu" +model_status = "not_loaded" +model_error = None + + +def _get_model(): + global processor, model, model_status, model_error + + if processor is None or model is None: + try: + model_status = "loading" + model_error = None + print("Loading MusicGen model...") + processor = AutoProcessor.from_pretrained(MODEL_NAME) + model = MusicgenForConditionalGeneration.from_pretrained(MODEL_NAME) + model.to(device) + model_status = "ready" + except Exception as exc: + model_status = "failed" + model_error = str(exc) + raise + + return processor, model + +def generate_music(prompt, duration): + processor, model = _get_model() + + inputs = processor( + text=[prompt], + padding=True, + return_tensors="pt" + ).to(device) + + audio_values = model.generate( + **inputs, + max_new_tokens=int(duration * 50) + ) + + filename = f"/tmp/{uuid.uuid4()}.wav" + + sampling_rate = model.config.audio_encoder.sampling_rate + + sf.write( + filename, + audio_values[0, 0].cpu().numpy(), + sampling_rate + ) + + return filename diff --git a/services/musicgen/requirements.txt b/services/musicgen/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a53c6ac9509ff5fb941d5472a2cec2d0837ee26b --- /dev/null +++ b/services/musicgen/requirements.txt @@ -0,0 +1,10 @@ +fastapi +uvicorn +gradio +torch +torchaudio +transformers +accelerate +soundfile +scipy +numpy \ No newline at end of file diff --git a/services/musicgen/storage.py b/services/musicgen/storage.py new file mode 100644 index 0000000000000000000000000000000000000000..5104f17f888768c43532efef62dcd7cfd3fffd98 --- /dev/null +++ b/services/musicgen/storage.py @@ -0,0 +1,24 @@ +import os +from huggingface_hub import HfApi + +HF_TOKEN = os.getenv("HF_TOKEN") +REPO_ID = "basyx/music-storage" + +api = HfApi() + +def upload_audio(filepath): + + if not HF_TOKEN: + return None + + filename = os.path.basename(filepath) + + api.upload_file( + path_or_fileobj=filepath, + path_in_repo=filename, + repo_id=REPO_ID, + repo_type="dataset", + token=HF_TOKEN + ) + + return f"https://huggingface.co/datasets/{REPO_ID}/resolve/main/{filename}" \ No newline at end of file diff --git a/services/musicgen/styles.py b/services/musicgen/styles.py new file mode 100644 index 0000000000000000000000000000000000000000..83faf10a3f13cbc8d83fe723f20af7c539dac7da --- /dev/null +++ b/services/musicgen/styles.py @@ -0,0 +1,21 @@ +STYLES = { + + "nasheed": """ +Islamic vocal style nasheed, +spiritual atmosphere, +no musical instruments, +warm emotional tone +""", + + "cinematic": """ +epic cinematic orchestral background, +film score mood, +emotional and inspirational +""", + + "lofi": """ +lofi chill background music, +soft ambience, +relaxed study atmosphere +""" +} \ No newline at end of file diff --git a/services/render_engine/.gitattributes b/services/render_engine/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/services/render_engine/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/services/render_engine/.gitignore b/services/render_engine/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b7f0769db61b15da07285ca862fc590149bf3856 --- /dev/null +++ b/services/render_engine/.gitignore @@ -0,0 +1,29 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ + +temp/ +exports/ +jobs/ +models/ +storage/ + +.env +.env.* +!.env.example + +node_modules/ +.next/ +out/ +dist/ +coverage/ +*.tsbuildinfo + +frontend/node_modules/ +frontend/.next/ +frontend/out/ +frontend/.vercel/ +frontend/.env +frontend/.env.* +!frontend/.env.example + diff --git a/services/render_engine/Dockerfile b/services/render_engine/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7e5265c078a7c2eac8ae9687429bb40a70c1cc81 --- /dev/null +++ b/services/render_engine/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + TEMP_DIR=/app/temp \ + EXPORTS_DIR=/app/exports \ + JOBS_DIR=/app/jobs \ + STORAGE_DIR=/app/storage \ + AVA2LON_BASE_DIR=/app \ + BASYX_BASE_DIR=/app \ + MAX_RENDER_WORKERS=1 \ + FFMPEG_TIMEOUT_SECONDS=900 \ + DOWNLOAD_TIMEOUT_SECONDS=60 \ + MAX_DOWNLOAD_BYTES=524288000 \ + ALLOW_PRIVATE_ASSET_URLS=false \ + WHISPER_MODEL_SIZE=tiny \ + WHISPER_COMPUTE_TYPE=int8 \ + WHISPER_DEVICE=cpu \ + WHISPER_MODEL_DIR=/app/models \ + MAX_RETRIES=3 \ + JOB_RETENTION_SECONDS=86400 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + fonts-dejavu-core \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p /app/temp /app/exports /app/jobs /app/models /app/storage \ + && chmod -R 777 /app/temp /app/exports /app/jobs /app/models /app/storage + +EXPOSE 7860 + +CMD ["python", "app.py"] diff --git a/services/render_engine/README.md b/services/render_engine/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4d0026c53b6b12895e71fbdfada22790f0e3c664 --- /dev/null +++ b/services/render_engine/README.md @@ -0,0 +1,365 @@ +--- +title: Basyx FFmpeg Rendering Engine +emoji: 🎬 +colorFrom: green +colorTo: yellow +sdk: docker +app_port: 7860 +pinned: false +license: mit +--- + +# Basyx FFmpeg Rendering Engine + +Basyx FFmpeg is a CPU-first rendering backend for short-form video automation. It is designed for Hugging Face Docker Spaces running on 2-8 CPU cores with 8-16 GB RAM and no GPU. The engine uses FFmpeg and FFprobe subprocess pipelines and never loads complete videos into Python memory. + +## Architecture + +```mermaid +flowchart TD + API["FastAPI REST API"] --> Jobs["Job Manager"] + Dashboard["Gradio Operator Dashboard"] --> Jobs + Jobs --> Engine["Render Engine"] + Engine --> Assets["Asset Probe + Metadata Cache"] + Engine --> Ingest["Remote URL + Upload Ingestion"] + Engine --> Normalize["Normalization"] + Engine --> Scenes["Scene Timeline"] + Engine --> Subtitles["SRT / ASS Subtitle Engine"] + Engine --> Transitions["Transition Filter Builder"] + Engine --> Audio["Voiceover + Ducking Mixer"] + Engine --> Exports["Export Manager"] + Ingest --> FFmpeg["FFmpeg / FFprobe"] + Normalize --> FFmpeg + Scenes --> FFmpeg + Subtitles --> FFmpeg + Transitions --> FFmpeg + Audio --> FFmpeg +``` + +Package layout: + +```text +renderer/ + core/ settings, models, ingestion, orchestration + ffmpeg/ command builder, runner, probing, normalization + subtitles/ SRT and ASS generation + transitions/ xfade graph generation + audio/ voiceover and music ducking + scenes/ timeline validation + templates/ caption templates + exports/ final deliverables + jobs/ durable job records and retries +``` + +## REST API + +All API endpoints are served at the Space root. Asset fields may be absolute container paths or public `http`/`https` URLs. Remote assets are downloaded into a job-local temp directory with timeout and size limits before rendering. + +### `POST /render` + +Submit one render job. + +```json +{ + "template": "tiktok_classic", + "output_name": "campaign_clip.mp4", + "voiceover": "https://cdn.example.com/voiceover.wav", + "background_music": "https://cdn.example.com/music.mp3", + "subtitle_format": "ass", + "auto_subtitles": true, + "subtitle_language": "en", + "whisper_model_size": "tiny", + "preset": "tiktok_9_16_fast", + "callback_url": "https://n8n.example.com/webhook/render-complete", + "export_target": "local", + "audio_normalize": true, + "preview": false, + "normalize": true, + "scenes": [ + { + "start": 0, + "duration": 5, + "media": "https://cdn.example.com/scene1.mp4", + "caption": "Launch faster with automated rendering", + "transition": "fade" + } + ] +} +``` + +Response: + +```json +{ + "job_id": "job_abc123", + "status_url": "/status/job_abc123", + "download_url": "/download/job_abc123?token=..." +} +``` + +### `GET /presets` + +Returns available render presets and caption templates. Presets let n8n submit compact requests such as `tiktok_9_16_fast`, `youtube_shorts_hd`, `podcast_square`, `reels_with_subtitles`, and `draft_preview`. + +### `POST /render/ai-reels` + +Submit a single-call AI Reels job using an existing voiceover and asset list. TTS is intentionally provider-pluggable in v1; the production path requires a supplied voiceover. + +```json +{ + "script": "Launch faster with automated rendering.", + "voiceover": "https://cdn.example.com/voiceover.wav", + "assets": ["https://cdn.example.com/scene1.jpg", "https://cdn.example.com/scene2.mp4"], + "template": "youtube_shorts", + "output_name": "ai_reel.mp4" +} +``` + +### `POST /render/batch` + +Submit multiple render jobs. The worker pool defaults to one active render to avoid RAM exhaustion. + +```json +{ + "jobs": [ + { + "template": "modern_minimal", + "output_name": "clip_a.mp4", + "scenes": [{"start": 0, "duration": 3, "media": "https://cdn.example.com/a.mp4"}] + } + ] +} +``` + +### `POST /scene-builder` + +Build a timeline from a script and asset list. + +```json +{ + "script": "One script can be split across several short scenes.", + "assets": ["https://cdn.example.com/a.mp4", "https://cdn.example.com/b.jpg"], + "duration": 8, + "transition": "fade" +} +``` + +### `POST /assets/upload` + +Upload files from automation tools such as n8n and receive staged paths that can be used in `/render`. + +Multipart form field: + +- `files`: one or more images, videos, GIFs, or audio files. + +Response: + +```json +{ + "assets": [ + { + "filename": "clip.mp4", + "path": "/app/temp/uploads/abc/clip.mp4", + "reference": "upload://clip.mp4" + } + ] +} +``` + +### `POST /render/upload` + +Submit a render job and files in one multipart request. + +Multipart fields: + +- `request_json`: render payload JSON. +- `files`: uploaded files referenced by `upload://filename`. + +Example `request_json`: + +```json +{ + "template": "tiktok_classic", + "output_name": "n8n_clip.mp4", + "scenes": [ + { + "start": 0, + "duration": 5, + "media": "upload://clip.mp4", + "caption": "Rendered from n8n" + } + ], + "voiceover": "upload://voice.wav" +} +``` + +### `POST /render/ai-reels/upload` + +Multipart AI Reels endpoint. Use `upload://filename` inside `voiceover`, `background_music`, and `assets`. + +### `POST /transcribe` + +Transcribe local or remote audio/video with faster-whisper. + +```json +{ + "audio": "https://cdn.example.com/voiceover.wav", + "model_size": "tiny", + "language": "en", + "task": "transcribe", + "word_timestamps": true, + "vad_filter": true +} +``` + +Response includes `text`, detected `language`, segment timings, and optional word timings. Use this endpoint from n8n when you want subtitles before rendering. + +### `POST /transcribe/upload` + +Multipart faster-whisper endpoint for direct n8n file uploads. + +Multipart fields: + +- `file`: audio or video file. +- `model_size`: optional Whisper model, default `WHISPER_MODEL_SIZE`. +- `language`: optional ISO language code. +- `word_timestamps`: optional boolean, default `true`. + +### `POST /subtitles` + +Generate SRT or ASS files from timed events. + +```json +{ + "format": "ass", + "template": "tiktok_zoom", + "events": [ + {"start": 0, "end": 1.2, "text": "Hello world"} + ] +} +``` + +### `GET /status/{job_id}` + +Returns job state, logs, FFmpeg commands, metrics, output path, and failure reason. + +States: `PENDING`, `RUNNING`, `FAILED`, `COMPLETED`. + +### `GET /download/{job_id}` + +Returns the completed MP4 deliverable. Downloads require the signed token returned by render submission. If the job is still running, the endpoint returns `409`. + +### `POST /cancel/{job_id}` + +Requests job cancellation. Pending jobs are marked `CANCELLED`; running jobs are marked `CANCEL_REQUESTED` and stop before the next guarded execution point. + +### `POST /admin/cleanup` + +Deletes expired job records, old exports, and staged upload folders. Default retention is controlled by `JOB_RETENTION_SECONDS`. + +### `POST /inspect` + +Probe one local asset path and return MIME type, duration, codecs, bitrate, resolution, FPS, stream list, and cache metadata. + +## n8n Workflow Integration + +Recommended URL-based flow: + +1. Use an HTTP Request node with `POST https://YOUR-SPACE.hf.space/render`. +2. Pass public file URLs from S3, Supabase Storage, Cloudinary, Google Drive direct-download links, or another CDN in `scenes[].media`, `voiceover`, and `background_music`. +3. Set `auto_subtitles` to `true` when you want faster-whisper captions generated from `voiceover`. +4. Add `callback_url` to receive a POST when the job completes or fails. +5. Poll `GET https://YOUR-SPACE.hf.space/status/{{$json.job_id}}` only if you do not use callbacks. +6. Download the final MP4 from the signed `download_url`. + +Multipart flow: + +1. Use an HTTP Request node set to multipart form-data. +2. Send `request_json` as a text field. +3. Attach files under the `files` field. +4. Reference those files in JSON as `upload://exact-filename.ext`. + +Security defaults: + +- Only `http` and `https` remote assets are accepted. +- Private, localhost, link-local, and multicast hosts are blocked by default. +- Set `ALLOW_PRIVATE_ASSET_URLS=true` only for trusted self-hosted deployments. +- `MAX_DOWNLOAD_BYTES` limits remote downloads and multipart upload totals. +- `WHISPER_MODEL_SIZE=tiny` and `WHISPER_COMPUTE_TYPE=int8` are the recommended CPU defaults. +- `BASYX_SIGNING_SECRET` should be set as a Space secret before public deployment. + +## Operator Dashboard + +The dashboard is available at `/dashboard`. It provides production operator tools for render submission, batch submission, AI Reels submission, job status, logs, downloads, and asset inspection. + +## Caption Templates + +Built-in templates: + +- `tiktok_classic` +- `tiktok_zoom` +- `alex_hormozi` +- `modern_minimal` +- `youtube_shorts` +- `podcast_style` +- `news_style` + +Templates are selected per request and do not require code changes. + +## CPU And RAM Tuning + +- Keep `MAX_RENDER_WORKERS=1` for 8 GB RAM deployments. +- Use `OUTPUT_PRESET=veryfast` or `ultrafast` for faster CPU rendering. +- Use `OUTPUT_CRF=23-28` to balance quality and file size. +- Keep source assets near the target duration to reduce normalization work. +- Prefer pre-trimmed voiceovers and assets for batch workloads. +- Start with `WHISPER_MODEL_SIZE=tiny` or `base` on free/CPU Spaces. +- Use `preview=true` for low-resolution draft renders. +- Use `audio_normalize=true` for voiceover or podcast-style content. + +## Reliability + +- FFmpeg subprocesses are killed after `FFMPEG_TIMEOUT_SECONDS`. +- Remote asset downloads are killed after `DOWNLOAD_TIMEOUT_SECONDS`. +- Remote and multipart input size is capped by `MAX_DOWNLOAD_BYTES`. +- faster-whisper models are loaded lazily and cached under `WHISPER_MODEL_DIR`. +- Webhook callbacks are best-effort and callback failures are written to job logs. +- `export_target=local` copies completed renders to `STORAGE_DIR`. +- `export_target=https://...` uploads the completed MP4 with HTTP `PUT`, which works with presigned URLs from S3, Supabase Storage, and similar providers. +- Jobs retry up to `MAX_RETRIES`. +- Intermediate files are created under `TEMP_DIR` and removed after export. +- Job records retain command history, logs, render time, output size, and failure reasons. + +## Docker Deployment + +Build locally: + +```bash +docker build -t basyx-ffmpeg . +docker run --rm -p 7860:7860 basyx-ffmpeg +``` + +Open: + +- API: `http://localhost:7860/health` +- Dashboard: `http://localhost:7860/dashboard` + +## Hugging Face Spaces Deployment + +Create a Docker Space, push this repository, and keep the README metadata above. The container listens on port `7860`, and the Space exposes the FastAPI service plus the dashboard. + +## Testing + +Run: + +```bash +pytest --cov=renderer --cov-report=term-missing +``` + +Integration tests use mocked FFmpeg/FFprobe where possible and small generated media where necessary. + +## Known v1 Limits + +- Bundled TTS is not included. The AI Reels endpoint requires a supplied voiceover and keeps narration generation behind a provider interface for future integration. +- Advanced caption animation is implemented through ASS effects and FFmpeg-compatible filter behavior, not GPU animation layers. +- Batch rendering is intentionally sequential by default for constrained CPU/RAM Spaces. diff --git a/services/render_engine/api.py b/services/render_engine/api.py new file mode 100644 index 0000000000000000000000000000000000000000..384bab14b412656288e5c8c485321b663ecce54c --- /dev/null +++ b/services/render_engine/api.py @@ -0,0 +1,1314 @@ +from __future__ import annotations + +import json +import shutil +import uuid +import zipfile +from pathlib import Path +from typing import Any + +from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field + +from renderer.core.config import Settings +from renderer.core.ingest import stage_upload +from renderer.core.models import AIReelsRequest, RenderRequest, Scene +from renderer.core.security import verify_download_token +from renderer.core.utils import safe_filename +from renderer.jobs import JobManager +from renderer.platform import PlatformProcessor, supported_toolkit_tasks +from renderer.scenes import Timeline +from renderer.studio import ( + ProjectStore, + StudioTaskProcessor, + add_effect, + add_filter, + add_keyframe, + add_transition, + capability_catalog, +) +from renderer import RenderEngine +from renderer.subtitles import SubtitleEvent, SubtitleGenerator +from renderer.templates import ( + apply_creative_style, + apply_preset, + creative_style_metadata, + get_creative_style, + list_creative_styles, + list_platform_profiles, + list_presets, + list_scene_effects, + list_templates, + platform_profile_metadata, + scene_effect_metadata, +) +from renderer.transitions import TransitionBuilder + +settings = Settings() +settings.ensure_dirs() +job_manager = JobManager(settings) +project_manager = ProjectStore(settings) +api = FastAPI(title="Ava2lon Studio AI", version="2.0.0") + +API_KEY_HEADER = "X-API-Key" + + +def _verify_api_key(api_key: str | None = Header(default=None, alias=API_KEY_HEADER)) -> None: + configured = getattr(settings, "api_key", "") + if configured and api_key != configured: + raise HTTPException(status_code=401, detail="Invalid or missing API key") + +def _submission_response(job_id: str) -> dict[str, str]: + download_url = f"/download/{job_id}" + try: + record = job_manager.get(job_id) + except KeyError: + token = None + else: + token = record.download_token + if token: + download_url = f"{download_url}?token={token}" + return {"job_id": job_id, "status_url": f"/status/{job_id}", "download_url": download_url} + + +class ScenePayload(BaseModel): + start: float = Field(ge=0) + duration: float = Field(gt=0) + media: str + caption: str = "" + transition: str = "fade" + background: str = "blur" + layout: str = "fill" + effect: str | None = None + + +class RenderPayload(BaseModel): + scenes: list[ScenePayload] + template: str = "tiktok_classic" + preset: str | None = None + creative_style: str | None = None + platform: str | None = None + output_name: str = "render.mp4" + voiceover: str | None = None + background_music: str | None = None + music_volume: float = Field(default=0.316, ge=0, le=2) + music_fade_in: float = Field(default=0.0, ge=0) + music_fade_out: float = Field(default=0.0, ge=0) + music_loop: bool = True + music_start: float = Field(default=0.0, ge=0) + music_ducking: bool = True + voice_volume: float = Field(default=1.0, ge=0, le=2) + subtitle_format: str = "ass" + auto_subtitles: bool = False + subtitle_language: str | None = None + whisper_model_size: str | None = None + preview: bool = False + audio_normalize: bool = False + watermark: str | None = None + watermark_position: str = "bottom-right" + intro: str | None = None + outro: str | None = None + callback_url: str | None = None + export_target: str | None = None + priority: int = 0 + scheduled_at: float | None = None + normalize: bool = True + metadata: dict[str, Any] = Field(default_factory=dict) + + +class AIReelsPayload(BaseModel): + script: str + voiceover: str + assets: list[str] + template: str = "tiktok_classic" + creative_style: str | None = None + platform: str | None = None + output_name: str = "ai_reel.mp4" + background_music: str | None = None + music_volume: float = Field(default=0.316, ge=0, le=2) + music_fade_in: float = Field(default=0.0, ge=0) + music_fade_out: float = Field(default=0.0, ge=0) + music_loop: bool = True + music_start: float = Field(default=0.0, ge=0) + music_ducking: bool = True + voice_volume: float = Field(default=1.0, ge=0, le=2) + + +class BatchPayload(BaseModel): + jobs: list[RenderPayload] + + +class UploadedAsset(BaseModel): + filename: str + path: str + reference: str + kind: str = "other" + metadata: dict[str, Any] | None = None + + +class TranscribePayload(BaseModel): + audio: str + model_size: str | None = None + language: str | None = None + task: str = "transcribe" + beam_size: int = Field(default=5, ge=1, le=10) + vad_filter: bool = True + word_timestamps: bool = True + + +class SubtitlePayload(BaseModel): + events: list[dict[str, Any]] + format: str = "srt" + template: str = "tiktok_classic" + + +class SceneBuildPayload(BaseModel): + script: str + assets: list[str] + duration: float | None = None + transition: str = "fade" + creative_style: str | None = None + + +class IngestSourcePayload(BaseModel): + url: str + type: str | None = None + name: str | None = None + + +class IngestPayload(BaseModel): + sources: list[IngestSourcePayload] + callback_url: str | None = None + + +class AnalyzePayload(BaseModel): + media: str + transcript: str = "" + platform: str | None = None + callback_url: str | None = None + + +class ClipsPayload(BaseModel): + media: str + clips: list[dict[str, Any]] | None = None + callback_url: str | None = None + + +class ToolkitPayload(BaseModel): + task: str + input: str | None = None + media: str | None = None + output_name: str | None = None + params: dict[str, Any] = Field(default_factory=dict) + callback_url: str | None = None + export_target: str | None = None + model_config = {"extra": "allow"} + + +class ThumbnailPayload(BaseModel): + media: str + text: str = "" + timestamp: float | None = None + template: str = "bold" + callback_url: str | None = None + + +class MetadataPayload(BaseModel): + topic: str = "" + transcript: str = "" + platform: str | None = None + callback_url: str | None = None + + +class PublishPayload(BaseModel): + media: str | None = None + asset: str | None = None + title: str | None = None + description: str | None = None + platforms: list[str] = Field(default_factory=list) + platform: str | None = None + scheduled_at: str | None = None + draft: bool = True + callback_url: str | None = None + + +class ProjectPayload(BaseModel): + name: str + metadata: dict[str, Any] = Field(default_factory=dict) + export_settings: dict[str, Any] = Field(default_factory=dict) + template: dict[str, Any] | None = None + + +class ProjectSavePayload(BaseModel): + project_id: str | None = None + project: dict[str, Any] + + +class ProjectAssetPayload(BaseModel): + project_id: str + asset: dict[str, Any] + + +class TimelineAddPayload(BaseModel): + project_id: str + track_type: str = "video" + track_id: str | None = None + item: dict[str, Any] = Field(default_factory=dict) + + +class TimelineOperationPayload(BaseModel): + project_id: str + operation: str = "drag" + item_id: str | None = None + params: dict[str, Any] = Field(default_factory=dict) + + +class EffectApplyPayload(BaseModel): + project_id: str | None = None + target_id: str | None = None + item_id: str | None = None + effect: str + params: dict[str, Any] = Field(default_factory=dict) + callback_url: str | None = None + + +class FilterApplyPayload(BaseModel): + project_id: str | None = None + target_id: str | None = None + item_id: str | None = None + filter: str + params: dict[str, Any] = Field(default_factory=dict) + lut: str | None = None + callback_url: str | None = None + + +class TransitionAddPayload(BaseModel): + project_id: str | None = None + from_item_id: str | None = None + to_item_id: str | None = None + target_id: str | None = None + transition: str + duration: float = Field(default=0.45, gt=0) + params: dict[str, Any] = Field(default_factory=dict) + callback_url: str | None = None + + +class KeyframePayload(BaseModel): + project_id: str + target_id: str + property: str + time: float = Field(ge=0) + value: Any + easing: str = "linear" + + +class GenerationPayload(BaseModel): + prompt: str | None = None + text: str | None = None + media: str | None = None + provider: str | None = None + callback_url: str | None = None + export_target: str | None = None + params: dict[str, Any] = Field(default_factory=dict) + model_config = {"extra": "allow"} + + +class CaptionGeneratePayload(BaseModel): + media: str | None = None + audio: str | None = None + text: str | None = None + transcript: str | None = None + events: list[dict[str, Any]] | None = None + template: str = "capcut" + language: str | None = None + engine: str = "whisper" + word_timestamps: bool = True + emoji_insertion: bool = False + speaker_detection: bool = False + karaoke: bool = True + animated: bool = True + callback_url: str | None = None + model_config = {"extra": "allow"} + + +class AIToolPayload(BaseModel): + project_id: str | None = None + media: str | None = None + transcript: str | None = None + text: str | None = None + platform: str | None = None + callback_url: str | None = None + params: dict[str, Any] = Field(default_factory=dict) + model_config = {"extra": "allow"} + + +class ProjectRenderPayload(BaseModel): + output_name: str = "project_render.mp4" + template: str | None = None + preset: str | None = None + creative_style: str | None = None + platform: str | None = None + callback_url: str | None = None + export_target: str | None = None + preview: bool = False + normalize: bool = True + metadata: dict[str, Any] = Field(default_factory=dict) + + +@api.get("/health") +def health() -> dict[str, str]: + return {"status": "ok"} + + +@api.get("/monitor") +def monitor() -> dict[str, Any]: + return _monitor_payload() + + +@api.get("/queue") +def queue() -> dict[str, Any]: + return job_manager.summary() + + +@api.get("/workers") +def workers() -> dict[str, Any]: + return { + "max_workers": settings.max_workers, + "ffmpeg_timeout_seconds": settings.ffmpeg_timeout_seconds, + "whisper_device": settings.whisper_device, + "whisper_model_size": settings.whisper_model_size, + "toolkit_tasks": supported_toolkit_tasks(), + } + + +@api.get("/presets") +def presets() -> dict[str, Any]: + return { + "presets": list_presets(), + "caption_templates": list_templates(), + "platforms": list_platform_profiles(), + "creative_styles": list_creative_styles(), + "scene_effects": list_scene_effects(), + "transitions": TransitionBuilder().list_transitions(), + "creative_style_metadata": creative_style_metadata(), + "scene_effect_metadata": scene_effect_metadata(), + "toolkit_tasks": supported_toolkit_tasks(), + } + + +@api.get("/platforms") +def platforms() -> dict[str, dict[str, Any]]: + return {"platforms": platform_profile_metadata()} + + +@api.get("/toolkit/tasks") +def toolkit_tasks() -> dict[str, list[str]]: + return {"tasks": supported_toolkit_tasks()} + + +@api.get("/capabilities") +def capabilities() -> dict[str, Any]: + catalog = capability_catalog() + catalog["runtime"] = { + "toolkit_tasks": supported_toolkit_tasks(), + "caption_templates": list_templates(), + "creative_styles": list_creative_styles(), + "scene_effects": list_scene_effects(), + "platforms": list_platform_profiles(), + "transitions": TransitionBuilder().list_transitions(), + } + return catalog + + +@api.get("/effects") +def effects() -> dict[str, Any]: + catalog = capability_catalog() + return {"effects": catalog["effects"], "scene_effects": scene_effect_metadata()} + + +@api.get("/filters") +def filters() -> dict[str, Any]: + return {"filters": capability_catalog()["filters"]} + + +@api.get("/transitions") +def transitions() -> dict[str, Any]: + return {"families": capability_catalog()["transitions"], "ffmpeg": TransitionBuilder().list_transitions()} + + +@api.get("/templates/catalog") +def templates_catalog() -> dict[str, Any]: + return { + "categories": capability_catalog()["templates"], + "caption_templates": list_templates(), + "render_presets": list_presets(), + "creative_styles": creative_style_metadata(), + } + + +@api.get("/projects") +def list_projects() -> dict[str, list[dict[str, Any]]]: + return {"projects": project_manager.list()} + + +@api.post("/projects", dependencies=[Depends(_verify_api_key)]) +def create_project(payload: ProjectPayload) -> dict[str, Any]: + project = project_manager.create(payload.name, metadata=payload.metadata, template=payload.template) + if payload.export_settings: + project["export_settings"].update(payload.export_settings) + project = project_manager.save(project["id"], project) + return {"project": project} + + +@api.post("/project/create", dependencies=[Depends(_verify_api_key)]) +def project_create(payload: ProjectPayload) -> dict[str, Any]: + return create_project(payload) + + +@api.get("/project/{project_id}") +def project_get(project_id: str) -> dict[str, Any]: + try: + return {"project": project_manager.get(project_id)} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project not found") from exc + + +@api.post("/project/save", dependencies=[Depends(_verify_api_key)]) +def project_save(payload: ProjectSavePayload) -> dict[str, Any]: + project_id = payload.project_id or str(payload.project.get("id") or safe_filename(payload.project.get("name", "project"))) + return {"project": project_manager.save(project_id, payload.project)} + + +@api.post("/project/assets/add", dependencies=[Depends(_verify_api_key)]) +def project_asset_add(payload: ProjectAssetPayload) -> dict[str, Any]: + try: + return {"project": project_manager.add_asset(payload.project_id, payload.asset)} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project not found") from exc + + +@api.post("/timeline/add", dependencies=[Depends(_verify_api_key)]) +def timeline_add(payload: TimelineAddPayload) -> dict[str, Any]: + try: + project = project_manager.add_to_timeline(payload.project_id, payload.item, payload.track_type, payload.track_id) + return {"project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project not found") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.post("/timeline/operation", dependencies=[Depends(_verify_api_key)]) +def timeline_operation(payload: TimelineOperationPayload) -> dict[str, Any]: + try: + project = project_manager.timeline_operation(payload.project_id, payload.operation, payload.item_id, payload.params) + return {"project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Timeline item or project not found") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.post("/timeline/split", dependencies=[Depends(_verify_api_key)]) +def timeline_split(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "split" + return timeline_operation(payload) + + +@api.post("/timeline/trim", dependencies=[Depends(_verify_api_key)]) +def timeline_trim(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "trim" + return timeline_operation(payload) + + +@api.post("/timeline/ripple-delete", dependencies=[Depends(_verify_api_key)]) +def timeline_ripple_delete(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "ripple_delete" + return timeline_operation(payload) + + +@api.post("/timeline/insert", dependencies=[Depends(_verify_api_key)]) +def timeline_insert(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "insert" + return timeline_operation(payload) + + +@api.post("/timeline/replace", dependencies=[Depends(_verify_api_key)]) +def timeline_replace(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "replace" + return timeline_operation(payload) + + +@api.post("/timeline/group", dependencies=[Depends(_verify_api_key)]) +def timeline_group(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "group" + return timeline_operation(payload) + + +@api.post("/timeline/lock", dependencies=[Depends(_verify_api_key)]) +def timeline_lock(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "lock" + return timeline_operation(payload) + + +@api.post("/timeline/hide", dependencies=[Depends(_verify_api_key)]) +def timeline_hide(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "hide" + return timeline_operation(payload) + + +@api.post("/timeline/duplicate", dependencies=[Depends(_verify_api_key)]) +def timeline_duplicate(payload: TimelineOperationPayload) -> dict[str, Any]: + payload.operation = "duplicate" + return timeline_operation(payload) + + +@api.post("/effect/apply", dependencies=[Depends(_verify_api_key)]) +def effect_apply(payload: EffectApplyPayload) -> dict[str, Any]: + target_id = payload.target_id or payload.item_id + if payload.project_id and target_id: + try: + project = project_manager.get(payload.project_id) + record = add_effect(project, target_id, payload.effect, payload.params) + project = project_manager.save(payload.project_id, project) + return {"effect": record, "project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project or target item not found") from exc + data = {"task": payload.effect, "input": target_id, "params": payload.params, "callback_url": payload.callback_url} + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).ai_tool("auto_edit", data, task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/filter/apply", dependencies=[Depends(_verify_api_key)]) +def filter_apply(payload: FilterApplyPayload) -> dict[str, Any]: + target_id = payload.target_id or payload.item_id + params = dict(payload.params) + if payload.lut: + params["lut"] = payload.lut + if payload.project_id and target_id: + try: + project = project_manager.get(payload.project_id) + record = add_filter(project, target_id, payload.filter, params) + project = project_manager.save(payload.project_id, project) + return {"filter": record, "project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project or target item not found") from exc + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).ai_tool("auto_color_match", payload.model_dump(), task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/transition/add", dependencies=[Depends(_verify_api_key)]) +def transition_add(payload: TransitionAddPayload) -> dict[str, Any]: + if payload.project_id and payload.from_item_id and payload.to_item_id: + try: + project = project_manager.get(payload.project_id) + record = add_transition(project, payload.from_item_id, payload.to_item_id, payload.transition, payload.duration) + project = project_manager.save(payload.project_id, project) + return {"transition": record, "project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project not found") from exc + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).ai_tool("auto_edit", payload.model_dump(), task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/keyframe/add", dependencies=[Depends(_verify_api_key)]) +def keyframe_add(payload: KeyframePayload) -> dict[str, Any]: + try: + project = project_manager.get(payload.project_id) + record = add_keyframe(project, payload.target_id, payload.property, payload.time, payload.value, payload.easing) + project = project_manager.save(payload.project_id, project) + return {"keyframe": record, "project": project} + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project or target item not found") from exc + + +@api.post("/caption/generate", dependencies=[Depends(_verify_api_key)]) +def caption_generate(payload: CaptionGeneratePayload) -> dict[str, str]: + data = payload.model_dump() + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).caption_generate(data, task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/music/generate", dependencies=[Depends(_verify_api_key)]) +def music_generate(payload: GenerationPayload) -> dict[str, str]: + data = _generation_data(payload) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).music_generate(data, task_id), + callback_url=payload.callback_url, + export_target=payload.export_target, + ) + return _submission_response(job_id) + + +@api.post("/voice/generate", dependencies=[Depends(_verify_api_key)]) +def voice_generate(payload: GenerationPayload) -> dict[str, str]: + data = _generation_data(payload) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).voice_generate(data, task_id), + callback_url=payload.callback_url, + export_target=payload.export_target, + ) + return _submission_response(job_id) + + +@api.post("/image/generate", dependencies=[Depends(_verify_api_key)]) +def image_generate(payload: GenerationPayload) -> dict[str, str]: + data = _generation_data(payload) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).image_generate(data, task_id), + callback_url=payload.callback_url, + export_target=payload.export_target, + ) + return _submission_response(job_id) + + +@api.post("/video/generate", dependencies=[Depends(_verify_api_key)]) +def video_generate(payload: GenerationPayload) -> dict[str, str]: + data = _generation_data(payload) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).video_generate(data, task_id), + callback_url=payload.callback_url, + export_target=payload.export_target, + ) + return _submission_response(job_id) + + +@api.post("/ai/{tool}", dependencies=[Depends(_verify_api_key)]) +def ai_tool(tool: str, payload: AIToolPayload) -> dict[str, str]: + data = payload.model_dump() + if payload.model_extra: + data.update(payload.model_extra) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).ai_tool(tool, data, task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/assistant/{tool}", dependencies=[Depends(_verify_api_key)]) +def assistant_tool(tool: str, payload: AIToolPayload) -> dict[str, str]: + data = payload.model_dump() + if payload.model_extra: + data.update(payload.model_extra) + job_id = job_manager.submit_task( + lambda task_id, log: StudioTaskProcessor(settings, log=log).assistant_tool(tool, data, task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/render") +def render(payload: RenderPayload | AIReelsPayload) -> dict[str, str]: + if isinstance(payload, AIReelsPayload): + job_id = job_manager.submit_ai_reels(_ai_reels_request(payload)) + else: + job_id = job_manager.submit_render(_render_request(payload)) + return _submission_response(job_id) + + + +@api.post("/render/ai-reels") +def render_ai_reels(payload: AIReelsPayload) -> dict[str, str]: + job_id = job_manager.submit_ai_reels(_ai_reels_request(payload)) + return _submission_response(job_id) + + +@api.post("/render/batch") +def render_batch(payload: BatchPayload) -> dict[str, list[str]]: + job_ids = job_manager.submit_batch([_render_request(job) for job in payload.jobs]) + return {"job_ids": job_ids} + + +@api.post("/automation/batch", dependencies=[Depends(_verify_api_key)]) +def automation_batch(payload: BatchPayload) -> dict[str, list[str]]: + return render_batch(payload) + + +@api.post("/project/{project_id}/render", dependencies=[Depends(_verify_api_key)]) +def project_render(project_id: str, payload: ProjectRenderPayload) -> dict[str, str]: + try: + project = project_manager.get(project_id) + request = _project_render_request(project, payload) + job_id = job_manager.submit_render(request) + return _submission_response(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Project not found") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.post("/render/upload") +async def render_upload(request_json: str = Form(...), files: list[UploadFile] = File(default=[])) -> dict[str, str]: + uploads = await _stage_uploads(files) + payload_data = _replace_upload_refs(json.loads(request_json), uploads) + payload = RenderPayload.model_validate(payload_data) + job_id = job_manager.submit_render(_render_request(payload)) + return _submission_response(job_id) + + +@api.post("/render/ai-reels/upload") +async def render_ai_reels_upload(request_json: str = Form(...), files: list[UploadFile] = File(default=[])) -> dict[str, str]: + uploads = await _stage_uploads(files) + payload_data = _replace_upload_refs(json.loads(request_json), uploads) + payload = AIReelsPayload.model_validate(payload_data) + job_id = job_manager.submit_ai_reels(_ai_reels_request(payload)) + return _submission_response(job_id) + + +@api.post("/assets/upload") +async def upload_assets(files: list[UploadFile] = File(...)) -> dict[str, list[UploadedAsset]]: + uploads = await _stage_uploads(files) + assets = [ + UploadedAsset(filename=filename, path=path, reference=f"upload://{filename}", kind=_asset_kind(filename)) for filename, path in uploads.items() + ] + return {"assets": [asset.model_dump() for asset in assets]} + + +@api.post("/upload") +async def upload(files: list[UploadFile] = File(...), expand_zip: bool = Form(default=True)) -> dict[str, list[UploadedAsset]]: + uploads = await _stage_uploads(files) + expanded: dict[str, str] = {} + for filename, path in uploads.items(): + if expand_zip and filename.lower().endswith(".zip"): + expanded.update(_extract_zip(Path(path))) + else: + expanded[filename] = path + assets = [ + UploadedAsset(filename=filename, path=path, reference=f"upload://{filename}", kind=_asset_kind(filename)) + for filename, path in expanded.items() + ] + return {"assets": [asset.model_dump() for asset in assets]} + + +@api.post("/ingest", dependencies=[Depends(_verify_api_key)]) +def ingest(payload: IngestPayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).ingest_sources([source.model_dump() for source in payload.sources], task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/analyze", dependencies=[Depends(_verify_api_key)]) +def analyze(payload: AnalyzePayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).analyze( + payload.media, + task_id, + transcript=payload.transcript, + platform=payload.platform, + ), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/clips", dependencies=[Depends(_verify_api_key)]) +def clips(payload: ClipsPayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).clips(payload.media, task_id, payload.clips), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/thumbnail", dependencies=[Depends(_verify_api_key)]) +def thumbnail(payload: ThumbnailPayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).thumbnail( + payload.media, + task_id, + text=payload.text, + timestamp=payload.timestamp, + template=payload.template, + ), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/thumbnail/create", dependencies=[Depends(_verify_api_key)]) +def thumbnail_create(payload: ThumbnailPayload) -> dict[str, str]: + return thumbnail(payload) + + +@api.post("/metadata", dependencies=[Depends(_verify_api_key)]) +def metadata(payload: MetadataPayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).metadata( + task_id, + topic=payload.topic, + transcript=payload.transcript, + platform=payload.platform, + ), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/publish", dependencies=[Depends(_verify_api_key)]) +def publish(payload: PublishPayload) -> dict[str, str]: + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).publish(payload.model_dump(), task_id), + callback_url=payload.callback_url, + ) + return _submission_response(job_id) + + +@api.post("/toolkit", dependencies=[Depends(_verify_api_key)]) +def toolkit(payload: ToolkitPayload) -> dict[str, str]: + data = payload.model_dump() + if payload.model_extra: + data.update(payload.model_extra) + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).toolkit(data, task_id), + callback_url=payload.callback_url, + export_target=payload.export_target, + ) + return _submission_response(job_id) + + +@api.post("/edit", dependencies=[Depends(_verify_api_key)]) +def edit(payload: ToolkitPayload) -> dict[str, str]: + return toolkit(payload) + + +@api.post("/transcribe") +def transcribe(payload: TranscribePayload) -> dict: + from renderer.core.ingest import AssetIngestor + from renderer.core.utils import temp_workdir + + try: + with temp_workdir(settings.temp_dir, "transcribe") as work: + audio = AssetIngestor(settings).resolve(payload.audio, Path(work) / "inputs", "audio") + return RenderEngine(settings).transcribe( + audio, + model_size=payload.model_size, + language=payload.language, + task=payload.task, + beam_size=payload.beam_size, + vad_filter=payload.vad_filter, + word_timestamps=payload.word_timestamps, + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.post("/subtitles") +def subtitles(payload: SubtitlePayload) -> FileResponse: + try: + events = [SubtitleEvent(float(event["start"]), float(event["end"]), str(event["text"])) for event in payload.events] + path = settings.temp_dir / f"subtitles_{uuid.uuid4().hex}.{payload.format}" + generator = SubtitleGenerator() + if payload.format == "ass": + generator.write_ass(events, path, payload.template) + media_type = "text/x-ssa" + else: + generator.write_srt(events, path) + media_type = "application/x-subrip" + return FileResponse(path, media_type=media_type, filename=path.name) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.post("/scene-builder") +def scene_builder(payload: SceneBuildPayload) -> dict[str, Any]: + if not payload.assets: + raise HTTPException(status_code=400, detail="At least one asset is required") + style = get_creative_style(payload.creative_style) + words = payload.script.split() + total_duration = payload.duration or max(style.scene_duration * len(payload.assets), len(words) * 0.35, 3.0) + per_scene = total_duration / len(payload.assets) + captions = _split_words_for_assets(words, len(payload.assets)) + scenes = [ + { + "start": round(index * per_scene, 3), + "duration": round(per_scene, 3), + "media": asset, + "caption": captions[index] if index < len(captions) else "", + "transition": _style_transition(payload.transition, style.transition_sequence, index), + "effect": style.scene_effect_sequence[index % len(style.scene_effect_sequence)], + "background": "blur", + "layout": "fill", + } + for index, asset in enumerate(payload.assets) + ] + return {"scenes": scenes, "creative_style": style.metadata_payload()} + + +@api.post("/transcribe/upload") +async def transcribe_upload( + file: UploadFile = File(...), + model_size: str | None = Form(default=None), + language: str | None = Form(default=None), + task: str = Form(default="transcribe"), + beam_size: int = Form(default=5), + vad_filter: bool = Form(default=True), + word_timestamps: bool = Form(default=True), +) -> dict: + try: + uploads = await _stage_uploads([file]) + audio = next(iter(uploads.values())) + return RenderEngine(settings).transcribe( + audio, + model_size=model_size, + language=language, + task=task, + beam_size=beam_size, + vad_filter=vad_filter, + word_timestamps=word_timestamps, + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@api.get("/status") +def status_query(job_id: str = Query(...)) -> dict: + return status(job_id) + + +@api.get("/status/{job_id}") +def status(job_id: str) -> dict: + try: + return job_manager.get(job_id).__dict__ + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + + +@api.post("/cancel/{job_id}") +def cancel(job_id: str) -> dict: + try: + return job_manager.cancel(job_id).__dict__ + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + + +@api.post("/admin/cleanup") +def cleanup(older_than_seconds: int | None = None) -> dict[str, int]: + return job_manager.cleanup(older_than_seconds) + + +@api.get("/download") +def download_query(job_id: str = Query(...), token: str | None = Query(default=None)) -> FileResponse: + return download(job_id, token) + + +@api.get("/download/{job_id}") +def download(job_id: str, token: str | None = Query(default=None)) -> FileResponse: + try: + record = job_manager.get(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + if record.state != "COMPLETED" or not record.output_path: + raise HTTPException(status_code=409, detail=f"Job is {record.state}") + if record.download_token and not verify_download_token(settings.signing_secret, job_id, token): + raise HTTPException(status_code=403, detail="Invalid or missing download token") + path = Path(record.output_path) + if not path.exists(): + raise HTTPException(status_code=404, detail="Output file is missing") + return FileResponse(path, media_type=_media_type(path), filename=path.name) + + +@api.post("/inspect") +def inspect_asset(path: str) -> dict: + try: + return RenderEngine(settings).inspect_asset(path) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +def _render_request(payload: RenderPayload) -> RenderRequest: + payload = RenderPayload.model_validate(apply_creative_style(apply_preset(payload.model_dump(exclude_unset=True)))) + request = RenderRequest( + scenes=[Scene(**scene.model_dump()) for scene in payload.scenes], + template=payload.template, + preset=payload.preset, + creative_style=payload.creative_style, + platform=payload.platform, + output_name=payload.output_name, + voiceover=payload.voiceover, + background_music=payload.background_music, + music_volume=payload.music_volume, + music_fade_in=payload.music_fade_in, + music_fade_out=payload.music_fade_out, + music_loop=payload.music_loop, + music_start=payload.music_start, + music_ducking=payload.music_ducking, + voice_volume=payload.voice_volume, + subtitle_format=payload.subtitle_format, # type: ignore[arg-type] + auto_subtitles=payload.auto_subtitles, + subtitle_language=payload.subtitle_language, + whisper_model_size=payload.whisper_model_size, + preview=payload.preview, + audio_normalize=payload.audio_normalize, + watermark=payload.watermark, + watermark_position=payload.watermark_position, + intro=payload.intro, + outro=payload.outro, + callback_url=payload.callback_url, + export_target=payload.export_target, + priority=payload.priority, + scheduled_at=payload.scheduled_at, + normalize=payload.normalize, + metadata=payload.metadata, + ) + Timeline(request.scenes) + return request + + +def _project_render_request(project: dict[str, Any], payload: ProjectRenderPayload) -> RenderRequest: + scenes: list[dict[str, Any]] = [] + for tracks in project.get("timeline", {}).get("tracks", {}).values(): + for track in tracks: + if track.get("hidden") or track.get("type") not in {"video", "overlay"}: + continue + for item in track.get("items", []): + if item.get("hidden"): + continue + media = _item_media(project, item) + if not media: + continue + scenes.append( + { + "start": float(item.get("start", 0.0)), + "duration": float(item.get("duration", 1.0)), + "media": media, + "caption": item.get("caption") or item.get("text") or "", + "transition": item.get("transition", "fade"), + "background": item.get("background", "blur"), + "layout": item.get("layout", "fill"), + "effect": _first_named(item.get("effects")), + } + ) + if not scenes: + raise ValueError("Project has no renderable video or overlay timeline items") + export_settings = project.get("export_settings", {}) + data = { + "scenes": sorted(scenes, key=lambda scene: scene["start"]), + "template": payload.template or export_settings.get("template", "tiktok_classic"), + "preset": payload.preset, + "creative_style": payload.creative_style or project.get("metadata", {}).get("creative_style"), + "platform": payload.platform or export_settings.get("platform"), + "output_name": payload.output_name, + "callback_url": payload.callback_url, + "export_target": payload.export_target, + "preview": payload.preview, + "normalize": payload.normalize, + "metadata": project.get("metadata", {}) | payload.metadata | {"project_id": project.get("id"), "project_name": project.get("name")}, + } + return _render_request(RenderPayload.model_validate(data)) + + +def _ai_reels_request(payload: AIReelsPayload) -> AIReelsRequest: + payload = AIReelsPayload.model_validate(apply_creative_style(apply_preset(payload.model_dump(exclude_unset=True)))) + return AIReelsRequest(**payload.model_dump()) + + +def _generation_data(payload: GenerationPayload) -> dict[str, Any]: + data = payload.model_dump() + if payload.model_extra: + data.update(payload.model_extra) + params = data.pop("params", {}) or {} + if isinstance(params, dict): + data.update(params) + return data + + +def _item_media(project: dict[str, Any], item: dict[str, Any]) -> str | None: + direct = item.get("media") or item.get("path") or item.get("source") + if direct: + return str(direct) + asset_id = item.get("asset_id") + if not asset_id: + return None + for asset in project.get("assets", []): + if asset.get("id") == asset_id: + return str(asset.get("path") or asset.get("url") or asset.get("source") or "") + return None + + +def _first_named(records: Any) -> str | None: + if not isinstance(records, list) or not records: + return None + first = records[0] + if isinstance(first, dict): + return first.get("effect") or first.get("name") + return str(first) + + +async def _stage_uploads(files: list[UploadFile]) -> dict[str, str]: + upload_dir = settings.temp_dir / "uploads" / uuid.uuid4().hex + staged: dict[str, str] = {} + total_bytes = 0 + for upload in files: + filename = safe_filename(upload.filename or f"asset_{len(staged)}") + if not _allowed_upload(filename): + raise HTTPException(status_code=415, detail=f"Unsupported asset type: {filename}") + target = upload_dir / filename + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as output: + while True: + chunk = await upload.read(1024 * 1024) + if not chunk: + break + total_bytes += len(chunk) + if total_bytes > settings.max_download_bytes: + shutil.rmtree(upload_dir, ignore_errors=True) + raise HTTPException(status_code=413, detail="Uploaded assets exceed MAX_DOWNLOAD_BYTES") + output.write(chunk) + staged[filename] = str(stage_upload(target, upload_dir, filename)) + return staged + + +def _monitor_payload() -> dict[str, Any]: + disk = shutil.disk_usage(settings.base_dir) + return { + "health": "ok", + "queue": job_manager.summary(), + "workers": { + "max_workers": settings.max_workers, + "ffmpeg_timeout_seconds": settings.ffmpeg_timeout_seconds, + "whisper_model_size": settings.whisper_model_size, + "whisper_device": settings.whisper_device, + }, + "disk": { + "total_bytes": disk.total, + "used_bytes": disk.used, + "free_bytes": disk.free, + }, + "directories": { + "temp": str(settings.temp_dir), + "exports": str(settings.exports_dir), + "jobs": str(settings.jobs_dir), + "storage": str(settings.storage_dir), + }, + } + + +def _extract_zip(path: Path) -> dict[str, str]: + output_dir = settings.temp_dir / "uploads" / f"zip_{uuid.uuid4().hex}" + output_dir.mkdir(parents=True, exist_ok=True) + extracted: dict[str, str] = {} + with zipfile.ZipFile(path) as archive: + for member in archive.infolist(): + if member.is_dir(): + continue + name = safe_filename(Path(member.filename).name) + if not _allowed_upload(name): + continue + target = output_dir / name + resolved = target.resolve() + if output_dir.resolve() not in resolved.parents and resolved != output_dir.resolve(): + raise HTTPException(status_code=400, detail="Unsafe ZIP member path") + with archive.open(member) as source, target.open("wb") as destination: + shutil.copyfileobj(source, destination) + extracted[name] = str(target) + return extracted + + +def _asset_kind(filename: str) -> str: + suffix = Path(filename).suffix.lower() + if suffix in {".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi", ".gif"}: + return "video" + if suffix in {".mp3", ".wav", ".m4a", ".aac", ".flac", ".ogg"}: + return "audio" + if suffix in {".jpg", ".jpeg", ".png", ".webp", ".avif"}: + return "image" + if suffix in {".srt", ".ass", ".vtt"}: + return "subtitle" + return "other" + + +def _allowed_upload(filename: str) -> bool: + suffix = Path(filename).suffix.lower() + return suffix in { + ".mp4", + ".mov", + ".m4v", + ".webm", + ".mkv", + ".avi", + ".gif", + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".jpg", + ".jpeg", + ".png", + ".webp", + ".avif", + ".srt", + ".ass", + ".vtt", + ".zip", + } + + +def _media_type(path: Path) -> str: + suffix = path.suffix.lower() + if suffix == ".json": + return "application/json" + if suffix == ".zip": + return "application/zip" + if suffix == ".jpg" or suffix == ".jpeg": + return "image/jpeg" + if suffix == ".png": + return "image/png" + if suffix == ".gif": + return "image/gif" + if suffix == ".mp3": + return "audio/mpeg" + if suffix in {".srt", ".vtt", ".ass"}: + return "text/plain" + return "video/mp4" + + +def _replace_upload_refs(value: Any, uploads: dict[str, str]) -> Any: + if isinstance(value, dict): + return {key: _replace_upload_refs(item, uploads) for key, item in value.items()} + if isinstance(value, list): + return [_replace_upload_refs(item, uploads) for item in value] + if isinstance(value, str) and value.startswith("upload://"): + name = safe_filename(value.removeprefix("upload://")) + if name not in uploads: + raise HTTPException(status_code=400, detail=f"Missing uploaded file for reference: upload://{name}") + return uploads[name] + return value + + +def _split_words_for_assets(words: list[str], count: int) -> list[str]: + if count <= 0: + return [] + if not words: + return [""] * count + base, remainder = divmod(len(words), count) + captions: list[str] = [] + cursor = 0 + for index in range(count): + size = base + (1 if index < remainder else 0) + size = max(1, size) + captions.append(" ".join(words[cursor : cursor + size])) + cursor += size + return captions + + +def _style_transition(requested: str, sequence: tuple[str, ...], index: int) -> str: + if requested and requested != "fade": + return requested + return sequence[index % len(sequence)] if sequence else "fade" diff --git a/services/render_engine/app.py b/services/render_engine/app.py new file mode 100644 index 0000000000000000000000000000000000000000..f297504b7ebfb68a66dc13a26e360a45e5ea6248 --- /dev/null +++ b/services/render_engine/app.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import gradio as gr +import uvicorn + +from api import api, job_manager, project_manager, settings +from renderer import RenderEngine +from renderer.core.models import AIReelsRequest +from renderer.scenes import Timeline +from renderer.studio import capability_catalog +from renderer.templates import apply_creative_style, apply_preset, list_creative_styles, list_platform_profiles, list_templates + + +def create_dashboard() -> gr.Blocks: + with gr.Blocks(title="Ava2lon Studio AI") as dashboard: + gr.Markdown( + "# Ava2lon Studio AI\n" + "CPU-first CapCut-class video automation studio with REST API parity, async jobs, webhooks, and project JSON." + ) + with gr.Tab("Projects"): + project_name = gr.Textbox(label="Project name", value="Untitled Ava2lon Project") + project_metadata = gr.Textbox(label="Metadata JSON", lines=5, value=json.dumps({"platform": "tiktok"}, indent=2)) + project_create = gr.Button("Create Project", variant="primary") + project_list = gr.Button("Refresh Projects") + project_output = gr.JSON(label="Projects") + project_create.click(fn=_create_project, inputs=[project_name, project_metadata], outputs=project_output) + project_list.click(fn=_list_projects, outputs=project_output) + + with gr.Tab("Assets"): + gr.Markdown("Use `/upload` or `/assets/upload` for multipart assets, then attach them to a project with `/project/assets/add`.") + asset_project_id = gr.Textbox(label="Project ID") + asset_json = gr.Textbox(label="Asset JSON", lines=6, value=json.dumps({"path": "clip.mp4", "kind": "video"}, indent=2)) + asset_button = gr.Button("Attach Asset", variant="primary") + asset_output = gr.JSON(label="Project") + asset_button.click(fn=_add_project_asset, inputs=[asset_project_id, asset_json], outputs=asset_output) + + with gr.Tab("Timeline"): + timeline_project_id = gr.Textbox(label="Project ID") + timeline_track_type = gr.Dropdown(choices=["video", "audio", "text", "overlay", "sticker", "subtitle"], value="video", label="Track type") + timeline_item = gr.Textbox( + label="Timeline item JSON", + lines=8, + value=json.dumps({"media": "clip.mp4", "start": 0, "duration": 5, "caption": "Hook"}, indent=2), + ) + timeline_add_button = gr.Button("Add To Timeline", variant="primary") + timeline_output = gr.JSON(label="Project") + timeline_add_button.click(fn=_timeline_add, inputs=[timeline_project_id, timeline_track_type, timeline_item], outputs=timeline_output) + + timeline_operation_json = gr.Textbox( + label="Operation JSON", + lines=8, + value=json.dumps({"operation": "split", "item_id": "clip_123", "params": {"offset": 2.5}}, indent=2), + ) + timeline_operation_button = gr.Button("Apply Operation") + timeline_operation_button.click(fn=_timeline_operation, inputs=[timeline_project_id, timeline_operation_json], outputs=timeline_output) + + with gr.Tab("Templates"): + template_button = gr.Button("Load Template Catalog") + template_output = gr.JSON(label="Templates") + template_button.click(fn=lambda: _catalog_section("templates"), outputs=template_output) + + with gr.Tab("Effects"): + effect_button = gr.Button("Load Effect Catalog") + effect_output = gr.JSON(label="Effects") + effect_button.click(fn=lambda: _catalog_section("effects"), outputs=effect_output) + + with gr.Tab("Filters"): + filter_button = gr.Button("Load Filter Catalog") + filter_output = gr.JSON(label="Filters") + filter_button.click(fn=lambda: _catalog_section("filters"), outputs=filter_output) + + with gr.Tab("Captions"): + caption_text = gr.Textbox(label="Caption source text", lines=6) + caption_button = gr.Button("Submit Caption Job", variant="primary") + caption_output = gr.JSON(label="Caption Job") + caption_button.click(fn=_submit_caption_generation, inputs=caption_text, outputs=caption_output) + + with gr.Tab("Audio"): + audio_button = gr.Button("Load Audio Catalog") + audio_output = gr.JSON(label="Audio") + audio_button.click(fn=lambda: {"audio": capability_catalog()["audio"], "music": capability_catalog()["music_generator"]}, outputs=audio_output) + + with gr.Tab("AI Tools"): + ai_tool = gr.Dropdown(choices=capability_catalog()["ai_editing"], value="auto_viral_score", label="AI tool") + ai_payload = gr.Textbox(label="AI payload JSON", lines=8, value=json.dumps({"platform": "tiktok", "text": "A strong opening hook"}, indent=2)) + ai_button = gr.Button("Submit AI Tool", variant="primary") + ai_output = gr.JSON(label="AI Job") + ai_button.click(fn=_submit_ai_tool, inputs=[ai_tool, ai_payload], outputs=ai_output) + + with gr.Tab("Rendering"): + render_json = gr.Textbox( + label="Render JSON", + lines=14, + value="", + placeholder="Paste a production render request JSON object with absolute or uploaded asset paths.", + ) + render_button = gr.Button("Submit Render", variant="primary") + render_output = gr.JSON(label="Submission") + render_button.click(fn=_submit_render_json, inputs=render_json, outputs=render_output) + + with gr.Tab("AI Reels"): + script = gr.Textbox(label="Script", lines=6) + voiceover = gr.File(label="Voiceover", file_types=["audio"], type="filepath") + assets = gr.File(label="Assets", file_count="multiple", type="filepath") + template = gr.Dropdown(choices=list_templates(), value="tiktok_classic", label="Caption Template") + creative_style = gr.Dropdown(choices=list_creative_styles(), value="viral_shorts", label="Creative Style") + platform = gr.Dropdown(choices=list_platform_profiles(), value="tiktok", label="Platform") + music = gr.File(label="Background Music", file_types=["audio"], type="filepath") + ai_button = gr.Button("Submit AI Reel", variant="primary") + ai_output = gr.JSON(label="Submission") + ai_button.click(fn=_submit_ai_reel, inputs=[script, voiceover, assets, template, creative_style, platform, music], outputs=ai_output) + + with gr.Tab("Batch Render"): + batch_json = gr.Textbox(label="Batch JSON", lines=14, value=json.dumps({"jobs": []}, indent=2)) + batch_button = gr.Button("Submit Batch", variant="primary") + batch_output = gr.JSON(label="Batch Submission") + batch_button.click(fn=_submit_batch_json, inputs=batch_json, outputs=batch_output) + + with gr.Tab("Job Status"): + status_job_id = gr.Textbox(label="Job ID") + status_button = gr.Button("Refresh") + status_output = gr.JSON(label="Status") + status_button.click(fn=_job_status, inputs=status_job_id, outputs=status_output) + + with gr.Tab("Logs"): + logs_job_id = gr.Textbox(label="Job ID") + logs_button = gr.Button("Load Logs") + logs_output = gr.Textbox(label="Logs", lines=20) + logs_button.click(fn=_job_logs, inputs=logs_job_id, outputs=logs_output) + + with gr.Tab("Downloads"): + download_job_id = gr.Textbox(label="Job ID") + download_button = gr.Button("Get Output") + download_output = gr.File(label="Rendered Video") + download_button.click(fn=_download_path, inputs=download_job_id, outputs=download_output) + + with gr.Tab("Transcribe"): + transcribe_audio = gr.File(label="Audio or Video", file_types=["audio", "video"], type="filepath") + transcribe_model = gr.Dropdown( + choices=["tiny", "base", "small", "medium", "large-v3"], + value=settings.whisper_model_size, + label="Whisper Model", + ) + transcribe_language = gr.Textbox(label="Language", placeholder="Optional ISO code, e.g. en") + transcribe_button = gr.Button("Transcribe", variant="primary") + transcribe_output = gr.JSON(label="Transcript") + transcribe_button.click( + fn=_transcribe_file, + inputs=[transcribe_audio, transcribe_model, transcribe_language], + outputs=transcribe_output, + ) + + with gr.Tab("Asset Inspector"): + asset_path = gr.Textbox(label="Asset path") + inspect_button = gr.Button("Inspect") + inspect_output = gr.JSON(label="Metadata") + inspect_button.click(fn=_inspect_asset, inputs=asset_path, outputs=inspect_output) + + with gr.Tab("AI Analysis"): + analysis_media = gr.Textbox(label="Media URL or path") + analysis_transcript = gr.Textbox(label="Transcript", lines=5) + analysis_platform = gr.Dropdown(choices=list_platform_profiles(), value="tiktok", label="Target Platform") + analysis_button = gr.Button("Submit Analysis", variant="primary") + analysis_output = gr.JSON(label="Analysis Job") + analysis_button.click( + fn=_submit_analysis, + inputs=[analysis_media, analysis_transcript, analysis_platform], + outputs=analysis_output, + ) + + with gr.Tab("Clip Generator"): + clip_media = gr.Textbox(label="Media URL or path") + clip_json = gr.Textbox(label="Clip JSON", lines=6, value=json.dumps([{"start": 0, "end": 8}], indent=2)) + clip_button = gr.Button("Generate Clips", variant="primary") + clip_output = gr.JSON(label="Clip Job") + clip_button.click(fn=_submit_clips, inputs=[clip_media, clip_json], outputs=clip_output) + + with gr.Tab("Metadata"): + metadata_topic = gr.Textbox(label="Topic or transcript", lines=5) + metadata_platform = gr.Dropdown(choices=list_platform_profiles(), value="tiktok", label="Platform") + metadata_button = gr.Button("Generate Metadata", variant="primary") + metadata_output = gr.JSON(label="Metadata Job") + metadata_button.click(fn=_submit_metadata, inputs=[metadata_topic, metadata_platform], outputs=metadata_output) + + with gr.Tab("Publishing"): + publish_media = gr.Textbox(label="Media URL or rendered output path") + publish_title = gr.Textbox(label="Title") + publish_platforms = gr.Textbox(label="Platforms", value="youtube,tiktok,instagram") + publish_button = gr.Button("Create Publish Draft", variant="primary") + publish_output = gr.JSON(label="Publish Job") + publish_button.click(fn=_submit_publish, inputs=[publish_media, publish_title, publish_platforms], outputs=publish_output) + + with gr.Tab("Settings"): + settings_button = gr.Button("Load Settings") + settings_output = gr.JSON(label="Settings") + settings_button.click(fn=_settings_payload, outputs=settings_output) + + with gr.Tab("Queue Monitor"): + queue_button = gr.Button("Refresh Queue") + queue_output = gr.JSON(label="Queue") + queue_button.click(fn=_queue_status, outputs=queue_output) + + return dashboard + + +def _submit_render_json(payload: str) -> dict[str, Any]: + data = apply_creative_style(apply_preset(json.loads(payload))) + request = Timeline.request_from_payload(data) + job_id = job_manager.submit_render(request) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _create_project(name: str, metadata_json: str) -> dict[str, Any]: + metadata = json.loads(metadata_json or "{}") + return {"project": project_manager.create(name, metadata=metadata)} + + +def _list_projects() -> dict[str, Any]: + return {"projects": project_manager.list()} + + +def _add_project_asset(project_id: str, asset_json: str) -> dict[str, Any]: + return {"project": project_manager.add_asset(project_id, json.loads(asset_json or "{}"))} + + +def _timeline_add(project_id: str, track_type: str, item_json: str) -> dict[str, Any]: + return {"project": project_manager.add_to_timeline(project_id, json.loads(item_json or "{}"), track_type=track_type)} + + +def _timeline_operation(project_id: str, operation_json: str) -> dict[str, Any]: + data = json.loads(operation_json or "{}") + return { + "project": project_manager.timeline_operation( + project_id, + data.get("operation", "drag"), + item_id=data.get("item_id"), + params=data.get("params", {}), + ) + } + + +def _catalog_section(section: str) -> dict[str, Any]: + catalog = capability_catalog() + return {section: catalog.get(section)} + + +def _submit_caption_generation(text: str) -> dict[str, Any]: + from renderer.studio import StudioTaskProcessor + + job_id = job_manager.submit_task(lambda task_id, log: StudioTaskProcessor(settings, log=log).caption_generate({"text": text}, task_id)) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _submit_ai_tool(tool: str, payload: str) -> dict[str, Any]: + from renderer.studio import StudioTaskProcessor + + data = json.loads(payload or "{}") + job_id = job_manager.submit_task(lambda task_id, log: StudioTaskProcessor(settings, log=log).ai_tool(tool, data, task_id)) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _submit_batch_json(payload: str) -> dict[str, Any]: + data = json.loads(payload) + requests = [Timeline.request_from_payload(apply_creative_style(apply_preset(job))) for job in data.get("jobs", [])] + return {"job_ids": job_manager.submit_batch(requests)} + + +def _submit_ai_reel( + script: str, + voiceover: str, + assets: list[str], + template: str, + creative_style: str, + platform: str, + music: str | None, +) -> dict[str, Any]: + request = AIReelsRequest( + script=script, + voiceover=voiceover, + assets=assets or [], + template=template, + creative_style=creative_style, + platform=platform, + background_music=music, + ) + job_id = job_manager.submit_ai_reels(request) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _job_status(job_id: str) -> dict[str, Any]: + return job_manager.get(job_id).__dict__ + + +def _job_logs(job_id: str) -> str: + return "\n\n".join(job_manager.get(job_id).logs) + + +def _download_path(job_id: str) -> str | None: + record = job_manager.get(job_id) + if record.state != "COMPLETED": + return None + return record.output_path + + +def _inspect_asset(path: str) -> dict[str, Any]: + return RenderEngine(settings).inspect_asset(path) + + +def _transcribe_file(path: str, model_size: str, language: str) -> dict[str, Any]: + return RenderEngine(settings).transcribe( + path, + model_size=model_size, + language=language.strip() or None, + word_timestamps=True, + ) + + +def _submit_analysis(media: str, transcript: str, platform: str) -> dict[str, Any]: + from renderer.platform import PlatformProcessor + + job_id = job_manager.submit_task( + lambda task_id, log: PlatformProcessor(settings, log=log).analyze(media, task_id, transcript=transcript, platform=platform) + ) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _submit_clips(media: str, clips_json: str) -> dict[str, Any]: + from renderer.platform import PlatformProcessor + + clips = json.loads(clips_json) + job_id = job_manager.submit_task(lambda task_id, log: PlatformProcessor(settings, log=log).clips(media, task_id, clips)) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _submit_metadata(topic: str, platform: str) -> dict[str, Any]: + from renderer.platform import PlatformProcessor + + job_id = job_manager.submit_task(lambda task_id, log: PlatformProcessor(settings, log=log).metadata(task_id, topic=topic, platform=platform)) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _submit_publish(media: str, title: str, platforms: str) -> dict[str, Any]: + from renderer.platform import PlatformProcessor + + payload = {"media": media, "title": title, "platforms": [item.strip() for item in platforms.split(",") if item.strip()], "draft": True} + job_id = job_manager.submit_task(lambda task_id, log: PlatformProcessor(settings, log=log).publish(payload, task_id)) + return {"job_id": job_id, "status": f"/status/{job_id}", "download": f"/download/{job_id}"} + + +def _queue_status() -> dict[str, Any]: + return job_manager.summary() + + +def _settings_payload() -> dict[str, Any]: + return { + "product": "Ava2lon Studio AI", + "base_dir": str(settings.base_dir), + "temp_dir": str(settings.temp_dir), + "exports_dir": str(settings.exports_dir), + "storage_dir": str(settings.storage_dir), + "max_workers": settings.max_workers, + "whisper_model_size": settings.whisper_model_size, + "whisper_device": settings.whisper_device, + "capabilities": capability_catalog()["principles"], + } + + +app = gr.mount_gradio_app(api, create_dashboard(), path="/dashboard") + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=7860) diff --git a/services/render_engine/renderer/__init__.py b/services/render_engine/renderer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..769631dca38fb3f6b754d27fbafba0eca8bbe2f9 --- /dev/null +++ b/services/render_engine/renderer/__init__.py @@ -0,0 +1,7 @@ +"""CPU-first video automation backend for Ava2lon Studio AI.""" + +from renderer.core.config import Settings +from renderer.core.models import RenderRequest, RenderResult +from renderer.core.render_engine import RenderEngine + +__all__ = ["RenderEngine", "RenderRequest", "RenderResult", "Settings"] diff --git a/services/render_engine/renderer/audio/__init__.py b/services/render_engine/renderer/audio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bdadb10548b7d4e9c6e703ca6cf3dec6b77c47a2 --- /dev/null +++ b/services/render_engine/renderer/audio/__init__.py @@ -0,0 +1,3 @@ +from renderer.audio.mixer import AudioMixer + +__all__ = ["AudioMixer"] diff --git a/services/render_engine/renderer/audio/mixer.py b/services/render_engine/renderer/audio/mixer.py new file mode 100644 index 0000000000000000000000000000000000000000..fafd9f5cf6eb7e2b608c4d9fe39717a3474be627 --- /dev/null +++ b/services/render_engine/renderer/audio/mixer.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from pathlib import Path + +from renderer.ffmpeg.command import FFmpegCommand +from renderer.ffmpeg.runner import FFmpegRunner + + +class AudioMixer: + def __init__(self, runner: FFmpegRunner) -> None: + self.runner = runner + + def ducking_filter(self) -> str: + # sidechaincompress lowers background music while voiceover is active. + return ( + "[1:a]volume=0.316[music_quiet];" + "[music_quiet][2:a]sidechaincompress=threshold=0.02:ratio=8:attack=30:release=600[ducked];" + "[2:a]volume=1.0[voice];" + "[ducked][voice]amix=inputs=2:duration=first:dropout_transition=2[aout]" + ) + + def mix( + self, + video: Path, + music: str | None, + voiceover: str | None, + output: Path, + normalize: bool = False, + *, + duration: float | None = None, + music_volume: float = 0.316, + music_fade_in: float = 0.0, + music_fade_out: float = 0.0, + music_loop: bool = True, + music_start: float = 0.0, + music_ducking: bool = True, + voice_volume: float = 1.0, + ) -> Path: + audio_tail = ",loudnorm=I=-16:TP=-1.5:LRA=11" if normalize else "" + if not music and not voiceover: + command = FFmpegCommand().add("-hide_banner").input(video).add("-c", "copy").overwrite().add(output).build() + self.runner.run(command) + return output + if voiceover and music: + music_filter = _music_filter( + music_volume, + fade_in=music_fade_in, + fade_out=music_fade_out, + duration=duration, + ) + if music_ducking: + ducking_filter = ( + f"[1:a]{music_filter}[music];" + "[music][2:a]sidechaincompress=threshold=0.02:ratio=8:attack=30:release=600[ducked];" + f"[2:a]volume={voice_volume}[voice];" + "[ducked][voice]amix=inputs=2:duration=first:dropout_transition=2[aout]" + ) + else: + ducking_filter = ( + f"[1:a]{music_filter}[music];" + f"[2:a]volume={voice_volume}[voice];" + "[music][voice]amix=inputs=2:duration=first:dropout_transition=2[aout]" + ) + if normalize: + ducking_filter += ";[aout]loudnorm=I=-16:TP=-1.5:LRA=11[anorm]" + audio_map = "[anorm]" + else: + audio_map = "[aout]" + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(video) + .input(music, **_music_input_options(music_loop, music_start)) + .input(voiceover) + .add("-filter_complex", ducking_filter) + .add("-map", "0:v", "-map", audio_map, "-c:v", "copy", "-c:a", "aac", "-shortest") + .overwrite() + .add(output) + .build() + ) + else: + audio = voiceover or music + input_options = {} if voiceover else _music_input_options(music_loop, music_start) + volume = voice_volume if voiceover else music_volume + audio_filter = f"volume={volume}" + if music and not voiceover: + audio_filter = _music_filter(music_volume, fade_in=music_fade_in, fade_out=music_fade_out, duration=duration) + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(video) + .input(audio, **input_options) + .add("-filter_complex", f"[1:a]{audio_filter}{audio_tail}[aout]") + .add("-map", "0:v", "-map", "[aout]", "-c:v", "copy", "-c:a", "aac", "-shortest") + .overwrite() + .add(output) + .build() + ) + self.runner.run(command) + return output + + +def _music_input_options(loop: bool, start: float) -> dict[str, object]: + options: dict[str, object] = {} + if loop: + options["stream_loop"] = -1 + if start > 0: + options["ss"] = start + return options + + +def _music_filter(volume: float, *, fade_in: float, fade_out: float, duration: float | None) -> str: + filters = [f"volume={volume}"] + if fade_in > 0: + filters.append(f"afade=t=in:st=0:d={fade_in}") + if fade_out > 0 and duration and duration > fade_out: + filters.append(f"afade=t=out:st={max(0.0, duration - fade_out):.3f}:d={fade_out}") + return ",".join(filters) diff --git a/services/render_engine/renderer/core/__init__.py b/services/render_engine/renderer/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..956aa7be3446b1e2c540d482ea1e95ed95cb3cea --- /dev/null +++ b/services/render_engine/renderer/core/__init__.py @@ -0,0 +1 @@ +"""Core configuration, models, and orchestration.""" diff --git a/services/render_engine/renderer/core/config.py b/services/render_engine/renderer/core/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c0abd0a5f343bc6bbb53f07268b33f10a49c21d0 --- /dev/null +++ b/services/render_engine/renderer/core/config.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_ROOT = Path.cwd() + + +@dataclass(frozen=True) +class Settings: + """Runtime settings tuned for small CPU-only Hugging Face Spaces.""" + + base_dir: Path = Path(os.getenv("AVA2LON_BASE_DIR", os.getenv("BASYX_BASE_DIR", str(DEFAULT_ROOT)))) + temp_dir: Path = Path(os.getenv("TEMP_DIR", str(DEFAULT_ROOT / "temp"))) + exports_dir: Path = Path(os.getenv("EXPORTS_DIR", str(DEFAULT_ROOT / "exports"))) + jobs_dir: Path = Path(os.getenv("JOBS_DIR", str(DEFAULT_ROOT / "jobs"))) + storage_dir: Path = Path(os.getenv("STORAGE_DIR", str(DEFAULT_ROOT / "storage"))) + metadata_cache: Path = Path(os.getenv("METADATA_CACHE", str(DEFAULT_ROOT / "temp" / "metadata_cache.json"))) + signing_secret: str = os.getenv("AVA2LON_SIGNING_SECRET", os.getenv("BASYX_SIGNING_SECRET", "")) + api_key: str = os.getenv("AVA2LON_API_KEY", os.getenv("BASYX_API_KEY", "")) + font_path: Path = Path(os.getenv("FONT_PATH", "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf")) + output_width: int = int(os.getenv("OUTPUT_WIDTH", "1080")) + output_height: int = int(os.getenv("OUTPUT_HEIGHT", "1920")) + output_fps: int = int(os.getenv("OUTPUT_FPS", "30")) + ffmpeg_timeout_seconds: int = int(os.getenv("FFMPEG_TIMEOUT_SECONDS", "900")) + download_timeout_seconds: int = int(os.getenv("DOWNLOAD_TIMEOUT_SECONDS", "60")) + max_download_bytes: int = int(os.getenv("MAX_DOWNLOAD_BYTES", str(500 * 1024 * 1024))) + allow_private_asset_urls: bool = os.getenv("ALLOW_PRIVATE_ASSET_URLS", "false").lower() == "true" + whisper_model_size: str = os.getenv("WHISPER_MODEL_SIZE", "tiny") + whisper_compute_type: str = os.getenv("WHISPER_COMPUTE_TYPE", "int8") + whisper_device: str = os.getenv("WHISPER_DEVICE", "cpu") + whisper_model_dir: Path = Path(os.getenv("WHISPER_MODEL_DIR", str(DEFAULT_ROOT / "models"))) + max_retries: int = int(os.getenv("MAX_RETRIES", "3")) + max_workers: int = int(os.getenv("MAX_RENDER_WORKERS", "1")) + job_retention_seconds: int = int(os.getenv("JOB_RETENTION_SECONDS", str(24 * 3600))) + crf: int = int(os.getenv("OUTPUT_CRF", "23")) + preset: str = os.getenv("OUTPUT_PRESET", "veryfast") + + def ensure_dirs(self) -> None: + for directory in ( + self.temp_dir, + self.exports_dir, + self.jobs_dir, + self.storage_dir, + self.metadata_cache.parent, + self.whisper_model_dir, + ): + directory.mkdir(parents=True, exist_ok=True) diff --git a/services/render_engine/renderer/core/ingest.py b/services/render_engine/renderer/core/ingest.py new file mode 100644 index 0000000000000000000000000000000000000000..9ff4018a3220f5350ce053c1dcdda66a9dfd7a58 --- /dev/null +++ b/services/render_engine/renderer/core/ingest.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import ipaddress +import mimetypes +import shutil +import socket +from dataclasses import replace +from pathlib import Path +from urllib.parse import unquote, urlparse +from urllib.request import Request, urlopen + +from renderer.core.config import Settings +from renderer.core.models import AIReelsRequest, RenderRequest, Scene +from renderer.core.utils import safe_filename + + +class IngestError(ValueError): + pass + + +class AssetIngestor: + """Resolve local paths and remote URLs into job-local files.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def resolve_render_request(self, request: RenderRequest, workdir: Path) -> RenderRequest: + return replace( + request, + scenes=[ + replace(scene, media=str(self.resolve(scene.media, workdir / "inputs", f"scene_{idx:03d}"))) + for idx, scene in enumerate(request.scenes) + ], + voiceover=self.resolve_optional(request.voiceover, workdir / "inputs", "voiceover"), + background_music=self.resolve_optional(request.background_music, workdir / "inputs", "music"), + ) + + def resolve_ai_reels_request(self, request: AIReelsRequest, workdir: Path) -> AIReelsRequest: + return replace( + request, + voiceover=str(self.resolve(request.voiceover, workdir / "inputs", "voiceover")), + assets=[str(self.resolve(asset, workdir / "inputs", f"asset_{idx:03d}")) for idx, asset in enumerate(request.assets)], + background_music=self.resolve_optional(request.background_music, workdir / "inputs", "music"), + ) + + def resolve_optional(self, value: str | None, directory: Path, stem: str) -> str | None: + if not value: + return None + return str(self.resolve(value, directory, stem)) + + def resolve(self, value: str, directory: Path, stem: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + if is_remote_url(value): + return self.download(value, directory, stem) + path = Path(value) + if not path.exists(): + raise IngestError(f"Asset does not exist: {value}") + return path + + def download(self, url: str, directory: Path, stem: str) -> Path: + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise IngestError("Only http and https asset URLs are supported") + if not self.settings.allow_private_asset_urls: + _reject_private_host(parsed.hostname) + request = Request(url, headers={"User-Agent": "basyx-ffmpeg-renderer/1.0"}) + with urlopen(request, timeout=self.settings.download_timeout_seconds) as response: + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > self.settings.max_download_bytes: + raise IngestError("Remote asset exceeds MAX_DOWNLOAD_BYTES") + suffix = _suffix_from_response(url, response.headers.get("Content-Type")) + target = directory / safe_filename(f"{stem}{suffix}") + total = 0 + with target.open("wb") as output: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > self.settings.max_download_bytes: + target.unlink(missing_ok=True) + raise IngestError("Remote asset exceeds MAX_DOWNLOAD_BYTES") + output.write(chunk) + return target + + +def stage_upload(source: Path, uploads_dir: Path, filename: str) -> Path: + uploads_dir.mkdir(parents=True, exist_ok=True) + target = uploads_dir / safe_filename(filename) + if source.resolve() != target.resolve(): + shutil.copy2(source, target) + return target + + +def is_remote_url(value: str) -> bool: + return urlparse(value).scheme in {"http", "https"} + + +def _suffix_from_response(url: str, content_type: str | None) -> str: + path_suffix = Path(unquote(urlparse(url).path)).suffix + if path_suffix: + return path_suffix[:16] + if content_type: + guessed = mimetypes.guess_extension(content_type.split(";", 1)[0].strip()) + if guessed: + return guessed + return ".bin" + + +def _reject_private_host(hostname: str) -> None: + try: + addresses = socket.getaddrinfo(hostname, None) + except socket.gaierror as exc: + raise IngestError(f"Could not resolve host: {hostname}") from exc + for address in addresses: + ip = ipaddress.ip_address(address[4][0]) + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast: + raise IngestError("Private, loopback, link-local, and multicast asset hosts are not allowed") diff --git a/services/render_engine/renderer/core/models.py b/services/render_engine/renderer/core/models.py new file mode 100644 index 0000000000000000000000000000000000000000..77e96f5c469449933c5dbf091666141ba1e813d6 --- /dev/null +++ b/services/render_engine/renderer/core/models.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +JobState = Literal["PENDING", "RUNNING", "FAILED", "COMPLETED", "CANCEL_REQUESTED", "CANCELLED"] + + +@dataclass +class Scene: + start: float + duration: float + media: str + caption: str = "" + transition: str = "fade" + background: str = "blur" + layout: str = "fill" + effect: str | None = None + + +@dataclass +class RenderRequest: + scenes: list[Scene] + template: str = "tiktok_classic" + preset: str | None = None + creative_style: str | None = None + platform: str | None = None + output_name: str = "render.mp4" + voiceover: str | None = None + background_music: str | None = None + music_volume: float = 0.316 + music_fade_in: float = 0.0 + music_fade_out: float = 0.0 + music_loop: bool = True + music_start: float = 0.0 + music_ducking: bool = True + voice_volume: float = 1.0 + subtitle_format: Literal["srt", "ass"] = "ass" + auto_subtitles: bool = False + subtitle_language: str | None = None + whisper_model_size: str | None = None + preview: bool = False + audio_normalize: bool = False + watermark: str | None = None + watermark_position: str = "bottom-right" + intro: str | None = None + outro: str | None = None + callback_url: str | None = None + export_target: str | None = None + priority: int = 0 + scheduled_at: float | None = None + normalize: bool = True + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class AIReelsRequest: + script: str + voiceover: str + assets: list[str] + template: str = "tiktok_classic" + creative_style: str | None = None + platform: str | None = None + output_name: str = "ai_reel.mp4" + background_music: str | None = None + music_volume: float = 0.316 + music_fade_in: float = 0.0 + music_fade_out: float = 0.0 + music_loop: bool = True + music_start: float = 0.0 + music_ducking: bool = True + voice_volume: float = 1.0 + + +@dataclass +class RenderResult: + output_path: Path + commands: list[list[str]] + metrics: dict[str, Any] + logs: list[str] = field(default_factory=list) + + +@dataclass +class TaskResult: + output_path: Path | None = None + commands: list[list[str]] = field(default_factory=list) + metrics: dict[str, Any] = field(default_factory=dict) + logs: list[str] = field(default_factory=list) + + +@dataclass +class AssetMetadata: + path: str + mime_type: str + size_bytes: int + mtime: float + duration: float = 0.0 + width: int | None = None + height: int | None = None + fps: float | None = None + bitrate: int | None = None + video_codec: str | None = None + audio_codec: str | None = None + has_audio: bool = False + streams: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass +class JobRecord: + job_id: str + state: JobState + created_at: float + updated_at: float + output_path: str | None = None + download_token: str | None = None + callback_url: str | None = None + export_target: str | None = None + export_path: str | None = None + failure_reason: str | None = None + commands: list[list[str]] = field(default_factory=list) + logs: list[str] = field(default_factory=list) + metrics: dict[str, Any] = field(default_factory=dict) diff --git a/services/render_engine/renderer/core/render_engine.py b/services/render_engine/renderer/core/render_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..cd320d2299eedbd12578a075a7d6fea9ccc9450c --- /dev/null +++ b/services/render_engine/renderer/core/render_engine.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import mimetypes +import shutil +import time +from pathlib import Path + +from renderer.audio import AudioMixer +from renderer.core.config import Settings +from renderer.core.ingest import AssetIngestor +from renderer.core.models import AIReelsRequest, RenderRequest, RenderResult +from renderer.core.utils import cleanup_directory, temp_workdir +from renderer.exports import ExportManager +from renderer.ffmpeg.assets import AssetProbe +from renderer.ffmpeg.command import FFmpegCommand +from renderer.ffmpeg.normalize import Normalizer +from renderer.ffmpeg.runner import FFmpegRunner +from renderer.scenes import Timeline +from renderer.subtitles import SubtitleGenerator +from renderer.templates import PlatformProfile, get_creative_style, get_platform_profile, scene_effect_filter +from renderer.transcription import WhisperTranscriber +from renderer.transitions import TransitionBuilder + + +class RenderEngine: + def __init__(self, settings: Settings | None = None, log=None) -> None: + self.settings = settings or Settings() + self.settings.ensure_dirs() + self._commands: list[list[str]] = [] + self._logs: list[str] = [] + self.runner = FFmpegRunner(self.settings.ffmpeg_timeout_seconds, log=log, on_command=self._record_command) + self.ingest = AssetIngestor(self.settings) + self.assets = AssetProbe(self.settings.metadata_cache) + self.normalizer = Normalizer(self.settings, self.runner) + self.subtitles = SubtitleGenerator() + self.transcriber = WhisperTranscriber(self.settings) + self.transitions = TransitionBuilder() + self.audio = AudioMixer(self.runner) + self.exports = ExportManager(self.settings.exports_dir) + + def render(self, request: RenderRequest, job_id: str) -> RenderResult: + self._commands = [] + self._logs = [] + started = time.time() + timeline = Timeline(request.scenes) + with temp_workdir(self.settings.temp_dir, job_id) as work: + workdir = Path(work) + resolved = self.ingest.resolve_render_request(request, workdir) + timeline = Timeline(resolved.scenes) + prepared = self._prepare_scene_media(resolved, workdir) + subtitles = self._write_subtitles(resolved, timeline, workdir) + video = self._compose_video(prepared, resolved, subtitles, workdir) + mixed = self.audio.mix( + video, + resolved.background_music, + resolved.voiceover, + workdir / "mixed.mp4", + normalize=resolved.audio_normalize, + duration=timeline.total_duration, + music_volume=resolved.music_volume, + music_fade_in=resolved.music_fade_in, + music_fade_out=resolved.music_fade_out, + music_loop=resolved.music_loop, + music_start=resolved.music_start, + music_ducking=resolved.music_ducking, + voice_volume=resolved.voice_volume, + ) + mixed = self._apply_watermark(mixed, resolved, workdir) + optimized = self._optimize_for_platform(mixed, resolved, workdir) + output = self.exports.save(optimized, job_id, request.output_name) + cleanup_directory(workdir, keep={mixed}) + profile = self._profile(request) + metrics = { + "render_time_seconds": round(time.time() - started, 3), + "output_size_bytes": output.stat().st_size, + "scene_count": len(request.scenes), + "creative_style": request.creative_style or request.metadata.get("creative_style"), + "scene_effects": [scene.effect for scene in request.scenes if scene.effect], + "platform": profile.metadata(), + } + return RenderResult(output_path=output, commands=list(self._commands), metrics=metrics, logs=list(self._logs)) + + def ai_reels(self, request: AIReelsRequest, job_id: str) -> RenderResult: + if not request.voiceover: + raise ValueError("AI reels v1 requires a provided voiceover path; TTS is pluggable but not bundled.") + if not request.assets: + raise ValueError("AI reels rendering requires at least one visual asset.") + with temp_workdir(self.settings.temp_dir, job_id) as work: + workdir = Path(work) + resolved = self.ingest.resolve_ai_reels_request(request, workdir) + style = get_creative_style(resolved.creative_style) + voice_meta = self.assets.probe(resolved.voiceover) + duration = max(voice_meta.duration, len(resolved.script.split()) * 0.35, style.scene_duration * len(resolved.assets), 3.0) + per_scene = duration / max(1, len(resolved.assets)) + captions = _split_script(resolved.script, len(resolved.assets)) + scenes = [ + { + "start": round(idx * per_scene, 3), + "duration": round(per_scene, 3), + "media": asset, + "caption": captions[idx] if idx < len(captions) else "", + "transition": style.transition_sequence[idx % len(style.transition_sequence)], + "effect": style.scene_effect_sequence[idx % len(style.scene_effect_sequence)], + } + for idx, asset in enumerate(resolved.assets) + ] + render_request = RenderRequest( + scenes=Timeline.request_from_payload({"scenes": scenes}).scenes, + template=resolved.template, + platform=resolved.platform, + output_name=resolved.output_name, + voiceover=resolved.voiceover, + background_music=resolved.background_music, + music_volume=resolved.music_volume, + music_fade_in=resolved.music_fade_in, + music_fade_out=resolved.music_fade_out, + music_loop=resolved.music_loop, + music_start=resolved.music_start, + music_ducking=resolved.music_ducking, + voice_volume=resolved.voice_volume, + creative_style=resolved.creative_style, + metadata={"creative_style": style.key, "creative_style_label": style.label}, + ) + return self._render_resolved(render_request, job_id, workdir) + + def inspect_asset(self, path: str | Path) -> dict: + return self.assets.probe(path).__dict__ + + def transcribe( + self, + audio_path: str | Path, + *, + model_size: str | None = None, + language: str | None = None, + task: str = "transcribe", + beam_size: int = 5, + vad_filter: bool = True, + word_timestamps: bool = True, + ) -> dict: + return self.transcriber.transcribe( + audio_path, + model_size=model_size, + language=language, + task=task, + beam_size=beam_size, + vad_filter=vad_filter, + word_timestamps=word_timestamps, + ).as_dict() + + def _render_resolved(self, request: RenderRequest, job_id: str, workdir: Path) -> RenderResult: + self._commands = [] + self._logs = [] + started = time.time() + timeline = Timeline(request.scenes) + prepared = self._prepare_scene_media(request, workdir) + subtitles = self._write_subtitles(request, timeline, workdir) + video = self._compose_video(prepared, request, subtitles, workdir) + mixed = self.audio.mix( + video, + request.background_music, + request.voiceover, + workdir / "mixed.mp4", + normalize=request.audio_normalize, + duration=timeline.total_duration, + music_volume=request.music_volume, + music_fade_in=request.music_fade_in, + music_fade_out=request.music_fade_out, + music_loop=request.music_loop, + music_start=request.music_start, + music_ducking=request.music_ducking, + voice_volume=request.voice_volume, + ) + mixed = self._apply_watermark(mixed, request, workdir) + optimized = self._optimize_for_platform(mixed, request, workdir) + output = self.exports.save(optimized, job_id, request.output_name) + profile = self._profile(request) + metrics = { + "render_time_seconds": round(time.time() - started, 3), + "output_size_bytes": output.stat().st_size, + "scene_count": len(request.scenes), + "creative_style": request.creative_style or request.metadata.get("creative_style"), + "scene_effects": [scene.effect for scene in request.scenes if scene.effect], + "platform": profile.metadata(), + } + return RenderResult(output_path=output, commands=list(self._commands), metrics=metrics, logs=list(self._logs)) + + def _prepare_scene_media(self, request: RenderRequest, workdir: Path) -> list[Path]: + prepared: list[Path] = [] + profile = self._profile(request) + for idx, scene in enumerate(request.scenes): + source = Path(scene.media) + metadata = self.assets.probe(source) + target = workdir / f"scene_{idx:03d}.mp4" + mime_type = metadata.mime_type or mimetypes.guess_type(str(source))[0] or "" + if mime_type.startswith("image/") or source.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".gif"}: + self.normalizer.image_to_video(source, target, scene.duration, profile=profile) + elif request.normalize and self.normalizer.needs_normalization(metadata, profile=profile): + self.normalizer.normalize(source, target, scene.duration, profile=profile) + else: + shutil.copy2(source, target) + if request.preview: + preview = workdir / f"scene_{idx:03d}_preview.mp4" + self._scale_preview(target, preview) + target = preview + effected = self._apply_scene_effect(target, scene.effect, workdir / f"scene_{idx:03d}_effect.mp4") + if effected != target: + target = effected + prepared.append(target) + return prepared + + def _write_subtitles(self, request: RenderRequest, timeline: Timeline, workdir: Path) -> Path | None: + if request.auto_subtitles and request.voiceover: + transcript = self.transcriber.transcribe( + request.voiceover, + model_size=request.whisper_model_size, + language=request.subtitle_language, + word_timestamps=True, + ) + events = transcript.subtitle_events(prefer_words=True) + else: + events = self.subtitles.from_scenes(request.scenes, timeline.total_duration) + if not events: + return None + if request.subtitle_format == "srt": + return self.subtitles.write_srt(events, workdir / "captions.srt") + return self.subtitles.write_ass(events, workdir / "captions.ass", request.template) + + def _compose_video(self, scenes: list[Path], request: RenderRequest, subtitle_path: Path | None, workdir: Path) -> Path: + if len(scenes) == 1: + composed = workdir / "composed.mp4" + shutil.copy2(scenes[0], composed) + else: + composed = workdir / "composed.mp4" + durations = [scene.duration for scene in request.scenes] + transitions = [scene.transition for scene in request.scenes] + filter_graph, final_stream = self.transitions.xfade_chain(len(scenes), durations, transitions) + cmd = FFmpegCommand().add("-hide_banner") + for scene in scenes: + cmd.input(scene) + cmd.add("-filter_complex", filter_graph) + cmd.add("-map", final_stream, "-an", "-c:v", "libx264", "-preset", self.settings.preset, "-crf", self.settings.crf) + command = cmd.overwrite().add(composed).build() + self._run(command) + if subtitle_path: + subtitled = workdir / "subtitled.mp4" + escaped = _ffmpeg_subtitle_path(subtitle_path) + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(composed) + .add("-vf", f"subtitles='{escaped}'", "-c:a", "copy", "-c:v", "libx264", "-preset", self.settings.preset, "-crf", self.settings.crf) + .overwrite() + .add(subtitled) + .build() + ) + self._run(command) + return subtitled + return composed + + def _apply_watermark(self, video: Path, request: RenderRequest, workdir: Path) -> Path: + if not request.watermark: + return video + watermark = Path(request.watermark) + if not watermark.exists(): + self._logs.append(f"Watermark skipped; file not found: {watermark}") + return video + output = workdir / "watermarked.mp4" + position = { + "top-left": "20:20", + "top-right": "W-w-20:20", + "bottom-left": "20:H-h-20", + "bottom-right": "W-w-20:H-h-20", + "center": "(W-w)/2:(H-h)/2", + }.get(request.watermark_position, "W-w-20:H-h-20") + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(video) + .input(watermark) + .add("-filter_complex", f"[1:v]scale=iw*0.22:-1[wm];[0:v][wm]overlay={position}") + .add("-c:a", "copy", "-c:v", "libx264", "-preset", self.settings.preset, "-crf", self.settings.crf) + .overwrite() + .add(output) + .build() + ) + self._run(command) + return output + + def _optimize_for_platform(self, video: Path, request: RenderRequest, workdir: Path) -> Path: + profile = self._profile(request) + output = workdir / "platform_optimized.mp4" + vf = ( + f"scale={profile.width}:{profile.height}:force_original_aspect_ratio=increase," + f"crop={profile.width}:{profile.height},fps={profile.fps},format=yuv420p" + ) + command_builder = ( + FFmpegCommand() + .add("-hide_banner") + .input(video) + .add( + "-vf", + vf, + "-c:v", + profile.video_codec, + "-profile:v", + "high", + "-pix_fmt", + "yuv420p", + "-preset", + self.settings.preset, + "-crf", + profile.crf, + "-r", + profile.fps, + "-g", + max(1, profile.fps * 2), + ) + ) + if profile.maxrate: + command_builder.add("-maxrate", profile.maxrate) + if profile.bufsize: + command_builder.add("-bufsize", profile.bufsize) + command = ( + command_builder.add( + "-c:a", + profile.audio_codec, + "-b:a", + profile.audio_bitrate, + "-ar", + profile.audio_sample_rate, + "-ac", + "2", + "-movflags", + "+faststart", + "-shortest", + ) + .overwrite() + .add(output) + .build() + ) + self._run(command) + return output + + def _profile(self, request: RenderRequest) -> PlatformProfile: + profile_key = request.platform or request.metadata.get("platform") or request.metadata.get("target_platform") + return get_platform_profile(str(profile_key) if profile_key else None) + + + def _scale_preview(self, source: Path, output: Path) -> None: + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(source) + .add("-vf", "scale=540:960:force_original_aspect_ratio=decrease,pad=540:960:(ow-iw)/2:(oh-ih)/2") + .add("-c:v", "libx264", "-preset", "ultrafast", "-crf", "30", "-c:a", "aac") + .overwrite() + .add(output) + .build() + ) + self._run(command) + + def _apply_scene_effect(self, source: Path, effect: str | None, output: Path) -> Path: + vf = scene_effect_filter(effect) + if not vf: + return source + command = ( + FFmpegCommand() + .add("-hide_banner") + .input(source) + .add("-vf", vf, "-c:v", "libx264", "-preset", self.settings.preset, "-crf", self.settings.crf, "-c:a", "copy") + .overwrite() + .add(output) + .build() + ) + self._run(command) + self._logs.append(f"Applied scene effect '{effect}' to {source.name}") + return output + + def _run(self, command: list[str]) -> None: + result = self.runner.run(command) + if result.stderr: + self._logs.append(result.stderr[-4000:]) + + def _record_command(self, command: list[str]) -> None: + self._commands.append(command) + + +def _split_script(script: str, chunks: int) -> list[str]: + words = script.split() + if chunks <= 0: + return [] + size = max(1, round(len(words) / chunks)) + return [" ".join(words[i : i + size]) for i in range(0, len(words), size)][:chunks] + + +def _ffmpeg_subtitle_path(path: Path) -> str: + return str(path).replace("\\", "/").replace(":", r"\:") diff --git a/services/render_engine/renderer/core/security.py b/services/render_engine/renderer/core/security.py new file mode 100644 index 0000000000000000000000000000000000000000..6e73b2134aa4012eeb8b4b411ec1a59993c4aaaa --- /dev/null +++ b/services/render_engine/renderer/core/security.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import hmac +import secrets + + +def create_download_token(secret: str, job_id: str) -> str: + nonce = secrets.token_urlsafe(18) + signature = hmac.digest(secret.encode("utf-8"), f"{job_id}:{nonce}".encode("utf-8"), "sha256").hex() + return f"{nonce}.{signature}" + + +def verify_download_token(secret: str, job_id: str, token: str | None) -> bool: + if not token or "." not in token: + return False + nonce, signature = token.split(".", 1) + expected = hmac.digest(secret.encode("utf-8"), f"{job_id}:{nonce}".encode("utf-8"), "sha256").hex() + return hmac.compare_digest(signature, expected) diff --git a/services/render_engine/renderer/core/utils.py b/services/render_engine/renderer/core/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..47dfcc531a3c8a7f75ce353ae4db0b786df3f7e2 --- /dev/null +++ b/services/render_engine/renderer/core/utils.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +import re +import shutil +import tempfile +import time +import uuid +from dataclasses import asdict, is_dataclass +from pathlib import Path +from typing import Any + + +def new_id(prefix: str = "job") -> str: + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +def safe_filename(name: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._") + return cleaned or "render.mp4" + + +def write_json(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + serializable = asdict(data) if is_dataclass(data) else data + path.write_text(json.dumps(serializable, indent=2, default=str), encoding="utf-8") + + +def read_json(path: Path, default: Any) -> Any: + if not path.exists(): + return default + return json.loads(path.read_text(encoding="utf-8")) + + +def now() -> float: + return time.time() + + +def temp_workdir(root: Path, prefix: str) -> tempfile.TemporaryDirectory[str]: + root.mkdir(parents=True, exist_ok=True) + return tempfile.TemporaryDirectory(prefix=f"{prefix}_", dir=str(root)) + + +def cleanup_directory(path: Path, keep: set[Path] | None = None) -> None: + keep = {p.resolve() for p in (keep or set())} + if not path.exists(): + return + for child in path.iterdir(): + if child.resolve() in keep: + continue + if child.is_dir(): + shutil.rmtree(child, ignore_errors=True) + else: + child.unlink(missing_ok=True) diff --git a/services/render_engine/renderer/exports/__init__.py b/services/render_engine/renderer/exports/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5b7a38a115d13fc7f5dcff93549f99f03b97e233 --- /dev/null +++ b/services/render_engine/renderer/exports/__init__.py @@ -0,0 +1,3 @@ +from renderer.exports.manager import ExportManager + +__all__ = ["ExportManager"] diff --git a/services/render_engine/renderer/exports/manager.py b/services/render_engine/renderer/exports/manager.py new file mode 100644 index 0000000000000000000000000000000000000000..ff9197f6342a28fb25fba32fe1a4a24d3e976b65 --- /dev/null +++ b/services/render_engine/renderer/exports/manager.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +from renderer.core.utils import safe_filename + + +class ExportManager: + def __init__(self, exports_dir: Path) -> None: + self.exports_dir = exports_dir + self.exports_dir.mkdir(parents=True, exist_ok=True) + + def save(self, source: Path, job_id: str, output_name: str) -> Path: + filename = safe_filename(output_name) + if not filename.lower().endswith(".mp4"): + filename += ".mp4" + target = self.exports_dir / f"{job_id}_{filename}" + shutil.copy2(source, target) + return target diff --git a/services/render_engine/renderer/ffmpeg/__init__.py b/services/render_engine/renderer/ffmpeg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c855700c271c1c8b2ad80895f8192ec3df694d9c --- /dev/null +++ b/services/render_engine/renderer/ffmpeg/__init__.py @@ -0,0 +1 @@ +"""FFmpeg subprocess and filter graph helpers.""" diff --git a/services/render_engine/renderer/ffmpeg/assets.py b/services/render_engine/renderer/ffmpeg/assets.py new file mode 100644 index 0000000000000000000000000000000000000000..2a42bdc65396fe67861b108f4e0af3fee28c69c7 --- /dev/null +++ b/services/render_engine/renderer/ffmpeg/assets.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json +import mimetypes +import subprocess +from dataclasses import asdict +from pathlib import Path + +from renderer.core.models import AssetMetadata +from renderer.core.utils import read_json, write_json + + +class AssetProbe: + def __init__(self, cache_path: Path) -> None: + self.cache_path = cache_path + self.cache: dict[str, dict] = read_json(cache_path, {}) + + def probe(self, path: str | Path) -> AssetMetadata: + media = Path(path) + stat = media.stat() + key = str(media.resolve()) + cached = self.cache.get(key) + if cached and cached.get("size_bytes") == stat.st_size and cached.get("mtime") == stat.st_mtime: + return AssetMetadata(**cached) + + raw = self._ffprobe(media) + streams = raw.get("streams", []) + fmt = raw.get("format", {}) + video_stream = next((s for s in streams if s.get("codec_type") == "video"), {}) + audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), {}) + metadata = AssetMetadata( + path=str(media), + mime_type=mimetypes.guess_type(str(media))[0] or "application/octet-stream", + size_bytes=stat.st_size, + mtime=stat.st_mtime, + duration=float(fmt.get("duration") or video_stream.get("duration") or audio_stream.get("duration") or 0), + width=_int_or_none(video_stream.get("width")), + height=_int_or_none(video_stream.get("height")), + fps=_parse_fps(video_stream.get("avg_frame_rate") or video_stream.get("r_frame_rate")), + bitrate=_int_or_none(fmt.get("bit_rate")), + video_codec=video_stream.get("codec_name"), + audio_codec=audio_stream.get("codec_name"), + has_audio=bool(audio_stream), + streams=streams, + ) + self.cache[key] = asdict(metadata) + write_json(self.cache_path, self.cache) + return metadata + + @staticmethod + def _ffprobe(path: Path) -> dict: + command = [ + "ffprobe", + "-v", + "error", + "-show_format", + "-show_streams", + "-print_format", + "json", + str(path), + ] + result = subprocess.run(command, check=True, capture_output=True, text=True) + return json.loads(result.stdout or "{}") + + +def _parse_fps(value: str | None) -> float | None: + if not value or value == "0/0": + return None + if "/" in value: + num, den = value.split("/", 1) + den_f = float(den) + return float(num) / den_f if den_f else None + return float(value) + + +def _int_or_none(value: object) -> int | None: + if value in (None, ""): + return None + return int(value) diff --git a/services/render_engine/renderer/ffmpeg/command.py b/services/render_engine/renderer/ffmpeg/command.py new file mode 100644 index 0000000000000000000000000000000000000000..e96ee030fbfba0809d9fd7c75764e89c8501d842 --- /dev/null +++ b/services/render_engine/renderer/ffmpeg/command.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class FFmpegCommand: + args: list[str] = field(default_factory=lambda: ["ffmpeg"]) + + def add(self, *args: object) -> "FFmpegCommand": + self.args.extend(str(arg) for arg in args) + return self + + def input(self, path: str | Path, **options: object) -> "FFmpegCommand": + for key, value in options.items(): + self.add(f"-{key}", value) + self.add("-i", path) + return self + + def overwrite(self) -> "FFmpegCommand": + self.add("-y") + return self + + def build(self) -> list[str]: + return list(self.args) diff --git a/services/render_engine/renderer/ffmpeg/normalize.py b/services/render_engine/renderer/ffmpeg/normalize.py new file mode 100644 index 0000000000000000000000000000000000000000..8820774b315c61909c791c1b306bce810912836b --- /dev/null +++ b/services/render_engine/renderer/ffmpeg/normalize.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from pathlib import Path + +from renderer.core.config import Settings +from renderer.core.models import AssetMetadata +from renderer.ffmpeg.command import FFmpegCommand +from renderer.ffmpeg.runner import FFmpegRunner +from renderer.templates import PlatformProfile + + +class Normalizer: + def __init__(self, settings: Settings, runner: FFmpegRunner) -> None: + self.settings = settings + self.runner = runner + + def needs_normalization(self, metadata: AssetMetadata, profile: PlatformProfile | None = None) -> bool: + width = profile.width if profile else self.settings.output_width + height = profile.height if profile else self.settings.output_height + fps = profile.fps if profile else self.settings.output_fps + return not ( + metadata.width == width + and metadata.height == height + and round(metadata.fps or 0) == fps + and metadata.video_codec == "h264" + and (metadata.audio_codec in ("aac", None)) + ) + + def normalize(self, path: str | Path, output: Path, duration: float | None = None, profile: PlatformProfile | None = None) -> Path: + width = profile.width if profile else self.settings.output_width + height = profile.height if profile else self.settings.output_height + fps = profile.fps if profile else self.settings.output_fps + crf = profile.crf if profile else self.settings.crf + vf = ( + f"scale={width}:{height}:" + "force_original_aspect_ratio=increase," + f"crop={width}:{height}," + f"fps={fps},format=yuv420p" + ) + cmd = ( + FFmpegCommand() + .add("-hide_banner") + .input(path) + .add("-vf", vf, "-c:v", "libx264", "-preset", self.settings.preset, "-crf", crf) + .add("-c:a", "aac", "-b:a", "128k", "-movflags", "+faststart") + ) + if duration: + cmd.add("-t", duration) + command = cmd.overwrite().add(output).build() + self.runner.run(command) + return output + + def image_to_video(self, path: str | Path, output: Path, duration: float, profile: PlatformProfile | None = None) -> Path: + width = profile.width if profile else self.settings.output_width + height = profile.height if profile else self.settings.output_height + fps = profile.fps if profile else self.settings.output_fps + crf = profile.crf if profile else self.settings.crf + vf = ( + f"scale={width}:{height}:" + "force_original_aspect_ratio=increase," + f"crop={width}:{height}," + f"fps={fps},format=yuv420p" + ) + command = ( + FFmpegCommand() + .add("-hide_banner", "-loop", "1", "-t", duration) + .input(path) + .add("-vf", vf, "-an", "-c:v", "libx264", "-preset", self.settings.preset, "-crf", crf) + .overwrite() + .add(output) + .build() + ) + self.runner.run(command) + return output diff --git a/services/render_engine/renderer/ffmpeg/runner.py b/services/render_engine/renderer/ffmpeg/runner.py new file mode 100644 index 0000000000000000000000000000000000000000..710a2f87615ecfc7cb1d40769d7ffdaaa4338a08 --- /dev/null +++ b/services/render_engine/renderer/ffmpeg/runner.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +try: + import psutil +except Exception: # pragma: no cover - optional runtime dependency fallback + psutil = None + + +class FFmpegError(RuntimeError): + def __init__(self, message: str, command: list[str], stderr: str = "") -> None: + super().__init__(message) + self.command = command + self.stderr = stderr + + +@dataclass +class CommandResult: + command: list[str] + returncode: int + duration_seconds: float + stdout: str = "" + stderr: str = "" + metrics: dict[str, float | int] = field(default_factory=dict) + + +class FFmpegRunner: + def __init__( + self, + timeout_seconds: int = 900, + log: Callable[[str], None] | None = None, + on_command: Callable[[list[str]], None] | None = None, + ) -> None: + self.timeout_seconds = timeout_seconds + self.log = log or (lambda _: None) + self.on_command = on_command or (lambda _: None) + + def run(self, command: list[str], cwd: Path | None = None) -> CommandResult: + started = time.time() + self.on_command(command) + self.log("$ " + " ".join(command)) + process = subprocess.Popen( + command, + cwd=str(cwd) if cwd else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + peak_rss = 0 + cpu_percent = 0.0 + proc = psutil.Process(process.pid) if psutil else None + try: + stdout, stderr = process.communicate(timeout=self.timeout_seconds) + if proc: + try: + peak_rss = max(peak_rss, proc.memory_info().rss) + cpu_percent = proc.cpu_percent(interval=None) + except Exception: + pass + except subprocess.TimeoutExpired as exc: + self._kill_process(process) + stdout, stderr = process.communicate() + raise FFmpegError(f"FFmpeg timed out after {self.timeout_seconds}s", command, stderr) from exc + + duration = time.time() - started + result = CommandResult( + command=command, + returncode=process.returncode, + duration_seconds=duration, + stdout=stdout, + stderr=stderr, + metrics={"duration_seconds": duration, "peak_rss_bytes": peak_rss, "cpu_percent": cpu_percent}, + ) + if process.returncode != 0: + raise FFmpegError("FFmpeg failed", command, stderr) + return result + + @staticmethod + def _kill_process(process: subprocess.Popen[str]) -> None: + if psutil: + try: + parent = psutil.Process(process.pid) + for child in parent.children(recursive=True): + child.kill() + parent.kill() + return + except Exception: + pass + process.kill() diff --git a/services/render_engine/renderer/jobs/__init__.py b/services/render_engine/renderer/jobs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..54783309519e3cd34c2e565657b929be1927000e --- /dev/null +++ b/services/render_engine/renderer/jobs/__init__.py @@ -0,0 +1,3 @@ +from renderer.jobs.manager import JobManager + +__all__ = ["JobManager"] diff --git a/services/render_engine/renderer/jobs/manager.py b/services/render_engine/renderer/jobs/manager.py new file mode 100644 index 0000000000000000000000000000000000000000..671730aed8438dada68904f8ab281d2468b9541e --- /dev/null +++ b/services/render_engine/renderer/jobs/manager.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import concurrent.futures +import http.client +import json +import shutil +import threading +import urllib.request +from urllib.parse import urlparse +from dataclasses import asdict +from pathlib import Path +from typing import Callable + +from renderer.core.config import Settings +from renderer.core.models import AIReelsRequest, JobRecord, RenderRequest +from renderer.core.render_engine import RenderEngine +from renderer.core.security import create_download_token +from renderer.core.utils import new_id, now, read_json, write_json + + +class JobManager: + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self.settings.ensure_dirs() + self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.settings.max_workers) + self.lock = threading.Lock() + + def submit_render(self, request: RenderRequest) -> str: + return self._submit( + lambda job_id, log: RenderEngine(self.settings, log=log).render(request, job_id), + callback_url=request.callback_url, + export_target=request.export_target, + ) + + def submit_ai_reels(self, request: AIReelsRequest) -> str: + return self._submit(lambda job_id, log: RenderEngine(self.settings, log=log).ai_reels(request, job_id)) + + def submit_task( + self, + handler: Callable[[str, Callable[[str], None]], object], + *, + callback_url: str | None = None, + export_target: str | None = None, + ) -> str: + return self._submit(handler, callback_url=callback_url, export_target=export_target) + + def submit_batch(self, requests: list[RenderRequest]) -> list[str]: + job_ids: list[str] = [] + for request in requests: + job_ids.append(self.submit_render(request)) + return job_ids + + def get(self, job_id: str) -> JobRecord: + data = read_json(self._record_path(job_id), None) + if data is None: + raise KeyError(job_id) + return JobRecord(**data) + + def cancel(self, job_id: str) -> JobRecord: + record = self.get(job_id) + if record.state == "PENDING": + self._update(job_id, state="CANCELLED", failure_reason="Cancelled before execution") + elif record.state == "RUNNING": + self._update(job_id, state="CANCEL_REQUESTED", failure_reason="Cancellation requested") + return self.get(job_id) + + def cleanup(self, older_than_seconds: int | None = None) -> dict[str, int]: + cutoff = now() - (older_than_seconds or self.settings.job_retention_seconds) + removed_jobs = 0 + removed_exports = 0 + for path in self.settings.jobs_dir.glob("*.json"): + record = JobRecord(**read_json(path, {})) + if record.updated_at >= cutoff or record.state in {"PENDING", "RUNNING", "CANCEL_REQUESTED"}: + continue + if record.output_path: + output = Path(record.output_path) + if output.exists(): + output.unlink() + removed_exports += 1 + path.unlink(missing_ok=True) + removed_jobs += 1 + uploads = self.settings.temp_dir / "uploads" + if uploads.exists(): + shutil.rmtree(uploads, ignore_errors=True) + return {"removed_jobs": removed_jobs, "removed_exports": removed_exports} + + def summary(self) -> dict: + records: list[JobRecord] = [] + for path in self.settings.jobs_dir.glob("*.json"): + try: + records.append(JobRecord(**read_json(path, {}))) + except Exception: + continue + state_counts: dict[str, int] = {} + for record in records: + state_counts[record.state] = state_counts.get(record.state, 0) + 1 + active = [ + { + "job_id": record.job_id, + "state": record.state, + "created_at": record.created_at, + "updated_at": record.updated_at, + "metrics": record.metrics, + } + for record in sorted(records, key=lambda item: item.updated_at, reverse=True) + if record.state in {"PENDING", "RUNNING", "CANCEL_REQUESTED"} + ] + return { + "total_jobs": len(records), + "state_counts": state_counts, + "active_jobs": active, + "max_workers": self.settings.max_workers, + } + + def _submit( + self, + handler: Callable[[str, Callable[[str], None]], object], + callback_url: str | None = None, + export_target: str | None = None, + ) -> str: + job_id = new_id() + record = JobRecord( + job_id=job_id, + state="PENDING", + created_at=now(), + updated_at=now(), + download_token=create_download_token(self.settings.signing_secret, job_id), + callback_url=callback_url, + export_target=export_target, + ) + self._save(record) + self.executor.submit(self._run_with_retries, job_id, handler) + return job_id + + def _run_with_retries(self, job_id: str, handler: Callable[[str, Callable[[str], None]], object]) -> None: + attempts = 0 + while attempts < self.settings.max_retries: + attempts += 1 + if self.get(job_id).state == "CANCELLED": + self._send_callback(job_id) + return + self._update(job_id, state="RUNNING", metrics={"attempt": attempts}) + try: + if self.get(job_id).state == "CANCEL_REQUESTED": + self._update(job_id, state="CANCELLED", failure_reason="Cancelled before render started") + self._send_callback(job_id) + return + result = handler(job_id, lambda message: self.append_log(job_id, message)) + record = self.get(job_id) + output_path = getattr(result, "output_path", None) + export_path = self._export_copy(output_path, record.export_target, job_id) if output_path else None + self._update( + job_id, + state="COMPLETED", + output_path=str(output_path) if output_path else None, + export_path=export_path, + commands=getattr(result, "commands", []), + logs=getattr(result, "logs", []), + metrics=getattr(result, "metrics", {}) | {"attempt": attempts}, + ) + self._send_callback(job_id) + return + except Exception as exc: + self.append_log(job_id, f"Attempt {attempts} failed: {exc}") + if attempts >= self.settings.max_retries: + self._update(job_id, state="FAILED", failure_reason=str(exc), metrics={"attempt": attempts}) + self._send_callback(job_id) + + def append_log(self, job_id: str, message: str) -> None: + with self.lock: + record = self.get(job_id) + record.logs.append(message) + record.updated_at = now() + self._save(record) + + def _update(self, job_id: str, **changes) -> None: + with self.lock: + record = self.get(job_id) + for key, value in changes.items(): + if key == "metrics" and record.metrics and isinstance(value, dict): + record.metrics.update(value) + else: + setattr(record, key, value) + record.updated_at = now() + self._save(record) + + def _record_path(self, job_id: str) -> Path: + return self.settings.jobs_dir / f"{job_id}.json" + + def _save(self, record: JobRecord) -> None: + write_json(self._record_path(record.job_id), asdict(record)) + + def _export_copy(self, output_path: Path, export_target: str | None, job_id: str) -> str | None: + if not export_target: + return None + if export_target != "local": + if export_target.startswith(("http://", "https://")): + _put_file(export_target, output_path) + return export_target + return None + target_dir = self.settings.storage_dir / job_id + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / output_path.name + shutil.copy2(output_path, target) + return str(target) + + def _send_callback(self, job_id: str) -> None: + try: + record = self.get(job_id) + except KeyError: + return + if not record.callback_url: + return + payload = json.dumps(asdict(record), default=str).encode("utf-8") + request = urllib.request.Request( + record.callback_url, + data=payload, + headers={"Content-Type": "application/json", "User-Agent": "ava2lon-studio-callback/2.0"}, + method="POST", + ) + try: + urllib.request.urlopen(request, timeout=10).read() + except Exception as exc: + self.append_log(job_id, f"Callback delivery failed: {exc}") + + +def _put_file(url: str, path: Path) -> None: + parsed = urlparse(url) + connection_cls = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + connection = connection_cls(parsed.netloc, timeout=60) + target = parsed.path or "/" + if parsed.query: + target += f"?{parsed.query}" + headers = { + "Content-Type": "video/mp4", + "Content-Length": str(path.stat().st_size), + "User-Agent": "ava2lon-studio-export/2.0", + } + connection.putrequest("PUT", target) + for key, value in headers.items(): + connection.putheader(key, value) + connection.endheaders() + with path.open("rb") as source: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + connection.send(chunk) + response = connection.getresponse() + body = response.read() + connection.close() + if response.status >= 400: + raise RuntimeError(f"Export upload failed with HTTP {response.status}: {body[:500]!r}") diff --git a/services/render_engine/renderer/platform/__init__.py b/services/render_engine/renderer/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eade10feeb9c8e89a2a5b0a274f94b86a482ca7b --- /dev/null +++ b/services/render_engine/renderer/platform/__init__.py @@ -0,0 +1,3 @@ +from renderer.platform.processor import PlatformProcessor, supported_toolkit_tasks + +__all__ = ["PlatformProcessor", "supported_toolkit_tasks"] diff --git a/services/render_engine/renderer/platform/processor.py b/services/render_engine/renderer/platform/processor.py new file mode 100644 index 0000000000000000000000000000000000000000..1c2dcd62fb6cbc482f79cf92286da8059ecd374b --- /dev/null +++ b/services/render_engine/renderer/platform/processor.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import json +import math +import mimetypes +import shutil +import zipfile +from pathlib import Path +from typing import Any + +from renderer.core.config import Settings +from renderer.core.ingest import AssetIngestor +from renderer.core.models import TaskResult +from renderer.core.utils import safe_filename, temp_workdir, write_json +from renderer.ffmpeg.assets import AssetProbe +from renderer.ffmpeg.command import FFmpegCommand +from renderer.ffmpeg.runner import FFmpegRunner +from renderer.templates import get_platform_profile + + +TOOLKIT_TASKS = { + "cut", + "trim", + "split", + "concat", + "merge", + "compress", + "normalize", + "resize", + "crop", + "rotate", + "flip", + "scale", + "zoom", + "pan", + "speed", + "speed_ramp", + "time_remap", + "slow_motion", + "fast_motion", + "reverse", + "reverse_playback", + "freeze_frame", + "motion_blur", + "stabilization", + "lens_correction", + "loop", + "extract_audio", + "thumbnail", + "gif", + "frames", + "watermark", + "overlay_text", + "blur_background", + "burn_subtitles", + "convert", + "merge_audio", + "noise_reduction", + "equalizer", + "compressor", + "limiter", + "pitch_shift", + "voice_changer", + "ai_enhancement", + "green_screen", + "chroma_key", + "blue_screen", + "ai_background_removal", +} + + +class PlatformProcessor: + def __init__(self, settings: Settings | None = None, log=None) -> None: + self.settings = settings or Settings() + self.settings.ensure_dirs() + self._commands: list[list[str]] = [] + self._logs: list[str] = [] + self.runner = FFmpegRunner(self.settings.ffmpeg_timeout_seconds, log=log, on_command=self._record_command) + self.ingest = AssetIngestor(self.settings) + self.assets = AssetProbe(self.settings.metadata_cache) + + def ingest_sources(self, sources: list[dict[str, Any]], job_id: str) -> TaskResult: + with temp_workdir(self.settings.temp_dir, f"{job_id}_ingest") as work: + workdir = Path(work) + staged: list[dict[str, Any]] = [] + for index, source in enumerate(sources): + url = str(source.get("url") or source.get("source") or "").strip() + if not url: + continue + source_type = str(source.get("type") or _source_type(url)) + if source_type == "youtube": + staged.append( + { + "source": url, + "source_type": source_type, + "status": "registered", + "note": "YouTube ingestion is registered for automation; provide a direct downloadable media URL or install yt-dlp for local extraction.", + } + ) + continue + resolved = self.ingest.resolve(url, workdir / "inputs", f"source_{index:03d}") + metadata = self.assets.probe(resolved).__dict__ + staged.append({"source": url, "source_type": source_type, "path": str(resolved), "metadata": metadata}) + output = self._json_artifact(job_id, "ingest_manifest", {"assets": staged, "asset_count": len(staged)}) + return self._result(output, {"task": "ingest", "asset_count": len(staged)}) + + def analyze(self, media: str, job_id: str, *, transcript: str = "", platform: str | None = None) -> TaskResult: + with temp_workdir(self.settings.temp_dir, f"{job_id}_analyze") as work: + workdir = Path(work) + source = self.ingest.resolve(media, workdir / "inputs", "media") + metadata = self.assets.probe(source) + duration = max(0.0, metadata.duration) + words = transcript.split() + words_per_minute = (len(words) / duration * 60) if duration > 0 and words else None + highlights = _highlight_windows(duration) + viral_score = _viral_score(duration, bool(words), metadata.width, metadata.height) + analysis = { + "media": str(source), + "metadata": metadata.__dict__, + "transcript": transcript, + "highlight_moments": highlights, + "viral_score": viral_score, + "hook_quality": _hook_quality(transcript), + "audience_retention_estimate": _retention_estimate(duration, viral_score), + "engagement_prediction": _engagement_prediction(viral_score), + "audience_persona": _persona(transcript), + "platform_recommendations": _platform_recommendations(duration, metadata.width, metadata.height, platform), + "scene_segmentation": highlights, + "speech_pacing": { + "words_per_minute": round(words_per_minute, 1) if words_per_minute else None, + "label": _pacing_label(words_per_minute), + }, + "silence_detection": { + "estimated_silence_ratio": 0.0 if transcript else 0.18, + "note": "Heuristic estimate; use transcription with word timestamps for precise silence spans.", + }, + } + output = self._json_artifact(job_id, "analysis", analysis) + return self._result(output, {"task": "analyze", "viral_score": viral_score, "duration": duration}) + + def metadata(self, job_id: str, *, topic: str = "", transcript: str = "", platform: str | None = None) -> TaskResult: + text = transcript or topic or "Untitled video" + title = _title_from_text(text, platform) + tags = _hashtags(text, platform) + payload = { + "title": title, + "description": _description(text, tags), + "hashtags": tags, + "keywords": _keywords(text), + "chapters": _chapters(text), + "seo_tags": _keywords(text) + [platform] if platform else _keywords(text), + "suggested_upload_schedule": _schedule(platform), + "platform": platform or "general", + } + output = self._json_artifact(job_id, "metadata", payload) + return self._result(output, {"task": "metadata", "keyword_count": len(payload["keywords"])}) + + def publish(self, payload: dict[str, Any], job_id: str) -> TaskResult: + platforms = payload.get("platforms") or [payload.get("platform") or "draft"] + manifest = { + "publish_state": "draft_ready" if payload.get("draft", True) else "credentials_required", + "platforms": platforms, + "scheduled_at": payload.get("scheduled_at"), + "asset": payload.get("asset") or payload.get("media"), + "title": payload.get("title"), + "description": payload.get("description"), + "retry_policy": {"max_attempts": 3, "backoff_seconds": 60}, + "note": "Direct publishing requires platform OAuth/API credentials configured outside this CPU render worker.", + } + output = self._json_artifact(job_id, "publish_manifest", manifest) + return self._result(output, {"task": "publish", "platform_count": len(platforms)}) + + def clips(self, media: str, job_id: str, clips: list[dict[str, Any]] | None = None) -> TaskResult: + with temp_workdir(self.settings.temp_dir, f"{job_id}_clips") as work: + workdir = Path(work) + source = self.ingest.resolve(media, workdir / "inputs", "media") + metadata = self.assets.probe(source) + clip_specs = clips or _highlight_windows(metadata.duration) + outputs: list[Path] = [] + for index, clip in enumerate(clip_specs): + start = max(0.0, float(clip.get("start", 0))) + end = float(clip.get("end", start + clip.get("duration", 8))) + duration = max(0.2, end - start) + target = workdir / f"clip_{index + 1:02d}.mp4" + command = ( + FFmpegCommand() + .add("-hide_banner", "-ss", start) + .input(source) + .add("-t", duration, "-c", "copy") + .overwrite() + .add(target) + .build() + ) + self._run(command) + outputs.append(target) + if len(outputs) == 1: + final = self._export(outputs[0], job_id, outputs[0].name) + else: + final = self.settings.exports_dir / f"{job_id}_clips.zip" + with zipfile.ZipFile(final, "w", zipfile.ZIP_DEFLATED) as archive: + for path in outputs: + archive.write(path, path.name) + return self._result(final, {"task": "clips", "clip_count": len(outputs)}) + + def thumbnail(self, media: str, job_id: str, *, text: str = "", timestamp: float | None = None, template: str = "bold") -> TaskResult: + with temp_workdir(self.settings.temp_dir, f"{job_id}_thumb") as work: + workdir = Path(work) + source = self.ingest.resolve(media, workdir / "inputs", "media") + metadata = self.assets.probe(source) + target = workdir / "thumbnail.jpg" + seek = timestamp if timestamp is not None else max(0.0, min(metadata.duration * 0.2, 8.0)) + vf = "scale=1280:720:force_original_aspect_ratio=increase,crop=1280:720" + if text: + vf += "," + _drawtext_filter(text, template) + command = ( + FFmpegCommand() + .add("-hide_banner", "-ss", seek) + .input(source) + .add("-frames:v", 1, "-vf", vf, "-q:v", 2) + .overwrite() + .add(target) + .build() + ) + self._run(command) + final = self._export(target, job_id, "thumbnail.jpg") + return self._result(final, {"task": "thumbnail", "timestamp": seek}) + + def toolkit(self, payload: dict[str, Any], job_id: str) -> TaskResult: + task = str(payload.get("task") or payload.get("operation") or "").strip() + if task not in TOOLKIT_TASKS: + raise ValueError(f"Unsupported toolkit task: {task}") + if task == "thumbnail": + return self.thumbnail(str(payload["input"]), job_id, text=str(payload.get("text") or ""), timestamp=payload.get("timestamp")) + if task == "split": + clips = payload.get("clips") if isinstance(payload.get("clips"), list) else payload.get("params", {}).get("clips") + return self.clips(str(payload.get("input") or payload.get("media")), job_id, clips) + + with temp_workdir(self.settings.temp_dir, f"{job_id}_{task}") as work: + workdir = Path(work) + source_value = payload.get("input") or payload.get("media") + source = self.ingest.resolve(str(source_value), workdir / "inputs", "media") if source_value else workdir / "concat_placeholder.mp4" + params = payload.get("params") if isinstance(payload.get("params"), dict) else payload + output_name = safe_filename(str(payload.get("output_name") or _default_output_name(task))) + output = workdir / output_name + if task == "extract_audio" and output.suffix.lower() != ".mp3": + output = output.with_suffix(".mp3") + if task == "gif" and output.suffix.lower() != ".gif": + output = output.with_suffix(".gif") + command = self._toolkit_command(task, source, output, params, workdir) + self._run(command) + if task == "frames": + final = self.settings.exports_dir / f"{job_id}_frames.zip" + with zipfile.ZipFile(final, "w", zipfile.ZIP_DEFLATED) as archive: + for frame in sorted(workdir.glob("frame_*.jpg")): + archive.write(frame, frame.name) + else: + final = self._export(output, job_id, output.name) + return self._result(final, {"task": task}) + + def _toolkit_command(self, task: str, source: Path, output: Path, params: dict[str, Any], workdir: Path) -> list[str]: + cmd = FFmpegCommand().add("-hide_banner") + if task == "loop": + cmd.add("-stream_loop", int(params.get("loops", -1))) + if task in {"cut", "trim", "gif", "freeze_frame"} and params.get("start") is not None: + cmd.add("-ss", float(params.get("start", 0))) + cmd.input(source) + + if task == "merge_audio": + audio = self.ingest.resolve(str(params.get("audio")), workdir / "inputs", "audio") + cmd.input(audio) + return cmd.add("-map", "0:v", "-map", "1:a", "-c:v", "copy", "-c:a", "aac", "-shortest").overwrite().add(output).build() + if task == "watermark": + image = self.ingest.resolve(str(params.get("watermark") or params.get("image")), workdir / "inputs", "watermark") + cmd.input(image) + return cmd.add("-filter_complex", "[1:v]scale=iw*0.18:-1[wm];[0:v][wm]overlay=W-w-24:H-h-24", "-c:a", "copy").overwrite().add(output).build() + if task in {"concat", "merge"}: + inputs = params.get("inputs") + if not isinstance(inputs, list) or not inputs: + raise ValueError("Concat requires params.inputs") + concat_file = workdir / "concat.txt" + lines: list[str] = [] + for index, item in enumerate(inputs): + media = self.ingest.resolve(str(item), workdir / "inputs", f"concat_{index:03d}") + lines.append(f"file '{str(media).replace(chr(39), chr(39) + chr(92) + chr(39) + chr(39))}'") + concat_file.write_text("\n".join(lines), encoding="utf-8") + return FFmpegCommand().add("-hide_banner", "-f", "concat", "-safe", "0").input(concat_file).add("-c", "copy").overwrite().add(output).build() + + duration = params.get("duration") + if task in {"cut", "trim", "gif", "loop", "freeze_frame"} and duration is not None: + cmd.add("-t", float(duration)) + + vf = _video_filter(task, params) + af = _audio_filter(task, params) + if vf: + cmd.add("-vf", vf) + if af: + cmd.add("-af", af) + + if task == "extract_audio": + return cmd.add("-vn", "-c:a", "mp3", "-b:a", "192k").overwrite().add(output).build() + if task == "frames": + return cmd.add("-vf", vf or "fps=1", "-q:v", 2).overwrite().add(workdir / "frame_%04d.jpg").build() + if task == "gif": + return cmd.add("-loop", 0).overwrite().add(output).build() + if task in { + "cut", + "trim", + "compress", + "normalize", + "resize", + "crop", + "rotate", + "flip", + "scale", + "zoom", + "pan", + "speed", + "speed_ramp", + "time_remap", + "slow_motion", + "fast_motion", + "reverse", + "reverse_playback", + "freeze_frame", + "motion_blur", + "stabilization", + "lens_correction", + "loop", + "overlay_text", + "blur_background", + "burn_subtitles", + "convert", + "noise_reduction", + "equalizer", + "compressor", + "limiter", + "pitch_shift", + "voice_changer", + "green_screen", + "chroma_key", + "blue_screen", + "ai_background_removal", + }: + cmd.add("-c:v", "libx264", "-preset", self.settings.preset, "-crf", int(params.get("crf", self.settings.crf)), "-c:a", "aac") + return cmd.overwrite().add(output).build() + + def _json_artifact(self, job_id: str, name: str, payload: dict[str, Any]) -> Path: + output = self.settings.exports_dir / f"{job_id}_{safe_filename(name)}.json" + write_json(output, payload) + return output + + def _export(self, source: Path, job_id: str, output_name: str) -> Path: + target = self.settings.exports_dir / f"{job_id}_{safe_filename(output_name)}" + shutil.copy2(source, target) + return target + + def _run(self, command: list[str]) -> None: + result = self.runner.run(command) + if result.stderr: + self._logs.append(result.stderr[-4000:]) + + def _record_command(self, command: list[str]) -> None: + self._commands.append(command) + + def _result(self, output: Path | None, metrics: dict[str, Any]) -> TaskResult: + return TaskResult(output_path=output, commands=list(self._commands), metrics=metrics, logs=list(self._logs)) + + +def supported_toolkit_tasks() -> list[str]: + return sorted(TOOLKIT_TASKS) + + +def _source_type(url: str) -> str: + lowered = url.lower() + if "youtube.com" in lowered or "youtu.be" in lowered: + return "youtube" + if "drive.google.com" in lowered: + return "google_drive" + if "s3" in lowered or "amazonaws.com" in lowered: + return "s3" + return "direct_url" + + +def _highlight_windows(duration: float) -> list[dict[str, Any]]: + if duration <= 0: + return [{"start": 0, "end": 8, "reason": "default opener"}] + windows = [{"start": 0, "end": min(duration, 8), "reason": "opening hook"}] + if duration > 18: + middle = max(0.0, duration * 0.42) + windows.append({"start": round(middle, 2), "end": round(min(duration, middle + 10), 2), "reason": "midpoint payoff"}) + if duration > 35: + end = max(0.0, duration - 12) + windows.append({"start": round(end, 2), "end": round(duration, 2), "reason": "closing CTA"}) + return windows + + +def _viral_score(duration: float, has_words: bool, width: int | None, height: int | None) -> int: + score = 48 + if 7 <= duration <= 60: + score += 20 + elif duration <= 180: + score += 8 + if width and height and height >= width: + score += 14 + if has_words: + score += 10 + return max(1, min(100, score)) + + +def _hook_quality(transcript: str) -> dict[str, Any]: + opener = " ".join(transcript.split()[:18]) + signals = sum(1 for token in ("how", "why", "secret", "mistake", "stop", "watch", "you") if token in opener.lower()) + return {"score": min(100, 45 + signals * 12), "opening_text": opener} + + +def _retention_estimate(duration: float, viral_score: int) -> dict[str, Any]: + first_3s = min(96, 55 + viral_score * 0.35) + completion = max(18, min(88, first_3s - math.log(max(duration, 1), 1.8))) + return {"first_3_seconds_percent": round(first_3s, 1), "completion_percent": round(completion, 1)} + + +def _engagement_prediction(score: int) -> str: + if score >= 80: + return "high" + if score >= 62: + return "medium" + return "needs_work" + + +def _persona(text: str) -> str: + lowered = text.lower() + if any(word in lowered for word in ("founder", "startup", "product", "launch")): + return "builders and product-led founders" + if any(word in lowered for word in ("money", "sales", "growth", "marketing")): + return "growth-minded operators" + if any(word in lowered for word in ("learn", "tutorial", "how")): + return "learners seeking practical instruction" + return "general short-form viewers" + + +def _platform_recommendations(duration: float, width: int | None, height: int | None, platform: str | None) -> list[dict[str, Any]]: + vertical = bool(width and height and height >= width) + candidates = ["tiktok", "instagram_reels", "youtube_shorts"] if vertical else ["youtube_1080p", "linkedin_video"] + if platform and platform not in candidates: + candidates.insert(0, platform) + return [{"platform": item, "fit": "strong" if duration <= 90 else "medium"} for item in candidates] + + +def _pacing_label(wpm: float | None) -> str: + if not wpm: + return "unknown" + if wpm < 125: + return "slow" + if wpm > 185: + return "fast" + return "clear" + + +def _title_from_text(text: str, platform: str | None) -> str: + words = [word.strip(".,:;!?") for word in text.split() if word.strip(".,:;!?")] + title = " ".join(words[:9]) or "Untitled Video" + suffix = " #Shorts" if platform in {"youtube_shorts", "tiktok", "instagram_reels"} else "" + return f"{title.title()}{suffix}" + + +def _description(text: str, tags: list[str]) -> str: + summary = " ".join(text.split()[:42]) + return f"{summary}\n\n{' '.join(tags)}".strip() + + +def _hashtags(text: str, platform: str | None) -> list[str]: + base = ["#video", "#content"] + if platform: + base.append(f"#{platform.replace('_', '')}") + for keyword in _keywords(text)[:5]: + tag = "#" + "".join(ch for ch in keyword.title() if ch.isalnum()) + if tag not in base: + base.append(tag) + return base[:8] + + +def _keywords(text: str) -> list[str]: + stop = {"the", "and", "for", "with", "that", "this", "your", "you", "are", "from", "into", "video"} + words = [word.strip(".,:;!?").lower() for word in text.split()] + unique: list[str] = [] + for word in words: + if len(word) < 4 or word in stop or word in unique: + continue + unique.append(word) + return unique[:12] + + +def _chapters(text: str) -> list[dict[str, Any]]: + sentences = [part.strip() for part in text.replace("?", ".").replace("!", ".").split(".") if part.strip()] + return [{"time": f"0:{index * 15:02d}", "title": sentence[:60]} for index, sentence in enumerate(sentences[:6])] + + +def _schedule(platform: str | None) -> dict[str, str]: + if platform in {"linkedin_video", "youtube_1080p"}: + return {"day": "Tuesday", "time": "09:00 local"} + return {"day": "Thursday", "time": "18:00 local"} + + +def _video_filter(task: str, params: dict[str, Any]) -> str: + if task in {"compress", "normalize"}: + profile = get_platform_profile(params.get("platform")) + return f"scale={profile.width}:{profile.height}:force_original_aspect_ratio=increase,crop={profile.width}:{profile.height},fps={profile.fps},format=yuv420p" + if task in {"resize", "scale"}: + return f"scale={int(params.get('width', 1080))}:{int(params.get('height', 1920))}" + if task == "crop": + return f"crop={int(params.get('width', 1080))}:{int(params.get('height', 1080))}:{int(params.get('x', 0))}:{int(params.get('y', 0))}" + if task == "rotate": + return {"90": "transpose=1", "180": "hflip,vflip", "270": "transpose=2"}.get(str(params.get("degrees", "90")), "transpose=1") + if task == "flip": + axis = str(params.get("axis", "horizontal")) + return "vflip" if axis in {"vertical", "y"} else "hflip" + if task in {"speed", "speed_ramp", "time_remap", "slow_motion", "fast_motion"}: + default = 0.5 if task == "slow_motion" else 2.0 if task == "fast_motion" else 1.0 + factor = max(0.25, min(4.0, float(params.get("factor", default)))) + return f"setpts={1 / factor:.4f}*PTS" + if task == "zoom": + factor = max(1.0, min(4.0, float(params.get("factor", 1.2)))) + return f"scale=iw*{factor:.3f}:ih*{factor:.3f},crop=iw/{factor:.3f}:ih/{factor:.3f}" + if task == "pan": + width = int(params.get("width", 1080)) + height = int(params.get("height", 1920)) + x = str(params.get("x", "(iw-ow)/2")) + y = str(params.get("y", "(ih-oh)/2")) + return f"crop={width}:{height}:{x}:{y}" + if task in {"motion_blur", "freeze_frame"}: + return "tmix=frames=3:weights='1 2 1'" + if task == "stabilization": + return "deshake" + if task == "lens_correction": + return f"lenscorrection=k1={float(params.get('k1', -0.15))}:k2={float(params.get('k2', 0.05))}" + if task in {"reverse", "reverse_playback"}: + return "reverse" + if task in {"green_screen", "chroma_key", "ai_background_removal"}: + color = str(params.get("color") or "0x00ff00") + similarity = float(params.get("similarity", 0.18)) + blend = float(params.get("blend", 0.08)) + return f"chromakey={color}:{similarity}:{blend}" + if task == "blue_screen": + similarity = float(params.get("similarity", 0.18)) + blend = float(params.get("blend", 0.08)) + return f"chromakey=0x0000ff:{similarity}:{blend}" + if task == "speed": + factor = max(0.25, min(4.0, float(params.get("factor", 1.0)))) + return f"setpts={1 / factor:.4f}*PTS" + if task == "gif": + return f"fps={int(params.get('fps', 12))},scale={int(params.get('width', 540))}:-1:flags=lanczos" + if task == "frames": + return f"fps={float(params.get('fps', 1))}" + if task == "overlay_text": + return _drawtext_filter(str(params.get("text") or "Text"), "bold") + if task == "blur_background": + return "gblur=sigma=18" + if task == "burn_subtitles": + subtitles = str(params.get("subtitles") or "").replace("\\", "/").replace(":", r"\:") + return f"subtitles='{subtitles}'" + return "" + + +def _audio_filter(task: str, params: dict[str, Any]) -> str: + if task in {"speed", "speed_ramp", "time_remap", "slow_motion", "fast_motion"}: + default = 0.5 if task == "slow_motion" else 2.0 if task == "fast_motion" else 1.0 + factor = max(0.5, min(2.0, float(params.get("factor", default)))) + return f"atempo={factor}" + if task in {"reverse", "reverse_playback"}: + return "areverse" + if task in {"noise_reduction", "ai_enhancement"}: + return "afftdn=nf=-25" + if task == "equalizer": + return f"equalizer=f={float(params.get('frequency', 1000))}:width_type=o:width={float(params.get('width', 1))}:g={float(params.get('gain', 3))}" + if task == "compressor": + return "acompressor=threshold=-18dB:ratio=3:attack=20:release=250" + if task == "limiter": + return "alimiter=limit=0.95" + if task in {"pitch_shift", "voice_changer"}: + factor = max(0.5, min(2.0, float(params.get("factor", 1.0)))) + return f"asetrate=48000*{factor:.4f},aresample=48000,atempo={1 / factor:.4f}" + return "" + + +def _drawtext_filter(text: str, template: str) -> str: + escaped = text.replace("\\", "\\\\").replace(":", r"\:").replace("'", r"\'") + color = "yellow" if template == "bold" else "white" + return ( + "drawtext=" + f"text='{escaped}':fontcolor={color}:fontsize=58:" + "box=1:boxcolor=black@0.55:boxborderw=24:" + "x=(w-text_w)/2:y=h-(text_h*3)" + ) + + +def _default_output_name(task: str) -> str: + if task == "extract_audio": + return "audio.mp3" + if task == "gif": + return "clip.gif" + if task == "thumbnail": + return "thumbnail.jpg" + return f"{task}.mp4" diff --git a/services/render_engine/renderer/scenes/__init__.py b/services/render_engine/renderer/scenes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a34a477c851afa81f723ea818a417098be202aba --- /dev/null +++ b/services/render_engine/renderer/scenes/__init__.py @@ -0,0 +1,3 @@ +from renderer.scenes.timeline import Timeline + +__all__ = ["Timeline"] diff --git a/services/render_engine/renderer/scenes/timeline.py b/services/render_engine/renderer/scenes/timeline.py new file mode 100644 index 0000000000000000000000000000000000000000..148f78f65025d5b6adff8cf4751b811fff6a6cc8 --- /dev/null +++ b/services/render_engine/renderer/scenes/timeline.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from renderer.core.models import RenderRequest, Scene + + +class Timeline: + def __init__(self, scenes: list[Scene]) -> None: + self.scenes = sorted(scenes, key=lambda scene: scene.start) + self.validate() + + @classmethod + def from_payload(cls, payload: dict) -> "Timeline": + return cls([Scene(**scene) for scene in payload.get("scenes", [])]) + + @classmethod + def request_from_payload(cls, payload: dict) -> RenderRequest: + scenes = [Scene(**scene) for scene in payload.get("scenes", [])] + return RenderRequest( + scenes=scenes, + template=payload.get("template", "tiktok_classic"), + preset=payload.get("preset"), + creative_style=payload.get("creative_style"), + platform=payload.get("platform"), + output_name=payload.get("output_name", "render.mp4"), + voiceover=payload.get("voiceover"), + background_music=payload.get("background_music"), + music_volume=payload.get("music_volume", 0.316), + music_fade_in=payload.get("music_fade_in", 0.0), + music_fade_out=payload.get("music_fade_out", 0.0), + music_loop=payload.get("music_loop", True), + music_start=payload.get("music_start", 0.0), + music_ducking=payload.get("music_ducking", True), + voice_volume=payload.get("voice_volume", 1.0), + subtitle_format=payload.get("subtitle_format", "ass"), + auto_subtitles=payload.get("auto_subtitles", False), + subtitle_language=payload.get("subtitle_language"), + whisper_model_size=payload.get("whisper_model_size"), + preview=payload.get("preview", False), + audio_normalize=payload.get("audio_normalize", False), + watermark=payload.get("watermark"), + watermark_position=payload.get("watermark_position", "bottom-right"), + intro=payload.get("intro"), + outro=payload.get("outro"), + callback_url=payload.get("callback_url"), + export_target=payload.get("export_target"), + priority=payload.get("priority", 0), + scheduled_at=payload.get("scheduled_at"), + normalize=payload.get("normalize", True), + metadata=payload.get("metadata", {}), + ) + + @property + def total_duration(self) -> float: + return max((scene.start + scene.duration for scene in self.scenes), default=0.0) + + def validate(self) -> None: + if not self.scenes: + raise ValueError("At least one scene is required") + for scene in self.scenes: + if scene.duration <= 0: + raise ValueError("Scene duration must be greater than zero") + if scene.start < 0: + raise ValueError("Scene start must be non-negative") + if not scene.media: + raise ValueError("Scene media path is required") diff --git a/services/render_engine/renderer/studio/__init__.py b/services/render_engine/renderer/studio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..444ce12da9eeaf9716a4af2b2b0c7d8b52e7ee4f --- /dev/null +++ b/services/render_engine/renderer/studio/__init__.py @@ -0,0 +1,27 @@ +from renderer.studio.capabilities import capability_catalog +from renderer.studio.projects import ( + ProjectStore, + add_effect, + add_filter, + add_keyframe, + add_timeline_item, + add_transition, + apply_timeline_operation, + default_project, + normalize_project, +) +from renderer.studio.tasks import StudioTaskProcessor + +__all__ = [ + "ProjectStore", + "StudioTaskProcessor", + "add_effect", + "add_filter", + "add_keyframe", + "add_timeline_item", + "add_transition", + "apply_timeline_operation", + "capability_catalog", + "default_project", + "normalize_project", +] diff --git a/services/render_engine/renderer/studio/capabilities.py b/services/render_engine/renderer/studio/capabilities.py new file mode 100644 index 0000000000000000000000000000000000000000..749278c59ad490c2d82e5c88738d30cd7ef11f7c --- /dev/null +++ b/services/render_engine/renderer/studio/capabilities.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +TIMELINE_TRACK_TYPES = [ + "video", + "audio", + "text", + "overlay", + "sticker", + "subtitle", +] + +TIMELINE_OPERATIONS = [ + "drag", + "split", + "trim", + "ripple_delete", + "insert", + "replace", + "group", + "lock", + "hide", + "duplicate", +] + +VIDEO_EDITING_OPERATIONS = [ + "cut", + "split", + "trim", + "merge", + "concat", + "reverse", + "freeze_frame", + "speed", + "speed_ramp", + "time_remap", + "slow_motion", + "fast_motion", + "reverse_playback", + "crop", + "rotate", + "flip", + "resize", + "scale", + "zoom", + "pan", + "motion_blur", + "stabilization", + "lens_correction", + "compress", + "normalize", + "loop", + "gif", + "frames", + "watermark", + "overlay_text", + "burn_subtitles", + "convert", +] + +AI_EDITING_FEATURES = [ + "auto_edit", + "auto_highlight_detection", + "auto_scene_detection", + "auto_reframe", + "auto_crop", + "auto_remove_silence", + "auto_beat_sync", + "auto_color_match", + "auto_motion_tracking", + "auto_subtitle_generation", + "auto_hook_detection", + "auto_thumbnail_selection", + "auto_music_selection", + "auto_b_roll_placement", + "auto_caption_animation", + "auto_viral_score", + "auto_platform_optimization", +] + +KEYFRAME_PROPERTIES = [ + "position", + "scale", + "rotation", + "opacity", + "blur", + "brightness", + "contrast", + "saturation", + "hue", + "volume", + "playback_speed", + "mask", + "shadow", + "glow", + "text_animation", +] + +TRANSITION_FAMILIES = [ + "basic", + "fade", + "dissolve", + "slide", + "push", + "zoom", + "spin", + "blur", + "whip", + "flash", + "glitch", + "light_leak", + "film_burn", + "camera_shake", + "3d_flip", + "cube", + "ripple", + "ink", + "morph", + "stretch", + "liquid", + "elastic", + "motion_blur", +] + +VIDEO_EFFECTS = [ + "glitch", + "rgb_split", + "shake", + "crt", + "vhs", + "noise", + "film_grain", + "bloom", + "glow", + "chromatic_aberration", + "lens_flare", + "dream", + "neon", + "cyberpunk", + "rain", + "snow", + "fog", + "lightning", + "fire", + "smoke", + "particle_system", + "spark", + "magic", + "comic", + "cartoon", + "anime", + "sketch", + "oil_painting", + "pixel_art", + "sharp_pop", + "clean_beauty", + "warm_glow", + "cinematic", + "dreamy", + "flash_pop", + "motion_blur", + "noir", +] + +FILTER_FORMATS = [".cube", ".3dl", ".csp"] + +FILTER_PRESETS = [ + "cinema", + "vintage", + "warm", + "cold", + "black_and_white", + "hdr", + "instagram", + "tiktok", + "moody", + "travel", + "nature", + "food", + "portrait", + "luxury", + "night", +] + +TEXT_FEATURES = [ + "rich_text", + "curved_text", + "vertical_text", + "gradient_text", + "outline", + "shadow", + "glow", + "stroke", + "letter_spacing", + "word_spacing", + "animation_presets", + "typing_animation", + "bounce", + "wave", + "zoom", + "pop", + "fade", + "roll", + "tracking", +] + +CAPTION_FEATURES = [ + "whisper_transcription", + "word_timestamps", + "sentence_timestamps", + "emoji_insertion", + "speaker_detection", + "karaoke_captions", + "tiktok_captions", + "capcut_captions", + "animated_captions", + "subtitle_templates", +] + +STICKER_PACKS = [ + "png", + "svg", + "gif", + "animated_stickers", + "emoji_packs", + "reaction_packs", + "social_media_packs", + "call_to_action_packs", +] + +SHAPES = [ + "rectangle", + "circle", + "triangle", + "arrow", + "line", + "polygon", + "speech_bubble", + "custom_svg", +] + +MASK_TYPES = [ + "rectangle", + "circle", + "linear", + "radial", + "freehand", + "bezier", + "ai_subject_mask", + "ai_sky_mask", + "ai_person_mask", +] + +CHROMA_KEY_FEATURES = [ + "green_screen", + "blue_screen", + "ai_background_removal", + "edge_feathering", + "spill_suppression", + "shadow_preservation", +] + +AUDIO_TOOLS = [ + "music", + "voiceover", + "noise_reduction", + "equalizer", + "compressor", + "limiter", + "pitch_shift", + "voice_changer", + "fade", + "ducking", + "normalization", + "ai_enhancement", + "beat_detection", + "beat_markers", +] + +MUSIC_PROVIDERS = ["musicgen", "suno_api", "stable_audio"] +MUSIC_STYLES = ["background_music", "lo_fi", "cinematic", "nasheed", "hip_hop", "corporate", "podcast", "meditation"] + +VOICE_PROVIDERS = ["kokoro", "xtts", "piper", "openvoice"] +VOICE_FEATURES = ["voice_cloning", "multi_speaker", "emotion", "speed", "pitch", "style_transfer"] + +IMAGE_GENERATION_PROVIDERS = ["flux", "sdxl", "controlnet"] +IMAGE_GENERATION_FEATURES = ["image_editing", "background_replacement", "object_removal", "upscaling"] + +VIDEO_GENERATION_PROVIDERS = ["wan", "ltx_video", "hunyuan_video", "veo_api"] +VIDEO_GENERATION_FEATURES = ["animate_images", "image_to_video", "text_to_video"] + +AI_ASSISTANTS = [ + "script_writer", + "hook_generator", + "title_generator", + "description_generator", + "hashtag_generator", + "seo_optimizer", + "thumbnail_prompt_generator", + "b_roll_planner", + "storyboard_generator", +] + +TEMPLATE_CATEGORIES = [ + "youtube_shorts", + "tiktok", + "instagram_reels", + "facebook_reels", + "motivational", + "podcasts", + "gaming", + "news", + "luxury", + "business", + "education", + "finance", + "relationship", + "wedding", + "birthday", + "travel", + "cooking", + "fitness", + "anime", + "sports", + "product_ads", + "real_estate", + "e_commerce", + "faceless_channels", + "quote_videos", + "audiograms", + "story_videos", + "before_and_after", + "reaction_videos", + "countdown_videos", +] + +EXPORT_FORMATS = ["mp4", "mov", "avi", "mkv", "gif", "webm", "png_sequence", "jpeg_sequence", "audio_only"] + +EXPORT_PRESETS = [ + "1080p", + "2k", + "4k", + "8k", + "tiktok", + "youtube", + "instagram", + "facebook", + "twitter", + "linkedin", +] + +API_ENDPOINTS = { + "upload": "POST /upload", + "project_create": "POST /project/create", + "project_save": "POST /project/save", + "timeline_add": "POST /timeline/add", + "timeline_operation": "POST /timeline/operation", + "effect_apply": "POST /effect/apply", + "filter_apply": "POST /filter/apply", + "transition_add": "POST /transition/add", + "caption_generate": "POST /caption/generate", + "music_generate": "POST /music/generate", + "voice_generate": "POST /voice/generate", + "image_generate": "POST /image/generate", + "video_generate": "POST /video/generate", + "thumbnail_create": "POST /thumbnail/create", + "render": "POST /render", + "status": "GET /status/{job_id}", + "download": "GET /download/{job_id}", + "publish": "POST /publish", +} + + +def capability_catalog() -> dict[str, Any]: + return deepcopy( + { + "product": "Ava2lon Studio AI", + "principles": { + "cpu_first": True, + "optional_gpu": True, + "api_parity": True, + "async_long_running_tasks": True, + "status_polling": True, + "webhooks": True, + "plugin_support": True, + "template_driven": True, + "non_destructive_projects": True, + "multi_platform_export": True, + }, + "timeline": { + "track_types": TIMELINE_TRACK_TYPES, + "operations": TIMELINE_OPERATIONS, + "unlimited_tracks": True, + }, + "editing": VIDEO_EDITING_OPERATIONS, + "ai_editing": AI_EDITING_FEATURES, + "keyframes": KEYFRAME_PROPERTIES, + "transitions": TRANSITION_FAMILIES, + "effects": VIDEO_EFFECTS, + "filters": {"formats": FILTER_FORMATS, "presets": FILTER_PRESETS}, + "text": TEXT_FEATURES, + "captions": CAPTION_FEATURES, + "stickers": STICKER_PACKS, + "shapes": SHAPES, + "masks": MASK_TYPES, + "chroma_key": CHROMA_KEY_FEATURES, + "audio": AUDIO_TOOLS, + "music_generator": {"providers": MUSIC_PROVIDERS, "styles": MUSIC_STYLES}, + "voice_generator": {"providers": VOICE_PROVIDERS, "features": VOICE_FEATURES}, + "image_generation": {"providers": IMAGE_GENERATION_PROVIDERS, "features": IMAGE_GENERATION_FEATURES}, + "video_generation": {"providers": VIDEO_GENERATION_PROVIDERS, "features": VIDEO_GENERATION_FEATURES}, + "assistants": AI_ASSISTANTS, + "templates": TEMPLATE_CATEGORIES, + "exports": {"formats": EXPORT_FORMATS, "presets": EXPORT_PRESETS}, + "api_endpoints": API_ENDPOINTS, + } + ) + + +def is_timeline_operation(operation: str) -> bool: + return operation in TIMELINE_OPERATIONS + + +def is_track_type(track_type: str) -> bool: + return track_type in TIMELINE_TRACK_TYPES diff --git a/services/render_engine/renderer/studio/projects.py b/services/render_engine/renderer/studio/projects.py new file mode 100644 index 0000000000000000000000000000000000000000..b83b299e151670fc0c7d7172e441c74065e8db8f --- /dev/null +++ b/services/render_engine/renderer/studio/projects.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +from typing import Any + +from renderer.core.config import Settings +from renderer.core.utils import new_id, now, read_json, safe_filename, write_json +from renderer.studio.capabilities import TIMELINE_TRACK_TYPES, is_timeline_operation, is_track_type + + +DEFAULT_EXPORT_SETTINGS: dict[str, Any] = { + "format": "mp4", + "preset": "tiktok", + "platform": "tiktok", + "resolution": "1080p", + "fps": 30, + "codec": "h264", + "audio_codec": "aac", +} + + +def default_project(name: str, metadata: dict[str, Any] | None = None, project_id: str | None = None) -> dict[str, Any]: + created = now() + project_id = project_id or new_id("project") + return { + "id": project_id, + "name": name, + "slug": safe_filename(name), + "version": 1, + "schema": "ava2lon.project.v1", + "created_at": created, + "updated_at": created, + "metadata": metadata or {}, + "timeline": { + "duration": 0.0, + "fps": 30, + "tracks": {track_type: [] for track_type in TIMELINE_TRACK_TYPES}, + "groups": [], + "markers": [], + }, + "assets": [], + "audio_tracks": [], + "video_tracks": [], + "text_layers": [], + "sticker_layers": [], + "effects": [], + "filters": [], + "keyframes": [], + "captions": [], + "templates": [], + "export_settings": deepcopy(DEFAULT_EXPORT_SETTINGS), + "plugins": [], + "automation": {"webhooks": [], "batch": {}, "n8n": {"compatible": True}}, + } + + +class ProjectStore: + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self.settings.ensure_dirs() + self.root = self.settings.storage_dir / "projects" + self.root.mkdir(parents=True, exist_ok=True) + + def list(self) -> list[dict[str, Any]]: + projects: list[dict[str, Any]] = [] + for manifest in sorted(self.root.glob("*/project.json")): + try: + data = normalize_project(read_json(manifest, {})) + projects.append(_summary(data, manifest.parent)) + except Exception: + continue + return projects + + def create(self, name: str, metadata: dict[str, Any] | None = None, template: dict[str, Any] | None = None) -> dict[str, Any]: + project = normalize_project(template or default_project(name, metadata)) + project["name"] = name + project["metadata"] = metadata or project.get("metadata", {}) + if not project.get("id"): + project["id"] = new_id("project") + project["slug"] = safe_filename(str(project.get("slug") or name or project["id"])) + project["created_at"] = project.get("created_at") or now() + project["updated_at"] = now() + self.save(project["id"], project) + return project + + def get(self, project_id: str) -> dict[str, Any]: + path = self._path(project_id) + data = read_json(path, None) + if data is None: + raise KeyError(project_id) + return normalize_project(data) + + def save(self, project_id: str, project: dict[str, Any]) -> dict[str, Any]: + normalized = normalize_project(project) + normalized["id"] = project_id or normalized.get("id") or new_id("project") + normalized["slug"] = safe_filename(str(normalized.get("slug") or normalized.get("name") or normalized["id"])) + normalized["updated_at"] = now() + write_json(self._path(normalized["id"]), normalized) + return normalized + + def delete(self, project_id: str) -> None: + path = self._path(project_id) + if not path.exists(): + raise KeyError(project_id) + directory = path.parent + for child in directory.glob("*"): + if child.is_file(): + child.unlink() + try: + directory.rmdir() + except OSError: + pass + + def add_asset(self, project_id: str, asset: dict[str, Any]) -> dict[str, Any]: + project = self.get(project_id) + asset = deepcopy(asset) + asset.setdefault("id", new_id("asset")) + asset.setdefault("created_at", now()) + project["assets"].append(asset) + return self.save(project_id, project) + + def add_to_timeline(self, project_id: str, item: dict[str, Any], track_type: str = "video", track_id: str | None = None) -> dict[str, Any]: + project = self.get(project_id) + add_timeline_item(project, item, track_type=track_type, track_id=track_id) + return self.save(project_id, project) + + def timeline_operation(self, project_id: str, operation: str, item_id: str | None = None, params: dict[str, Any] | None = None) -> dict[str, Any]: + project = self.get(project_id) + apply_timeline_operation(project, operation, item_id=item_id, params=params or {}) + return self.save(project_id, project) + + def _path(self, project_id: str) -> Path: + project_id = safe_filename(project_id) + return self.root / project_id / "project.json" + + +def normalize_project(project: dict[str, Any]) -> dict[str, Any]: + normalized = deepcopy(project or {}) + normalized.setdefault("id", new_id("project")) + normalized.setdefault("name", "Untitled Project") + normalized.setdefault("slug", safe_filename(str(normalized["name"]))) + normalized.setdefault("version", 1) + normalized.setdefault("schema", "ava2lon.project.v1") + normalized.setdefault("created_at", now()) + normalized.setdefault("updated_at", now()) + normalized.setdefault("metadata", {}) + normalized.setdefault("timeline", {}) + timeline = normalized["timeline"] + timeline.setdefault("duration", 0.0) + timeline.setdefault("fps", 30) + timeline.setdefault("tracks", {}) + for track_type in TIMELINE_TRACK_TYPES: + timeline["tracks"].setdefault(track_type, []) + timeline.setdefault("groups", []) + timeline.setdefault("markers", []) + for key in ( + "assets", + "audio_tracks", + "video_tracks", + "text_layers", + "sticker_layers", + "effects", + "filters", + "keyframes", + "captions", + "templates", + "plugins", + ): + normalized.setdefault(key, []) + normalized.setdefault("export_settings", deepcopy(DEFAULT_EXPORT_SETTINGS)) + normalized.setdefault("automation", {"webhooks": [], "batch": {}, "n8n": {"compatible": True}}) + _recalculate_duration(normalized) + return normalized + + +def add_timeline_item(project: dict[str, Any], item: dict[str, Any], *, track_type: str = "video", track_id: str | None = None) -> dict[str, Any]: + if not is_track_type(track_type): + raise ValueError(f"Unsupported track type: {track_type}") + normalized = normalize_project(project) + item = deepcopy(item) + item.setdefault("id", new_id("clip")) + item.setdefault("type", track_type) + item.setdefault("start", 0.0) + item.setdefault("duration", max(float(item.get("end", 0.0)) - float(item.get("start", 0.0)), 0.0) or 1.0) + item.setdefault("source_start", 0.0) + item.setdefault("locked", False) + item.setdefault("hidden", False) + item.setdefault("keyframes", []) + item.setdefault("effects", []) + item.setdefault("filters", []) + item.setdefault("metadata", {}) + track = _ensure_track(normalized, track_type, track_id) + track["items"].append(item) + track["items"].sort(key=lambda entry: float(entry.get("start", 0.0))) + project.clear() + project.update(normalized) + _mirror_layers(project, track_type, item) + _recalculate_duration(project) + return item + + +def apply_timeline_operation(project: dict[str, Any], operation: str, *, item_id: str | None = None, params: dict[str, Any] | None = None) -> dict[str, Any]: + if not is_timeline_operation(operation): + raise ValueError(f"Unsupported timeline operation: {operation}") + params = params or {} + normalized = normalize_project(project) + + if operation == "insert": + add_timeline_item( + normalized, + params.get("item", {}), + track_type=str(params.get("track_type", "video")), + track_id=params.get("track_id"), + ) + elif operation == "group": + group_id = str(params.get("group_id") or new_id("group")) + item_ids = [str(value) for value in params.get("item_ids", [])] + normalized["timeline"]["groups"].append({"id": group_id, "item_ids": item_ids, "metadata": params.get("metadata", {})}) + for grouped_id in item_ids: + try: + grouped_item, _ = _find_item(normalized, grouped_id) + grouped_item["group_id"] = group_id + except KeyError: + continue + else: + if not item_id: + raise ValueError(f"{operation} requires item_id") + item, track = _find_item(normalized, item_id) + if operation == "drag": + item["start"] = max(0.0, float(params.get("start", item.get("start", 0.0)))) + elif operation == "trim": + if "start" in params: + item["start"] = max(0.0, float(params["start"])) + if "duration" in params: + item["duration"] = max(0.001, float(params["duration"])) + if "source_start" in params: + item["source_start"] = max(0.0, float(params["source_start"])) + elif operation == "split": + offset = float(params.get("offset", 0.0)) + duration = float(item.get("duration", 0.0)) + if offset <= 0 or offset >= duration: + raise ValueError("split offset must be inside the item duration") + new_item = deepcopy(item) + new_item["id"] = str(params.get("new_item_id") or new_id("clip")) + new_item["start"] = float(item.get("start", 0.0)) + offset + new_item["duration"] = duration - offset + new_item["source_start"] = float(item.get("source_start", 0.0)) + offset + item["duration"] = offset + track["items"].append(new_item) + track["items"].sort(key=lambda entry: float(entry.get("start", 0.0))) + elif operation == "ripple_delete": + start = float(item.get("start", 0.0)) + duration = float(item.get("duration", 0.0)) + track["items"] = [entry for entry in track["items"] if entry.get("id") != item_id] + for entry in track["items"]: + if float(entry.get("start", 0.0)) > start: + entry["start"] = max(start, float(entry.get("start", 0.0)) - duration) + elif operation == "replace": + replacement = deepcopy(params.get("item", {})) + replacement.setdefault("id", item_id) + replacement.setdefault("start", item.get("start", 0.0)) + replacement.setdefault("duration", item.get("duration", 1.0)) + replacement.setdefault("type", item.get("type", track.get("type"))) + index = track["items"].index(item) + track["items"][index] = replacement + elif operation == "lock": + item["locked"] = bool(params.get("locked", True)) + elif operation == "hide": + item["hidden"] = bool(params.get("hidden", True)) + elif operation == "duplicate": + duplicate = deepcopy(item) + duplicate["id"] = str(params.get("new_item_id") or new_id("clip")) + duplicate["start"] = float(params.get("start", float(item.get("start", 0.0)) + float(item.get("duration", 1.0)))) + track["items"].append(duplicate) + track["items"].sort(key=lambda entry: float(entry.get("start", 0.0))) + + project.clear() + project.update(normalized) + _recalculate_duration(project) + return project + + +def add_effect(project: dict[str, Any], target_id: str, effect: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + effect_record = {"id": new_id("effect"), "target_id": target_id, "effect": effect, "params": params or {}, "created_at": now()} + project.setdefault("effects", []).append(effect_record) + try: + item, _ = _find_item(project, target_id) + item.setdefault("effects", []).append(effect_record) + except KeyError: + pass + return effect_record + + +def add_filter(project: dict[str, Any], target_id: str, filter_name: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + filter_record = {"id": new_id("filter"), "target_id": target_id, "filter": filter_name, "params": params or {}, "created_at": now()} + project.setdefault("filters", []).append(filter_record) + try: + item, _ = _find_item(project, target_id) + item.setdefault("filters", []).append(filter_record) + except KeyError: + pass + return filter_record + + +def add_transition(project: dict[str, Any], from_item_id: str, to_item_id: str, transition: str, duration: float = 0.45) -> dict[str, Any]: + record = { + "id": new_id("transition"), + "from_item_id": from_item_id, + "to_item_id": to_item_id, + "transition": transition, + "duration": duration, + "created_at": now(), + } + project.setdefault("timeline", {}).setdefault("transitions", []).append(record) + return record + + +def add_keyframe( + project: dict[str, Any], + target_id: str, + property_name: str, + time: float, + value: Any, + easing: str = "linear", +) -> dict[str, Any]: + record = { + "id": new_id("keyframe"), + "target_id": target_id, + "property": property_name, + "time": max(0.0, float(time)), + "value": value, + "easing": easing, + } + project.setdefault("keyframes", []).append(record) + try: + item, _ = _find_item(project, target_id) + item.setdefault("keyframes", []).append(record) + except KeyError: + pass + return record + + +def _ensure_track(project: dict[str, Any], track_type: str, track_id: str | None = None) -> dict[str, Any]: + tracks = project["timeline"]["tracks"].setdefault(track_type, []) + if track_id: + for track in tracks: + if track.get("id") == track_id: + return track + if not tracks: + track_id = track_id or f"{track_type}_1" + else: + track_id = track_id or f"{track_type}_{len(tracks) + 1}" + track = {"id": track_id, "type": track_type, "name": f"{track_type.title()} {len(tracks) + 1}", "locked": False, "hidden": False, "items": []} + tracks.append(track) + return track + + +def _find_item(project: dict[str, Any], item_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + for tracks in project.get("timeline", {}).get("tracks", {}).values(): + for track in tracks: + for item in track.get("items", []): + if item.get("id") == item_id: + return item, track + raise KeyError(item_id) + + +def _mirror_layers(project: dict[str, Any], track_type: str, item: dict[str, Any]) -> None: + mirror_key = { + "video": "video_tracks", + "audio": "audio_tracks", + "text": "text_layers", + "sticker": "sticker_layers", + "subtitle": "captions", + }.get(track_type) + if mirror_key: + project.setdefault(mirror_key, []).append({"item_id": item["id"], **deepcopy(item)}) + + +def _recalculate_duration(project: dict[str, Any]) -> None: + duration = 0.0 + for tracks in project.get("timeline", {}).get("tracks", {}).values(): + for track in tracks: + for item in track.get("items", []): + duration = max(duration, float(item.get("start", 0.0)) + float(item.get("duration", 0.0))) + project.setdefault("timeline", {})["duration"] = round(duration, 3) + + +def _summary(project: dict[str, Any], directory: Path) -> dict[str, Any]: + return { + "id": project.get("id"), + "name": project.get("name"), + "slug": project.get("slug"), + "path": str(directory), + "updated_at": project.get("updated_at"), + "duration": project.get("timeline", {}).get("duration", 0.0), + "asset_count": len(project.get("assets", [])), + "metadata": project.get("metadata", {}), + } diff --git a/services/render_engine/renderer/studio/tasks.py b/services/render_engine/renderer/studio/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..1e79cf6241054ee2b171ddc0b3218a09be10b6eb --- /dev/null +++ b/services/render_engine/renderer/studio/tasks.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from renderer.core.config import Settings +from renderer.core.models import TaskResult +from renderer.core.utils import now, safe_filename, write_json +from renderer.studio.capabilities import ( + AI_ASSISTANTS, + AI_EDITING_FEATURES, + IMAGE_GENERATION_PROVIDERS, + MUSIC_PROVIDERS, + VIDEO_GENERATION_PROVIDERS, + VOICE_PROVIDERS, +) + + +class StudioTaskProcessor: + """Manifest-producing async handlers for optional AI providers and studio automation.""" + + def __init__(self, settings: Settings | None = None, log=None) -> None: + self.settings = settings or Settings() + self.settings.ensure_dirs() + self._logs: list[str] = [] + self._log = log + + def caption_generate(self, payload: dict[str, Any], job_id: str) -> TaskResult: + text = str(payload.get("text") or payload.get("transcript") or "") + captions = payload.get("events") if isinstance(payload.get("events"), list) else _captions_from_text(text) + manifest = { + "type": "caption_generation", + "status": "ready", + "engine": payload.get("engine", "whisper"), + "media": payload.get("media") or payload.get("audio"), + "template": payload.get("template", "capcut"), + "language": payload.get("language"), + "features": { + "word_timestamps": bool(payload.get("word_timestamps", True)), + "sentence_timestamps": True, + "emoji_insertion": bool(payload.get("emoji_insertion", False)), + "speaker_detection": bool(payload.get("speaker_detection", False)), + "karaoke": bool(payload.get("karaoke", True)), + "animated": bool(payload.get("animated", True)), + }, + "captions": captions, + } + output = self._json_artifact(job_id, "captions", manifest) + return self._result(output, {"task": "caption_generate", "caption_count": len(captions)}) + + def music_generate(self, payload: dict[str, Any], job_id: str) -> TaskResult: + provider = _provider(payload.get("provider"), MUSIC_PROVIDERS, "musicgen") + prompt = str(payload.get("prompt") or payload.get("style") or "background music") + duration = float(payload.get("duration", 30)) + manifest = { + "type": "music_generation", + "status": "provider_required", + "provider": provider, + "prompt": prompt, + "style": payload.get("style", "background_music"), + "duration": duration, + "bpm": payload.get("bpm"), + "license": payload.get("license", "user_configured"), + "next_step": "Configure provider credentials or connect this manifest to a local MusicGen runner.", + } + output = self._json_artifact(job_id, "music_request", manifest) + return self._result(output, {"task": "music_generate", "provider": provider, "duration": duration}) + + def voice_generate(self, payload: dict[str, Any], job_id: str) -> TaskResult: + provider = _provider(payload.get("provider"), VOICE_PROVIDERS, "kokoro") + text = str(payload.get("text") or "") + manifest = { + "type": "voice_generation", + "status": "provider_required", + "provider": provider, + "text": text, + "voice": payload.get("voice", "default"), + "emotion": payload.get("emotion"), + "speed": float(payload.get("speed", 1.0)), + "pitch": float(payload.get("pitch", 1.0)), + "clone_reference": payload.get("clone_reference"), + "multi_speaker": payload.get("speakers", []), + "next_step": "Configure the selected TTS backend to render audio for this manifest.", + } + output = self._json_artifact(job_id, "voice_request", manifest) + return self._result(output, {"task": "voice_generate", "provider": provider, "characters": len(text)}) + + def image_generate(self, payload: dict[str, Any], job_id: str) -> TaskResult: + provider = _provider(payload.get("provider"), IMAGE_GENERATION_PROVIDERS, "flux") + manifest = { + "type": "image_generation", + "status": "provider_required", + "provider": provider, + "prompt": payload.get("prompt", ""), + "negative_prompt": payload.get("negative_prompt", ""), + "mode": payload.get("mode", "text_to_image"), + "control_image": payload.get("control_image"), + "source_image": payload.get("source_image"), + "size": payload.get("size", "1024x1024"), + "features": { + "background_replacement": bool(payload.get("background_replacement", False)), + "object_removal": bool(payload.get("object_removal", False)), + "upscaling": bool(payload.get("upscaling", False)), + }, + } + output = self._json_artifact(job_id, "image_request", manifest) + return self._result(output, {"task": "image_generate", "provider": provider}) + + def video_generate(self, payload: dict[str, Any], job_id: str) -> TaskResult: + provider = _provider(payload.get("provider"), VIDEO_GENERATION_PROVIDERS, "ltx_video") + manifest = { + "type": "video_generation", + "status": "provider_required", + "provider": provider, + "prompt": payload.get("prompt", ""), + "mode": payload.get("mode", "text_to_video"), + "image": payload.get("image"), + "duration": float(payload.get("duration", 5)), + "fps": int(payload.get("fps", 24)), + "size": payload.get("size", "1280x720"), + "next_step": "Connect Wan, LTX Video, Hunyuan Video, or Veo credentials/runtime to execute this request.", + } + output = self._json_artifact(job_id, "video_request", manifest) + return self._result(output, {"task": "video_generate", "provider": provider}) + + def ai_tool(self, tool: str, payload: dict[str, Any], job_id: str) -> TaskResult: + tool = _canonical(tool) + if tool not in AI_EDITING_FEATURES: + raise ValueError(f"Unsupported AI editing tool: {tool}") + manifest = { + "type": "ai_editing", + "tool": tool, + "status": "ready", + "media": payload.get("media"), + "project_id": payload.get("project_id"), + "platform": payload.get("platform", "tiktok"), + "result": _ai_result(tool, payload), + "created_at": now(), + } + output = self._json_artifact(job_id, tool, manifest) + return self._result(output, {"task": tool}) + + def assistant_tool(self, tool: str, payload: dict[str, Any], job_id: str) -> TaskResult: + tool = _canonical(tool) + if tool not in AI_ASSISTANTS: + raise ValueError(f"Unsupported assistant: {tool}") + text = str(payload.get("topic") or payload.get("transcript") or payload.get("prompt") or "") + manifest = { + "type": "assistant", + "tool": tool, + "status": "ready", + "input": text, + "platform": payload.get("platform", "general"), + "result": _assistant_result(tool, text, payload), + "created_at": now(), + } + output = self._json_artifact(job_id, tool, manifest) + return self._result(output, {"task": tool, "characters": len(text)}) + + def _json_artifact(self, job_id: str, name: str, payload: dict[str, Any]) -> Path: + output = self.settings.exports_dir / f"{job_id}_{safe_filename(name)}.json" + write_json(output, payload) + self._message(f"Wrote {name} manifest") + return output + + def _result(self, output: Path, metrics: dict[str, Any]) -> TaskResult: + return TaskResult(output_path=output, commands=[], metrics=metrics, logs=list(self._logs)) + + def _message(self, message: str) -> None: + self._logs.append(message) + if self._log: + self._log(message) + + +def _provider(value: Any, supported: list[str], default: str) -> str: + provider = _canonical(str(value or default)) + return provider if provider in supported else default + + +def _canonical(value: str) -> str: + return value.strip().lower().replace("-", "_").replace(" ", "_") + + +def _captions_from_text(text: str) -> list[dict[str, Any]]: + if not text: + return [] + words = text.split() + chunks: list[list[str]] = [] + while words: + chunks.append(words[:8]) + words = words[8:] + captions = [] + cursor = 0.0 + for chunk in chunks: + duration = max(1.2, len(chunk) * 0.34) + captions.append({"start": round(cursor, 2), "end": round(cursor + duration, 2), "text": " ".join(chunk)}) + cursor += duration + return captions + + +def _ai_result(tool: str, payload: dict[str, Any]) -> dict[str, Any]: + platform = str(payload.get("platform") or "tiktok") + if tool == "auto_highlight_detection": + return {"highlights": [{"start": 0, "end": 8, "reason": "opening hook"}]} + if tool == "auto_scene_detection": + return {"scenes": [{"start": 0, "end": 5, "label": "intro"}, {"start": 5, "end": 12, "label": "body"}]} + if tool in {"auto_reframe", "auto_crop", "auto_platform_optimization"}: + return {"platform": platform, "safe_zone": "vertical_center", "aspect_ratio": "9:16"} + if tool == "auto_viral_score": + return {"score": 74, "signals": ["short duration", "caption-ready", platform]} + if tool == "auto_hook_detection": + return {"hook": str(payload.get("transcript") or payload.get("text") or "")[:120], "score": 68} + if tool == "auto_thumbnail_selection": + return {"frames": [{"timestamp": 2.0, "score": 82}, {"timestamp": 6.5, "score": 75}]} + return {"plan": f"{tool} plan generated", "confidence": "heuristic", "platform": platform} + + +def _assistant_result(tool: str, text: str, payload: dict[str, Any]) -> dict[str, Any]: + subject = text.strip() or "your video" + short = " ".join(subject.split()[:12]) + if tool == "script_writer": + return {"script": f"Hook: {short}\nValue: show the clearest proof.\nCTA: invite viewers to take the next step."} + if tool == "hook_generator": + return {"hooks": [f"Stop scrolling if you care about {short}", f"Nobody tells you this about {short}"]} + if tool == "title_generator": + return {"titles": [short.title(), f"How {short.title()} Changes Everything"]} + if tool == "description_generator": + return {"description": f"{subject}\n\nBuilt with Ava2lon Studio AI."} + if tool == "hashtag_generator": + tags = [word.strip(".,!?").lower() for word in subject.split() if len(word.strip(".,!?")) > 3] + return {"hashtags": ["#" + tag for tag in tags[:8]] or ["#video", "#creator"]} + if tool == "storyboard_generator": + return {"beats": [{"scene": 1, "goal": "hook"}, {"scene": 2, "goal": "proof"}, {"scene": 3, "goal": "CTA"}]} + if tool == "b_roll_planner": + return {"shots": [{"type": "close_up", "description": short}, {"type": "screen_recording", "description": "show the result"}]} + if tool == "thumbnail_prompt_generator": + return {"prompt": f"High contrast thumbnail for {short}, expressive face, bold text, clean background"} + if tool == "seo_optimizer": + return {"keywords": [word.strip(".,!?").lower() for word in subject.split()[:10]], "score": 72} + return {"result": subject, "options": payload} diff --git a/services/render_engine/renderer/subtitles/__init__.py b/services/render_engine/renderer/subtitles/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b349b5fa65990b05984c46c1fd2632dc7ca12f74 --- /dev/null +++ b/services/render_engine/renderer/subtitles/__init__.py @@ -0,0 +1,3 @@ +from renderer.subtitles.generator import SubtitleEvent, SubtitleGenerator + +__all__ = ["SubtitleEvent", "SubtitleGenerator"] diff --git a/services/render_engine/renderer/subtitles/generator.py b/services/render_engine/renderer/subtitles/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..1d6b0942e9c74adf9bc906ec033901660d160263 --- /dev/null +++ b/services/render_engine/renderer/subtitles/generator.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import html +from dataclasses import dataclass +from pathlib import Path + +from renderer.templates import get_template + + +@dataclass +class SubtitleEvent: + start: float + end: float + text: str + + +class SubtitleGenerator: + def from_scenes(self, scenes: list, total_duration: float | None = None) -> list[SubtitleEvent]: + events: list[SubtitleEvent] = [] + for scene in scenes: + if scene.caption: + events.append(SubtitleEvent(scene.start, scene.start + scene.duration, scene.caption)) + if not events and total_duration: + events.append(SubtitleEvent(0, total_duration, "")) + return events + + def write_srt(self, events: list[SubtitleEvent], output: Path) -> Path: + lines: list[str] = [] + for idx, event in enumerate(events, start=1): + lines.extend([str(idx), f"{_srt_time(event.start)} --> {_srt_time(event.end)}", event.text, ""]) + output.write_text("\n".join(lines), encoding="utf-8") + return output + + def write_ass(self, events: list[SubtitleEvent], output: Path, template_key: str) -> Path: + template = get_template(template_key) + body = [ + "[Script Info]", + "ScriptType: v4.00+", + "PlayResX: 1080", + "PlayResY: 1920", + "", + "[V4+ Styles]", + "Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,OutlineColour,BackColour," + "Bold,Italic,Underline,StrikeOut,ScaleX,ScaleY,Spacing,Angle,BorderStyle,Outline,Shadow," + "Alignment,MarginL,MarginR,MarginV,Encoding", + template.ass_style(), + "", + "[Events]", + "Format: Layer,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text", + ] + for event in events: + text = _ass_escape(event.text) + if template.effect == "karaoke": + text = _karaoke_text(text, event.end - event.start) + elif template.effect == "zoom": + text = r"{\t(0,180,\fscx115\fscy115)\t(180,360,\fscx100\fscy100)}" + text + elif template.effect == "bounce": + text = r"{\t(0,120,\frz-2)\t(120,240,\frz2)\t(240,360,\frz0)}" + text + body.append(f"Dialogue: 0,{_ass_time(event.start)},{_ass_time(event.end)},Default,,0,0,0,,{text}") + output.write_text("\n".join(body), encoding="utf-8") + return output + + +def _karaoke_text(text: str, duration: float) -> str: + words = text.split() + if not words: + return text + centiseconds = max(1, int(duration * 100 / len(words))) + return "".join(f"{{\\k{centiseconds}}}{word} " for word in words).strip() + + +def _srt_time(seconds: float) -> str: + ms = int(round(seconds * 1000)) + h, rem = divmod(ms, 3600000) + m, rem = divmod(rem, 60000) + s, ms = divmod(rem, 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + +def _ass_time(seconds: float) -> str: + cs = int(round(seconds * 100)) + h, rem = divmod(cs, 360000) + m, rem = divmod(rem, 6000) + s, cs = divmod(rem, 100) + return f"{h}:{m:02d}:{s:02d}.{cs:02d}" + + +def _ass_escape(text: str) -> str: + return html.escape(text).replace("\n", r"\N").replace("{", r"\{").replace("}", r"\}") diff --git a/services/render_engine/renderer/templates/__init__.py b/services/render_engine/renderer/templates/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..90fb97acee86b43aa1dd1df2c9ff1f40c8c11e6d --- /dev/null +++ b/services/render_engine/renderer/templates/__init__.py @@ -0,0 +1,31 @@ +from renderer.templates.caption_templates import CaptionTemplate, get_template, list_templates +from renderer.templates.creative import ( + apply_creative_style, + creative_style_metadata, + get_creative_style, + list_creative_styles, + list_scene_effects, + scene_effect_filter, + scene_effect_metadata, +) +from renderer.templates.platforms import PlatformProfile, get_platform_profile, list_platform_profiles, platform_profile_metadata +from renderer.templates.presets import apply_preset, list_presets + +__all__ = [ + "CaptionTemplate", + "PlatformProfile", + "apply_creative_style", + "apply_preset", + "creative_style_metadata", + "get_creative_style", + "get_platform_profile", + "get_template", + "list_creative_styles", + "list_platform_profiles", + "list_scene_effects", + "list_presets", + "list_templates", + "platform_profile_metadata", + "scene_effect_filter", + "scene_effect_metadata", +] diff --git a/services/render_engine/renderer/templates/caption_templates.py b/services/render_engine/renderer/templates/caption_templates.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2da9e2b5a0207b4619158b680ac852e59d5644 --- /dev/null +++ b/services/render_engine/renderer/templates/caption_templates.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CaptionTemplate: + key: str + label: str + font_size: int + primary_color: str + secondary_color: str + outline_color: str = "&H000000" + back_color: str = "&H80000000" + alignment: int = 2 + margin_v: int = 180 + bold: bool = True + effect: str = "none" + + def ass_style(self) -> str: + bold = -1 if self.bold else 0 + return ( + "Style: Default,DejaVu Sans," + f"{self.font_size},{self.primary_color},{self.secondary_color}," + f"{self.outline_color},{self.back_color},{bold},0,0,0,100,100,0,0,3,3,1," + f"{self.alignment},80,80,{self.margin_v},1" + ) + + +TEMPLATES: dict[str, CaptionTemplate] = { + "tiktok_classic": CaptionTemplate("tiktok_classic", "TikTok Classic", 64, "&H00FFFFFF", "&H0000FFFF", effect="karaoke"), + "tiktok_zoom": CaptionTemplate("tiktok_zoom", "TikTok Zoom", 72, "&H00FFFFFF", "&H0000E5FF", effect="zoom"), + "alex_hormozi": CaptionTemplate("alex_hormozi", "Alex Hormozi", 70, "&H0000FFFF", "&H00FFFFFF", effect="bounce"), + "modern_minimal": CaptionTemplate("modern_minimal", "Modern Minimal", 52, "&H00FFFFFF", "&H00DDDDDD", margin_v=240), + "youtube_shorts": CaptionTemplate("youtube_shorts", "YouTube Shorts", 62, "&H00FFFFFF", "&H000000FF", effect="karaoke"), + "podcast_style": CaptionTemplate("podcast_style", "Podcast Style", 48, "&H00F5F5F5", "&H0099CCFF", margin_v=120), + "news_style": CaptionTemplate("news_style", "News Style", 46, "&H00FFFFFF", "&H0000FFFF", alignment=2, margin_v=100), + "neon_pop": CaptionTemplate("neon_pop", "Neon Pop", 76, "&H00FFFFFF", "&H0000E5FF", outline_color="&H00FF2BD6", effect="karaoke"), + "product_demo": CaptionTemplate("product_demo", "Product Demo", 54, "&H00FFFFFF", "&H00C7F9CC", margin_v=210, effect="zoom"), + "cinematic_gold": CaptionTemplate("cinematic_gold", "Cinematic Gold", 50, "&H00F4E7B2", "&H00FFFFFF", outline_color="&H00111111", margin_v=180), + "creator_clean": CaptionTemplate("creator_clean", "Creator Clean", 58, "&H00FFFFFF", "&H00BCE7FD", margin_v=220, effect="bounce"), +} + + +def get_template(key: str) -> CaptionTemplate: + return TEMPLATES.get(key, TEMPLATES["tiktok_classic"]) + + +def list_templates() -> list[str]: + return list(TEMPLATES.keys()) diff --git a/services/render_engine/renderer/templates/creative.py b/services/render_engine/renderer/templates/creative.py new file mode 100644 index 0000000000000000000000000000000000000000..319acba7f2540b992422021ae3a02c50df59bc14 --- /dev/null +++ b/services/render_engine/renderer/templates/creative.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class CreativeStyle: + key: str + label: str + description: str + platform: str + template: str + scene_duration: float + transition_sequence: tuple[str, ...] + scene_effect_sequence: tuple[str, ...] + render_defaults: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + + def metadata_payload(self) -> dict[str, Any]: + return { + "key": self.key, + "label": self.label, + "description": self.description, + "platform": self.platform, + "template": self.template, + "scene_duration": self.scene_duration, + "transition_sequence": list(self.transition_sequence), + "scene_effect_sequence": list(self.scene_effect_sequence), + "render_defaults": deepcopy(self.render_defaults), + "metadata": deepcopy(self.metadata), + } + + +SCENE_EFFECTS: dict[str, dict[str, str]] = { + "none": {"label": "None", "filter": ""}, + "sharp_pop": { + "label": "Sharp Pop", + "filter": "eq=contrast=1.08:saturation=1.20:brightness=0.01,unsharp=5:5:0.8:3:3:0.4", + }, + "clean_beauty": { + "label": "Clean Beauty", + "filter": "hqdn3d=1.5:1.5:6:6,eq=saturation=1.08:contrast=1.03", + }, + "warm_glow": { + "label": "Warm Glow", + "filter": "eq=contrast=1.04:saturation=1.18:gamma_r=1.04:gamma_b=0.96,gblur=sigma=0.25", + }, + "cinematic": { + "label": "Cinematic", + "filter": "eq=contrast=1.14:saturation=0.95:brightness=-0.015,vignette=PI/6", + }, + "dreamy": { + "label": "Dreamy", + "filter": "gblur=sigma=0.6,eq=contrast=1.04:saturation=1.18:brightness=0.02", + }, + "flash_pop": { + "label": "Flash Pop", + "filter": "eq=contrast=1.16:saturation=1.25:brightness=0.035", + }, + "grain": { + "label": "Fine Grain", + "filter": "noise=alls=8:allf=t+u,eq=contrast=1.07:saturation=1.02", + }, + "motion_blur": { + "label": "Motion Blur", + "filter": "tmix=frames=3:weights='1 2 1',eq=contrast=1.05:saturation=1.08", + }, + "noir": { + "label": "Noir", + "filter": "hue=s=0,eq=contrast=1.18:brightness=-0.02", + }, + "glitch": { + "label": "Glitch", + "filter": "rgbashift=rh=4:bh=-4,eq=contrast=1.12:saturation=1.18", + }, + "rgb_split": { + "label": "RGB Split", + "filter": "rgbashift=rh=3:gv=1:bh=-3", + }, + "vhs": { + "label": "VHS", + "filter": "noise=alls=18:allf=t+u,eq=saturation=0.82:contrast=1.08", + }, + "crt": { + "label": "CRT", + "filter": "vignette=PI/4,noise=alls=10:allf=t+u,eq=contrast=1.15:saturation=0.9", + }, + "bloom": { + "label": "Bloom", + "filter": "gblur=sigma=0.35,eq=brightness=0.025:saturation=1.14", + }, + "glow": { + "label": "Glow", + "filter": "gblur=sigma=0.45,eq=contrast=1.05:brightness=0.03", + }, + "chromatic_aberration": { + "label": "Chromatic Aberration", + "filter": "rgbashift=rh=2:rv=1:bh=-2:bv=-1", + }, + "neon": { + "label": "Neon", + "filter": "eq=contrast=1.2:saturation=1.55:brightness=0.02", + }, + "cyberpunk": { + "label": "Cyberpunk", + "filter": "eq=contrast=1.18:saturation=1.45:gamma_r=1.08:gamma_b=1.18", + }, + "comic": { + "label": "Comic", + "filter": "edgedetect=low=0.08:high=0.25,eq=contrast=1.2:saturation=1.35", + }, + "cartoon": { + "label": "Cartoon", + "filter": "edgedetect=low=0.05:high=0.2,eq=saturation=1.45:contrast=1.15", + }, + "anime": { + "label": "Anime", + "filter": "eq=saturation=1.35:contrast=1.12:brightness=0.02,unsharp=5:5:0.6", + }, + "sketch": { + "label": "Sketch", + "filter": "edgedetect=low=0.03:high=0.18,hue=s=0", + }, + "oil_painting": { + "label": "Oil Painting", + "filter": "gblur=sigma=0.7,eq=saturation=1.25:contrast=1.1", + }, + "pixel_art": { + "label": "Pixel Art", + "filter": "scale=iw/8:ih/8,scale=iw*8:ih*8:flags=neighbor", + }, +} + + +CREATIVE_STYLES: dict[str, CreativeStyle] = { + "viral_shorts": CreativeStyle( + key="viral_shorts", + label="Viral Shorts", + description="Fast vertical pacing, punchy captions, bright contrast, and whip-style movement.", + platform="tiktok", + template="tiktok_zoom", + scene_duration=2.4, + transition_sequence=("whip", "flash", "zoom", "glitch"), + scene_effect_sequence=("sharp_pop", "flash_pop", "clean_beauty"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.24, + "music_fade_in": 0.25, + "music_fade_out": 0.8, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "high", "best_for": "hooks, offers, memes, short promos"}, + ), + "product_demo": CreativeStyle( + key="product_demo", + label="Product Demo", + description="Clean cuts, readable captions, and polished color for launches and tutorials.", + platform="instagram_reels", + template="modern_minimal", + scene_duration=3.2, + transition_sequence=("slide", "wipe_left", "push", "dissolve"), + scene_effect_sequence=("clean_beauty", "sharp_pop"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.18, + "music_fade_in": 0.3, + "music_fade_out": 0.7, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "medium", "best_for": "software demos, ecommerce, tutorials"}, + ), + "story_vlog": CreativeStyle( + key="story_vlog", + label="Story Vlog", + description="Warm color, softer movement, and natural pacing for personality-led edits.", + platform="instagram_reels", + template="tiktok_classic", + scene_duration=3.8, + transition_sequence=("fade", "smooth_right", "dissolve"), + scene_effect_sequence=("warm_glow", "dreamy", "clean_beauty"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.22, + "music_fade_in": 0.5, + "music_fade_out": 1.0, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "medium", "best_for": "founder updates, day-in-life edits, testimonials"}, + ), + "podcast_clip": CreativeStyle( + key="podcast_clip", + label="Podcast Clip", + description="Square-safe framing, calmer captions, and narration-first audio treatment.", + platform="instagram_feed_square", + template="podcast_style", + scene_duration=5.0, + transition_sequence=("fade", "dissolve"), + scene_effect_sequence=("clean_beauty", "sharp_pop"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.12, + "music_fade_in": 0.6, + "music_fade_out": 1.2, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "low", "best_for": "interviews, audiograms, education"}, + ), + "cinematic_story": CreativeStyle( + key="cinematic_story", + label="Cinematic Story", + description="Deeper contrast, film grain, and slower transitions for mini-documentary edits.", + platform="youtube_shorts", + template="modern_minimal", + scene_duration=4.2, + transition_sequence=("dissolve", "fadeblack", "smooth_left"), + scene_effect_sequence=("cinematic", "grain"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.26, + "music_fade_in": 0.8, + "music_fade_out": 1.4, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "low", "best_for": "brand films, travel, documentary shorts"}, + ), + "news_explainer": CreativeStyle( + key="news_explainer", + label="News Explainer", + description="Readable lower-third style captions with stable landscape or vertical exports.", + platform="youtube_1080p", + template="news_style", + scene_duration=4.0, + transition_sequence=("wipe_left", "slide", "fade"), + scene_effect_sequence=("sharp_pop", "clean_beauty"), + render_defaults={ + "subtitle_format": "ass", + "auto_subtitles": False, + "audio_normalize": True, + "music_volume": 0.1, + "music_fade_in": 0.5, + "music_fade_out": 1.0, + "music_ducking": True, + "normalize": True, + }, + metadata={"energy": "medium", "best_for": "explainers, news, thought leadership"}, + ), +} + + +def apply_creative_style(payload: dict[str, Any]) -> dict[str, Any]: + style_key = payload.get("creative_style") or payload.get("metadata", {}).get("creative_style") + if not style_key: + return payload + + style = get_creative_style(str(style_key)) + output = deepcopy(payload) + output["creative_style"] = style.key + _set_default(output, "platform", style.platform) + _set_default(output, "template", style.template) + for key, value in style.render_defaults.items(): + _set_default(output, key, deepcopy(value)) + + metadata = deepcopy(style.metadata) + metadata.update(output.get("metadata", {})) + metadata["creative_style"] = style.key + metadata["creative_style_label"] = style.label + output["metadata"] = metadata + + scenes = output.get("scenes") + if isinstance(scenes, list): + output["scenes"] = [_style_scene(scene, style, index) for index, scene in enumerate(scenes)] + return output + + +def get_creative_style(key: str | None) -> CreativeStyle: + if not key: + return CREATIVE_STYLES["viral_shorts"] + return CREATIVE_STYLES.get(key, CREATIVE_STYLES["viral_shorts"]) + + +def list_creative_styles() -> list[str]: + return list(CREATIVE_STYLES.keys()) + + +def creative_style_metadata() -> dict[str, dict[str, Any]]: + return {key: style.metadata_payload() for key, style in CREATIVE_STYLES.items()} + + +def list_scene_effects() -> list[str]: + return list(SCENE_EFFECTS.keys()) + + +def scene_effect_metadata() -> dict[str, dict[str, str]]: + return {key: {"label": value["label"]} for key, value in SCENE_EFFECTS.items()} + + +def scene_effect_filter(key: str | None) -> str: + if not key: + return "" + return SCENE_EFFECTS.get(key, SCENE_EFFECTS["none"])["filter"] + + +def _set_default(payload: dict[str, Any], key: str, value: Any) -> None: + if key not in payload or payload[key] in (None, ""): + payload[key] = value + + +def _style_scene(scene: Any, style: CreativeStyle, index: int) -> Any: + if not isinstance(scene, dict): + return scene + styled = deepcopy(scene) + transition = styled.get("transition") + if transition in (None, "", "fade"): + styled["transition"] = style.transition_sequence[index % len(style.transition_sequence)] + if styled.get("effect") in (None, ""): + styled["effect"] = style.scene_effect_sequence[index % len(style.scene_effect_sequence)] + _set_default(styled, "background", "blur") + _set_default(styled, "layout", "fill") + return styled diff --git a/services/render_engine/renderer/templates/platforms.py b/services/render_engine/renderer/templates/platforms.py new file mode 100644 index 0000000000000000000000000000000000000000..82a1f0eae130237b104388e31fe28298d1e7cc8b --- /dev/null +++ b/services/render_engine/renderer/templates/platforms.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class PlatformProfile: + key: str + label: str + width: int + height: int + fps: int = 30 + video_codec: str = "libx264" + audio_codec: str = "aac" + audio_bitrate: str = "128k" + audio_sample_rate: int = 48000 + crf: int = 23 + maxrate: str | None = None + bufsize: str | None = None + max_duration_seconds: int | None = None + recommended_duration_seconds: tuple[int, int] | None = None + safe_zones: dict[str, int] = field(default_factory=dict) + notes: tuple[str, ...] = () + + @property + def aspect_ratio(self) -> str: + return f"{self.width}:{self.height}" + + def metadata(self) -> dict[str, Any]: + return { + "platform": self.key, + "label": self.label, + "width": self.width, + "height": self.height, + "fps": self.fps, + "aspect_ratio": self.aspect_ratio, + "video_codec": self.video_codec, + "audio_codec": self.audio_codec, + "audio_bitrate": self.audio_bitrate, + "max_duration_seconds": self.max_duration_seconds, + "recommended_duration_seconds": self.recommended_duration_seconds, + "safe_zones": self.safe_zones, + "notes": list(self.notes), + } + + +COMMON_VERTICAL_SAFE_ZONES = { + "top_px": 220, + "bottom_px": 340, + "left_px": 80, + "right_px": 80, +} + + +PLATFORM_PROFILES: dict[str, PlatformProfile] = { + "tiktok": PlatformProfile( + key="tiktok", + label="TikTok vertical", + width=1080, + height=1920, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=600, + recommended_duration_seconds=(12, 60), + safe_zones=COMMON_VERTICAL_SAFE_ZONES, + notes=( + "9:16 vertical MP4 keeps the frame full-screen in the For You feed.", + "Use licensed or original music to avoid muted audio or takedowns.", + ), + ), + "instagram_reels": PlatformProfile( + key="instagram_reels", + label="Instagram Reels", + width=1080, + height=1920, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=180, + recommended_duration_seconds=(7, 90), + safe_zones=COMMON_VERTICAL_SAFE_ZONES, + notes=( + "9:16 is the safest Reels export to avoid cropping or blank space.", + "Keep captions and logos away from top and bottom app chrome.", + ), + ), + "facebook_reels": PlatformProfile( + key="facebook_reels", + label="Facebook Reels", + width=1080, + height=1920, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=180, + recommended_duration_seconds=(7, 90), + safe_zones=COMMON_VERTICAL_SAFE_ZONES, + ), + "youtube_shorts": PlatformProfile( + key="youtube_shorts", + label="YouTube Shorts", + width=1080, + height=1920, + fps=30, + audio_bitrate="192k", + maxrate="10M", + bufsize="20M", + max_duration_seconds=180, + recommended_duration_seconds=(15, 60), + safe_zones={"top_px": 180, "bottom_px": 300, "left_px": 80, "right_px": 80}, + notes=( + "YouTube categorizes square or vertical videos up to 3 minutes as Shorts.", + "Avoid Content ID-claimed music in Shorts longer than 60 seconds.", + ), + ), + "youtube_1080p": PlatformProfile( + key="youtube_1080p", + label="YouTube 1080p landscape", + width=1920, + height=1080, + fps=30, + audio_bitrate="192k", + maxrate="12M", + bufsize="24M", + recommended_duration_seconds=(60, 600), + notes=("16:9 H.264/AAC output for standard YouTube uploads.",), + ), + "instagram_feed_square": PlatformProfile( + key="instagram_feed_square", + label="Instagram feed square", + width=1080, + height=1080, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=3600, + recommended_duration_seconds=(5, 60), + safe_zones={"top_px": 80, "bottom_px": 120, "left_px": 80, "right_px": 80}, + ), + "instagram_feed_portrait": PlatformProfile( + key="instagram_feed_portrait", + label="Instagram feed portrait", + width=1080, + height=1350, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=3600, + recommended_duration_seconds=(5, 60), + safe_zones={"top_px": 80, "bottom_px": 140, "left_px": 80, "right_px": 80}, + ), + "snapchat_spotlight": PlatformProfile( + key="snapchat_spotlight", + label="Snapchat Spotlight", + width=1080, + height=1920, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=60, + recommended_duration_seconds=(5, 30), + safe_zones=COMMON_VERTICAL_SAFE_ZONES, + notes=("Vertical, fast-paced edits perform best in Spotlight.",), + ), + "pinterest_idea_pins": PlatformProfile( + key="pinterest_idea_pins", + label="Pinterest Idea Pins", + width=1080, + height=1920, + fps=30, + maxrate="8M", + bufsize="16M", + max_duration_seconds=300, + recommended_duration_seconds=(6, 45), + safe_zones={"top_px": 160, "bottom_px": 240, "left_px": 80, "right_px": 80}, + notes=("Use clear text overlays and evergreen discovery keywords.",), + ), + "linkedin_video": PlatformProfile( + key="linkedin_video", + label="LinkedIn video", + width=1920, + height=1080, + fps=30, + audio_bitrate="192k", + maxrate="10M", + bufsize="20M", + max_duration_seconds=600, + recommended_duration_seconds=(30, 180), + safe_zones={"top_px": 80, "bottom_px": 100, "left_px": 80, "right_px": 80}, + notes=("Landscape explainers and square clips both work; captions are strongly recommended.",), + ), +} + + +def get_platform_profile(key: str | None) -> PlatformProfile: + if not key: + return PLATFORM_PROFILES["tiktok"] + return PLATFORM_PROFILES.get(key, PLATFORM_PROFILES["tiktok"]) + + +def list_platform_profiles() -> list[str]: + return sorted(PLATFORM_PROFILES) + + +def platform_profile_metadata() -> dict[str, dict[str, Any]]: + return {key: profile.metadata() for key, profile in sorted(PLATFORM_PROFILES.items())} diff --git a/services/render_engine/renderer/templates/presets.py b/services/render_engine/renderer/templates/presets.py new file mode 100644 index 0000000000000000000000000000000000000000..9fdc067f404b9037f10a9ec5a3fb98d8b6e85456 --- /dev/null +++ b/services/render_engine/renderer/templates/presets.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +PRESETS: dict[str, dict[str, Any]] = { + "tiktok_9_16_fast": { + "creative_style": "viral_shorts", + "platform": "tiktok", + "template": "tiktok_classic", + "subtitle_format": "ass", + "auto_subtitles": True, + "preview": False, + "normalize": True, + }, + "youtube_shorts_hd": { + "creative_style": "viral_shorts", + "platform": "youtube_shorts", + "template": "youtube_shorts", + "subtitle_format": "ass", + "auto_subtitles": True, + "normalize": True, + }, + "podcast_square": { + "creative_style": "podcast_clip", + "platform": "instagram_feed_square", + "template": "podcast_style", + "subtitle_format": "ass", + "auto_subtitles": True, + "normalize": True, + "metadata": {"target_aspect": "1:1"}, + }, + "reels_with_subtitles": { + "creative_style": "story_vlog", + "platform": "instagram_reels", + "template": "modern_minimal", + "subtitle_format": "ass", + "auto_subtitles": True, + "normalize": True, + }, + "draft_preview": { + "creative_style": "product_demo", + "platform": "tiktok", + "template": "modern_minimal", + "subtitle_format": "ass", + "preview": True, + "normalize": True, + }, + "tiktok_music_ducked": { + "creative_style": "viral_shorts", + "platform": "tiktok", + "template": "tiktok_classic", + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.25, + "music_fade_in": 0.4, + "music_fade_out": 1.0, + "music_loop": True, + "music_ducking": True, + "normalize": True, + }, + "instagram_reels_music": { + "creative_style": "story_vlog", + "platform": "instagram_reels", + "template": "modern_minimal", + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.28, + "music_fade_in": 0.3, + "music_fade_out": 0.8, + "music_loop": True, + "music_ducking": True, + "normalize": True, + }, + "youtube_landscape_1080p": { + "creative_style": "news_explainer", + "platform": "youtube_1080p", + "template": "news_style", + "subtitle_format": "ass", + "auto_subtitles": False, + "audio_normalize": True, + "normalize": True, + }, + "capcut_viral_auto": { + "creative_style": "viral_shorts", + "platform": "tiktok", + "template": "neon_pop", + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.24, + "music_fade_in": 0.2, + "music_fade_out": 0.8, + "music_loop": True, + "music_ducking": True, + "normalize": True, + }, + "capcut_product_launch": { + "creative_style": "product_demo", + "platform": "instagram_reels", + "template": "product_demo", + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.18, + "music_fade_in": 0.3, + "music_fade_out": 0.7, + "music_loop": True, + "music_ducking": True, + "normalize": True, + }, + "capcut_cinematic_story": { + "creative_style": "cinematic_story", + "platform": "youtube_shorts", + "template": "cinematic_gold", + "subtitle_format": "ass", + "auto_subtitles": True, + "audio_normalize": True, + "music_volume": 0.26, + "music_fade_in": 0.8, + "music_fade_out": 1.4, + "music_loop": True, + "music_ducking": True, + "normalize": True, + }, +} + + +def list_presets() -> list[str]: + return sorted(PRESETS) + + +def apply_preset(payload: dict[str, Any]) -> dict[str, Any]: + preset_name = payload.get("preset") + if not preset_name: + return payload + preset = deepcopy(PRESETS.get(preset_name, {})) + preset.update(payload) + if "metadata" in PRESETS.get(preset_name, {}) or "metadata" in payload: + metadata = deepcopy(PRESETS.get(preset_name, {}).get("metadata", {})) + metadata.update(payload.get("metadata", {})) + preset["metadata"] = metadata + return preset diff --git a/services/render_engine/renderer/transcription/__init__.py b/services/render_engine/renderer/transcription/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0a2279eb2b61e21d3bb959cff730686d7bd69a4d --- /dev/null +++ b/services/render_engine/renderer/transcription/__init__.py @@ -0,0 +1,3 @@ +from renderer.transcription.whisper import TranscriptionResult, WhisperTranscriber + +__all__ = ["TranscriptionResult", "WhisperTranscriber"] diff --git a/services/render_engine/renderer/transcription/whisper.py b/services/render_engine/renderer/transcription/whisper.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0076563f1f429224c6511b5d5e319df26ad031 --- /dev/null +++ b/services/render_engine/renderer/transcription/whisper.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from pathlib import Path +from threading import Lock + +from renderer.core.config import Settings +from renderer.subtitles import SubtitleEvent + + +@dataclass +class TranscriptionWord: + start: float + end: float + word: str + probability: float | None = None + + +@dataclass +class TranscriptionSegment: + id: int + start: float + end: float + text: str + words: list[TranscriptionWord] = field(default_factory=list) + + +@dataclass +class TranscriptionResult: + text: str + language: str | None + language_probability: float | None + duration: float | None + segments: list[TranscriptionSegment] + + def as_dict(self) -> dict: + return asdict(self) + + def subtitle_events(self, prefer_words: bool = False) -> list[SubtitleEvent]: + if prefer_words: + words = [ + SubtitleEvent(word.start, word.end, word.word.strip()) + for segment in self.segments + for word in segment.words + if word.word.strip() + ] + if words: + return words + return [SubtitleEvent(segment.start, segment.end, segment.text.strip()) for segment in self.segments if segment.text.strip()] + + +class WhisperTranscriber: + """Lazy CPU-first faster-whisper wrapper.""" + + _models: dict[tuple[str, str, str, str], object] = {} + _lock = Lock() + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def transcribe( + self, + audio_path: str | Path, + *, + model_size: str | None = None, + language: str | None = None, + task: str = "transcribe", + beam_size: int = 5, + vad_filter: bool = True, + word_timestamps: bool = True, + ) -> TranscriptionResult: + model = self._model(model_size or self.settings.whisper_model_size) + segments_iter, info = model.transcribe( + str(audio_path), + language=language, + task=task, + beam_size=beam_size, + vad_filter=vad_filter, + word_timestamps=word_timestamps, + ) + segments: list[TranscriptionSegment] = [] + for segment in segments_iter: + words = [ + TranscriptionWord( + start=float(word.start), + end=float(word.end), + word=word.word, + probability=getattr(word, "probability", None), + ) + for word in (segment.words or []) + ] + segments.append( + TranscriptionSegment( + id=int(segment.id), + start=float(segment.start), + end=float(segment.end), + text=segment.text.strip(), + words=words, + ) + ) + return TranscriptionResult( + text=" ".join(segment.text for segment in segments).strip(), + language=getattr(info, "language", None), + language_probability=getattr(info, "language_probability", None), + duration=getattr(info, "duration", None), + segments=segments, + ) + + def _model(self, model_size: str): + key = ( + model_size, + self.settings.whisper_device, + self.settings.whisper_compute_type, + str(self.settings.whisper_model_dir), + ) + with self._lock: + if key not in self._models: + try: + from faster_whisper import WhisperModel + except ImportError as exc: + raise RuntimeError("faster-whisper is not installed. Install requirements.txt to enable transcription.") from exc + self._models[key] = WhisperModel( + model_size, + device=self.settings.whisper_device, + compute_type=self.settings.whisper_compute_type, + download_root=str(self.settings.whisper_model_dir), + ) + return self._models[key] diff --git a/services/render_engine/renderer/transitions/__init__.py b/services/render_engine/renderer/transitions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..da35a5e81dfef185eac44d9eb28fa3a7bcf916af --- /dev/null +++ b/services/render_engine/renderer/transitions/__init__.py @@ -0,0 +1,3 @@ +from renderer.transitions.builder import TransitionBuilder + +__all__ = ["TransitionBuilder"] diff --git a/services/render_engine/renderer/transitions/builder.py b/services/render_engine/renderer/transitions/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..17d16ae11204005d900be4c55c7163aad62a0a7c --- /dev/null +++ b/services/render_engine/renderer/transitions/builder.py @@ -0,0 +1,69 @@ +from __future__ import annotations + + +class TransitionBuilder: + """Build FFmpeg xfade filters for scene joins.""" + + TRANSITIONS = { + "basic": "fade", + "fade": "fade", + "zoom": "zoomin", + "slide": "slideleft", + "slide_left": "slideleft", + "slide_right": "slideright", + "slide_up": "slideup", + "slide_down": "slidedown", + "push": "slideup", + "blur": "fade", + "whip": "smoothleft", + "dissolve": "dissolve", + "flash": "fadewhite", + "glitch": "hlslice", + "light_leak": "fadewhite", + "film_burn": "fadegrays", + "camera_shake": "smoothleft", + "spin": "circleopen", + "3d_flip": "vertopen", + "flip": "vertopen", + "cube": "rectcrop", + "ripple": "radial", + "ink": "distance", + "morph": "dissolve", + "stretch": "squeezeh", + "liquid": "pixelize", + "elastic": "smoothup", + "motion_blur": "smoothleft", + "wipe": "wipeleft", + "wipe_left": "wipeleft", + "wipe_right": "wiperight", + "wipe_up": "wipeup", + "wipe_down": "wipedown", + "smooth_left": "smoothleft", + "smooth_right": "smoothright", + "fadeblack": "fadeblack", + "fade_white": "fadewhite", + "pixel": "pixelize", + } + + def map_transition(self, name: str) -> str: + return self.TRANSITIONS.get(name, "fade") + + def list_transitions(self) -> list[str]: + return list(self.TRANSITIONS.keys()) + + def xfade_chain(self, stream_count: int, durations: list[float], transitions: list[str], transition_duration: float = 0.45) -> tuple[str, str]: + if stream_count <= 1: + return "", "[0:v]" + filters: list[str] = [] + current = "[0:v]" + offset = max(0.1, durations[0] - transition_duration) + for idx in range(1, stream_count): + out = f"[vx{idx}]" + transition = self.map_transition(transitions[idx - 1] if idx - 1 < len(transitions) else "fade") + filters.append( + f"{current}[{idx}:v]xfade=transition={transition}:duration={transition_duration}:offset={offset:.3f}{out}" + ) + current = out + if idx < len(durations): + offset += max(0.1, durations[idx] - transition_duration) + return ";".join(filters), current diff --git a/services/render_engine/requirements.txt b/services/render_engine/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1ee7776b5169daf6b49c0fe0b5957511a9da5e3d --- /dev/null +++ b/services/render_engine/requirements.txt @@ -0,0 +1,11 @@ +fastapi>=0.115.0 +uvicorn[standard]>=0.30.0 +gradio>=5.0.0 +huggingface_hub<1.0 +pydantic>=2.7.0 +python-multipart>=0.0.9 +psutil>=5.9.8 +pytest>=8.2.0 +pytest-cov>=5.0.0 +httpx>=0.27.0 +faster-whisper>=1.0.0 diff --git a/services/whisper/.dockerignore b/services/whisper/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..3d038972fd0b7d8b9c10cc6ef8461e6c4b4fde35 --- /dev/null +++ b/services/whisper/.dockerignore @@ -0,0 +1,9 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +*.db +.env +.git +.gitignore +.cache \ No newline at end of file diff --git a/services/whisper/.gitattributes b/services/whisper/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..0d5792e1859e20ece1544bbe28bb4284ba221742 --- /dev/null +++ b/services/whisper/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +fonts/TikTok-Bold.ttf filter=lfs diff=lfs merge=lfs -text diff --git a/services/whisper/Dockerfile b/services/whisper/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5a59393213b3cd8d649a9eec511135eae8ee3cc8 --- /dev/null +++ b/services/whisper/Dockerfile @@ -0,0 +1,53 @@ +FROM python:3.10-slim + +WORKDIR /app + +# ========================= +# SYSTEM DEPENDENCIES (CRITICAL) +# ========================= +RUN apt-get update && apt-get install -y \ + build-essential \ + gcc \ + g++ \ + ffmpeg \ + imagemagick \ + fonts-dejavu-core \ + libpq-dev \ + libsm6 \ + libxext6 \ + libglib2.0-0 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# ========================= +# PYTHON BASE TOOLS +# ========================= +RUN pip install --upgrade pip setuptools wheel + +# ========================= +# DEPENDENCIES (SINGLE LAYER) +# ========================= +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +# ========================= +# APPLICATION CODE +# ========================= +COPY . . + +# ========================= +# PERMISSIONS +# ========================= +RUN chmod +x start.sh + +# ========================= +# ENV SAFETY (OPTIONAL BUT RECOMMENDED) +# ========================= +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +# ========================= +# START +# ========================= +CMD ["bash", "start.sh"] \ No newline at end of file diff --git a/services/whisper/README.md b/services/whisper/README.md new file mode 100644 index 0000000000000000000000000000000000000000..87cbea73f6483b2bb2b06afc1554803c8f9b64a0 --- /dev/null +++ b/services/whisper/README.md @@ -0,0 +1,11 @@ +--- +title: Whisper +emoji: 🏆 +colorFrom: blue +colorTo: indigo +sdk: docker +pinned: false +license: mit +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/services/whisper/auth/database.py b/services/whisper/auth/database.py new file mode 100644 index 0000000000000000000000000000000000000000..af5f6cfffae64f17eeba670d3a812d76cb064887 --- /dev/null +++ b/services/whisper/auth/database.py @@ -0,0 +1,88 @@ +import os +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy.exc import OperationalError + +# ========================================================== +# DATABASE URL +# ========================================================== +DATABASE_URL = os.getenv("DATABASE_URL") + +if not DATABASE_URL: + raise RuntimeError( + "DATABASE_URL environment variable is not set" + ) + +# ========================================================== +# SUPABASE ENTERPRISE ENGINE CONFIG +# ========================================================== +# Optimized for: +# - Supabase Pooler +# - FastAPI async workload +# - Background workers +# - Long-running autonomous services + +engine = create_engine( + DATABASE_URL, + + # --- Pool Stability --- + pool_pre_ping=True, # validates dead connections + pool_recycle=300, # refresh connections + pool_size=5, # safe baseline + max_overflow=10, # burst capacity + + # --- Reliability --- + echo=False, + future=True, + + # --- Supabase Requirement --- + connect_args={ + "sslmode": "require", + "connect_timeout": 10, + }, +) + +# ========================================================== +# SESSION FACTORY +# ========================================================== +SessionLocal = sessionmaker( + autocommit=False, + autoflush=False, + bind=engine, +) + +# ========================================================== +# BASE MODEL +# ========================================================== +Base = declarative_base() + +# ========================================================== +# DEPENDENCY (FASTAPI) +# ========================================================== +def get_db(): + """ + FastAPI dependency injection session. + Ensures connection cleanup even on crash. + """ + db = SessionLocal() + try: + yield db + finally: + db.close() + + +# ========================================================== +# CONNECTION TEST (STARTUP SAFE) +# ========================================================== +def verify_database_connection(): + """ + Validates database connectivity during startup. + Prevents silent runtime failures. + """ + try: + with engine.connect() as conn: + conn.execute("SELECT 1") + except OperationalError as e: + raise RuntimeError( + f"Database connection failed: {str(e)}" + ) \ No newline at end of file diff --git a/services/whisper/auth/models.py b/services/whisper/auth/models.py new file mode 100644 index 0000000000000000000000000000000000000000..b9eb23d5d0dce83b21d7f8983cd329ba371fea68 --- /dev/null +++ b/services/whisper/auth/models.py @@ -0,0 +1,96 @@ +import uuid +from sqlalchemy import Column, String, Boolean, DateTime, func, Index +from sqlalchemy.dialects.postgresql import UUID + +from auth.database import Base + + +# ========================================================= +# USERS TABLE (CORE AUTH ENTITY) +# ========================================================= +class User(Base): + __tablename__ = "users" + + # ----------------------------- + # Primary Key (UUID for Supabase compatibility) + # ----------------------------- + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + nullable=False, + ) + + # ----------------------------- + # Identity Fields + # ----------------------------- + email = Column(String(255), unique=True, nullable=False, index=True) + username = Column(String(100), unique=True, nullable=True, index=True) + + # ----------------------------- + # Security Fields + # NOTE: stores hashed password only (never plaintext) + # ----------------------------- + hashed_password = Column(String(255), nullable=False) + + # ----------------------------- + # Account State + # ----------------------------- + is_active = Column(Boolean, default=True, nullable=False) + is_verified = Column(Boolean, default=False, nullable=False) + + # ----------------------------- + # Audit Fields + # ----------------------------- + created_at = Column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + + updated_at = Column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + +# ========================================================= +# OPTIONAL: API KEY TABLE (FOR AUTOMATION / N8N / WORKFLOWS) +# ========================================================= +class ApiKey(Base): + __tablename__ = "api_keys" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + nullable=False, + ) + + user_id = Column( + UUID(as_uuid=True), + nullable=False, + index=True, + ) + + key_hash = Column(String(255), nullable=False, unique=True) + + name = Column(String(120), nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + + created_at = Column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + + +# ========================================================= +# INDEXES (PERFORMANCE OPTIMIZATION) +# ========================================================= +Index("idx_users_email", User.email) +Index("idx_users_username", User.username) +Index("idx_api_keys_user_id", ApiKey.user_id) \ No newline at end of file diff --git a/services/whisper/auth/routes.py b/services/whisper/auth/routes.py new file mode 100644 index 0000000000000000000000000000000000000000..d2fe1c1448f7da86a66adb17d13ee76ffa523b4c --- /dev/null +++ b/services/whisper/auth/routes.py @@ -0,0 +1,340 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError +from datetime import datetime + +from auth.database import get_db +from auth.models import User +from auth.schemas import ( + SignupSchema, + LoginSchema, + TokenSchema, +) +from auth.security import ( + hash_password, + verify_password, + create_access_token, + decode_token, +) + +router = APIRouter( + prefix="/api/auth", + tags=["Authentication"], +) + + +# ========================================================= +# SIGNUP +# ========================================================= + +@router.post("/signup", status_code=201) +def signup( + data: SignupSchema, + db: Session = Depends(get_db), +): + + try: + + # ================================================= + # NORMALIZATION + # ================================================= + + email = data.email.strip().lower() + username = data.username.strip() + + # ================================================= + # EXISTING EMAIL CHECK + # ================================================= + + existing_email = ( + db.query(User) + .filter(User.email == email) + .first() + ) + + if existing_email: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Email already registered", + ) + + # ================================================= + # EXISTING USERNAME CHECK + # ================================================= + + existing_username = ( + db.query(User) + .filter(User.username == username) + .first() + ) + + if existing_username: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already taken", + ) + + # ================================================= + # HASH PASSWORD + # ================================================= + + password_hash = hash_password(data.password) + + # ================================================= + # CREATE USER + # ================================================= + + user = User( + email=email, + username=username, + password_hash=password_hash, + created_at=datetime.utcnow(), + ) + + db.add(user) + db.commit() + db.refresh(user) + + # ================================================= + # CREATE JWT + # ================================================= + + token = create_access_token( + { + "sub": str(user.id), + "email": user.email, + } + ) + + return { + "status": "success", + "message": "Account created successfully", + "token": token, + "user": { + "id": str(user.id), + "email": user.email, + "username": user.username, + "created_at": ( + user.created_at.isoformat() + if user.created_at + else None + ), + }, + } + + except HTTPException: + raise + + except IntegrityError as e: + + db.rollback() + + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="User already exists", + ) + + except ValueError as e: + + db.rollback() + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + + except Exception as e: + + db.rollback() + + print("SIGNUP ERROR:", str(e)) + + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Signup failed: {str(e)}", + ) + + +# ========================================================= +# LOGIN +# ========================================================= + +@router.post("/login") +def login( + data: LoginSchema, + db: Session = Depends(get_db), +): + + try: + + email = data.email.strip().lower() + + user = ( + db.query(User) + .filter(User.email == email) + .first() + ) + + if not user: + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid email or password", + ) + + if not verify_password( + data.password, + user.password_hash, + ): + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid email or password", + ) + + token = create_access_token( + { + "sub": str(user.id), + "email": user.email, + } + ) + + return { + "status": "success", + "message": "Login successful", + "token": token, + "user": { + "id": str(user.id), + "email": user.email, + "username": user.username, + "created_at": ( + user.created_at.isoformat() + if user.created_at + else None + ), + }, + } + + except HTTPException: + raise + + except Exception as e: + + print("LOGIN ERROR:", str(e)) + + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Login failed: {str(e)}", + ) + + +# ========================================================= +# VERIFY TOKEN +# ========================================================= + +@router.post("/verify") +def verify_token( + data: TokenSchema, +): + + try: + + payload = decode_token(data.token) + + if not payload: + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + ) + + return { + "valid": True, + "payload": payload, + } + + except HTTPException: + raise + + except Exception as e: + + print("VERIFY ERROR:", str(e)) + + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Token verification failed: {str(e)}", + ) + + +# ========================================================= +# REFRESH TOKEN +# ========================================================= + +@router.post("/refresh") +def refresh_token( + data: TokenSchema, + db: Session = Depends(get_db), +): + + try: + + payload = decode_token(data.token) + + user_id = payload.get("sub") + + if not user_id: + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token payload", + ) + + user = ( + db.query(User) + .filter(User.id == user_id) + .first() + ) + + if not user: + + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User not found", + ) + + new_token = create_access_token( + { + "sub": str(user.id), + "email": user.email, + } + ) + + return { + "status": "success", + "token": new_token, + } + + except HTTPException: + raise + + except Exception as e: + + print("REFRESH ERROR:", str(e)) + + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Refresh failed: {str(e)}", + ) + + +# ========================================================= +# LOGOUT +# ========================================================= + +@router.post("/logout") +def logout(): + + return { + "status": "success", + "message": "Logout successful", + } \ No newline at end of file diff --git a/services/whisper/auth/schemas.py b/services/whisper/auth/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..01472622737511de3ef76a09b389f50ac39971cc --- /dev/null +++ b/services/whisper/auth/schemas.py @@ -0,0 +1,59 @@ +from pydantic import BaseModel, EmailStr, Field, ConfigDict + + +# ========================================================= +# BASE CONFIG (STRICT MODE → PREVENTS SILENT DATA COERCION) +# ========================================================= +class StrictSchema(BaseModel): + model_config = ConfigDict( + strict=True, + extra="forbid", + validate_assignment=True, + ) + + +# ========================================================= +# SIGNUP SCHEMA +# ========================================================= +class SignupSchema(StrictSchema): + email: EmailStr + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-zA-Z0-9_]+$" + ) + password: str = Field( + min_length=8, + max_length=72, + ) + + +# ========================================================= +# LOGIN SCHEMA +# ========================================================= +class LoginSchema(StrictSchema): + email: EmailStr + password: str = Field( + min_length=1, + max_length=72, + ) + + +# ========================================================= +# TOKEN REQUEST SCHEMA +# ========================================================= +class TokenSchema(StrictSchema): + token: str = Field( + min_length=10, + max_length=2048 + ) + + +# ========================================================= +# USER RESPONSE SCHEMA (SAFE OUTPUT MODEL) +# ========================================================= +class UserOutSchema(StrictSchema): + id: int + email: EmailStr + username: str + created_at: str \ No newline at end of file diff --git a/services/whisper/auth/security.py b/services/whisper/auth/security.py new file mode 100644 index 0000000000000000000000000000000000000000..248b4b97de7017df7f15c14588ad43444965901b --- /dev/null +++ b/services/whisper/auth/security.py @@ -0,0 +1,137 @@ +import os +from datetime import datetime, timedelta, timezone +from typing import Optional, Dict, Any + +from jose import jwt, JWTError +from passlib.context import CryptContext +from fastapi import HTTPException, status, Depends +from fastapi.security import OAuth2PasswordBearer + + +# ========================================================= +# ENV CONFIG +# ========================================================= + +SECRET_KEY = os.getenv("SECRET_KEY") + +if not SECRET_KEY: + raise RuntimeError("SECRET_KEY environment variable missing") + +ALGORITHM = os.getenv("ALGORITHM", "HS256") + +ACCESS_TOKEN_EXPIRE_MINUTES = int( + os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "1440") +) + + +# ========================================================= +# PASSWORD HASHING +# ========================================================= +# IMPORTANT: +# pbkdf2_sha256 avoids: +# - bcrypt crashes +# - bcrypt native dependency issues +# - 72-byte limits +# - passlib backend bugs +# ========================================================= + +pwd_context = CryptContext( + schemes=["pbkdf2_sha256"], + deprecated="auto", +) + + +def hash_password(password: str) -> str: + + if not password: + raise ValueError("Password required") + + if len(password) < 8: + raise ValueError("Password too short") + + return pwd_context.hash(password) + + +def verify_password( + plain_password: str, + hashed_password: str, +) -> bool: + + try: + return pwd_context.verify( + plain_password, + hashed_password, + ) + except Exception: + return False + + +# ========================================================= +# JWT +# ========================================================= + +def create_access_token( + data: Dict[str, Any], + expires_delta: Optional[timedelta] = None, +) -> str: + + to_encode = data.copy() + + expire = datetime.now(timezone.utc) + ( + expires_delta + if expires_delta + else timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) + ) + + to_encode.update({"exp": expire}) + + return jwt.encode( + to_encode, + SECRET_KEY, + algorithm=ALGORITHM, + ) + + +def decode_token(token: str): + + try: + return jwt.decode( + token, + SECRET_KEY, + algorithms=[ALGORITHM], + ) + + except JWTError: + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +# ========================================================= +# AUTH DEPENDENCY +# ========================================================= + +oauth2_scheme = OAuth2PasswordBearer( + tokenUrl="/api/auth/login" +) + + +def get_current_user( + token: str = Depends(oauth2_scheme) +): + + payload = decode_token(token) + + user_id = payload.get("sub") + + if not user_id: + + raise HTTPException( + status_code=401, + detail="Invalid authentication", + ) + + return payload \ No newline at end of file diff --git a/services/whisper/core/builders/api_builder.py b/services/whisper/core/builders/api_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..d2b911003465c5c10d4d2151d24c7194e51b8bbf --- /dev/null +++ b/services/whisper/core/builders/api_builder.py @@ -0,0 +1,142 @@ +from fastapi import APIRouter, UploadFile, File, Form, HTTPException +from fastapi.responses import JSONResponse, FileResponse +from typing import Optional +import tempfile +import shutil +import os + +from core.registry.loader import get_tasks +from core.execution.executor import execute_task + + +# ===================================================== +# ROUTER BUILDER +# ===================================================== + +def build_api_router() -> APIRouter: + """ + Dynamically builds API routes from TASK REGISTRY. + + Generated endpoints: + + POST /execute/{task} + GET /tasks + GET /health + """ + + router = APIRouter(tags=["API"]) + + # ================================================= + # EXECUTE TASK + # ================================================= + @router.post("/execute/{task_name}") + async def execute( + task_name: str, + file: Optional[UploadFile] = File(None), + url_input: Optional[str] = Form(None), + ): + """ + Universal execution endpoint. + Accepts: + - file upload + - url_input + """ + + tasks = get_tasks() + + if task_name not in tasks: + raise HTTPException( + status_code=404, + detail=f"Task '{task_name}' not found", + ) + + temp_path = None + + try: + # ----------------------------------------- + # SAVE UPLOADED FILE + # ----------------------------------------- + if file: + suffix = os.path.splitext(file.filename)[1] + + with tempfile.NamedTemporaryFile( + delete=False, + suffix=suffix, + ) as tmp: + shutil.copyfileobj(file.file, tmp) + temp_path = tmp.name + + # ----------------------------------------- + # BUILD INPUT PAYLOAD + # ----------------------------------------- + payload = { + "file_path": temp_path, + "url_input": url_input, + } + + # ----------------------------------------- + # EXECUTE TASK + # ----------------------------------------- + result = await execute_task(task_name, payload) + + # ----------------------------------------- + # FILE RESPONSE + # ----------------------------------------- + if isinstance(result, dict) and result.get("file"): + output_file = result["file"] + + if os.path.exists(output_file): + return FileResponse( + output_file, + filename=os.path.basename(output_file), + ) + + # ----------------------------------------- + # JSON RESPONSE + # ----------------------------------------- + return JSONResponse(result) + + except Exception as e: + raise HTTPException( + status_code=500, + detail=str(e), + ) + + finally: + # ----------------------------------------- + # CLEANUP TEMP FILE + # ----------------------------------------- + if temp_path and os.path.exists(temp_path): + os.unlink(temp_path) + + # ================================================= + # TASK LIST + # ================================================= + @router.get("/tasks") + async def list_tasks(): + """ + Returns registry tasks. + Used by UI and Docs builder. + """ + + tasks = get_tasks() + + return { + name: { + "category": getattr(t, "category", "general"), + "description": getattr(t, "description", ""), + } + for name, t in tasks.items() + } + + # ================================================= + # HEALTH CHECK + # ================================================= + @router.get("/health") + async def health(): + return { + "status": "ok", + "tasks_loaded": len(get_tasks()), + } + + return router \ No newline at end of file diff --git a/services/whisper/core/builders/docs_builder.py b/services/whisper/core/builders/docs_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..86cda92f82a022e7f7c7318fbeca2e9c0624bbb1 --- /dev/null +++ b/services/whisper/core/builders/docs_builder.py @@ -0,0 +1,89 @@ +from fastapi import APIRouter +from fastapi.responses import JSONResponse +from typing import Dict, Any, List + +from core.registry.tasks_registry import TASKS + + +# ===================================================== +# INTERNAL HELPERS +# ===================================================== + +def _serialize_task(task) -> Dict[str, Any]: + """ + Convert TaskDefinition → API-safe metadata. + Must NOT import task modules. + """ + + return { + "name": getattr(task, "name", None), + "description": getattr(task, "description", ""), + "category": getattr(task, "category", "general"), + "module": getattr(task, "module", None), + "callable": getattr(task, "callable_name", "run"), + "async": getattr(task, "async_task", False), + "enabled": getattr(task, "enabled", True), + } + + +def _group_tasks(tasks: List[Any]) -> Dict[str, List[Dict[str, Any]]]: + """ + Groups tasks by category for UI rendering. + """ + + grouped: Dict[str, List[Dict[str, Any]]] = {} + + for task in tasks: + category = getattr(task, "category", "general") + grouped.setdefault(category, []).append(_serialize_task(task)) + + return grouped + + +# ===================================================== +# DOCS ROUTER BUILDER (V11) +# ===================================================== + +def build_docs_router() -> APIRouter: + """ + Builds dynamic documentation endpoints. + + Provides: + /docs/tasks → flat task list + /docs/catalog → grouped tasks + /docs/health → docs status + """ + + router = APIRouter( + prefix="/docs", + tags=["Documentation"], + ) + + # ------------------------------------------------- + # List all tasks + # ------------------------------------------------- + @router.get("/tasks") + async def list_tasks(): + return JSONResponse( + [_serialize_task(task) for task in TASKS] + ) + + # ------------------------------------------------- + # Categorized task catalog + # ------------------------------------------------- + @router.get("/catalog") + async def task_catalog(): + return JSONResponse(_group_tasks(TASKS)) + + # ------------------------------------------------- + # Docs health endpoint + # ------------------------------------------------- + @router.get("/health") + async def docs_health(): + return { + "status": "ok", + "service": "docs", + "tasks_registered": len(TASKS), + } + + return router \ No newline at end of file diff --git a/services/whisper/core/builders/ui_builder.py b/services/whisper/core/builders/ui_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..c9d3f29e53d0e3e198f4622e150aa865427aa98b --- /dev/null +++ b/services/whisper/core/builders/ui_builder.py @@ -0,0 +1,107 @@ +from fastapi import APIRouter +from fastapi.responses import HTMLResponse, FileResponse +from pathlib import Path +from typing import Dict, List + +from core.registry.tasks_registry import TASKS + + +# ===================================================== +# CONFIG +# ===================================================== + +BASE_DIR = Path(__file__).resolve().parents[2] +UI_DIR = BASE_DIR / "ui" +INDEX_FILE = UI_DIR / "index.html" + + +# ===================================================== +# TASK GROUPING +# ===================================================== + +def group_tasks() -> Dict[str, List[dict]]: + """ + Converts registry → categorized UI structure. + """ + + categories: Dict[str, List[dict]] = {} + + for task in TASKS: + if not getattr(task, "enabled", True): + continue + + category = getattr(task, "category", "general") + + categories.setdefault(category, []).append( + { + "name": task.name, + "description": getattr(task, "description", ""), + } + ) + + return categories + + +# ===================================================== +# ROUTER BUILDER +# ===================================================== + +def build_ui_router() -> APIRouter: + """ + Serves UI + dynamic UI metadata. + + Endpoints: + / + /ui + /ui/tasks + /ui/health + """ + + router = APIRouter(tags=["UI"]) + + # ------------------------------------------------- + # MAIN UI + # ------------------------------------------------- + @router.get("/", response_class=HTMLResponse) + async def serve_root(): + """ + Serves index.html + """ + + if not INDEX_FILE.exists(): + return HTMLResponse( + "

Basyx UI Missing

", + status_code=500, + ) + + return FileResponse(INDEX_FILE) + + # ------------------------------------------------- + # Explicit UI route + # ------------------------------------------------- + @router.get("/ui", response_class=HTMLResponse) + async def serve_ui(): + return await serve_root() + + # ------------------------------------------------- + # UI TASK CATALOG + # ------------------------------------------------- + @router.get("/ui/tasks") + async def ui_tasks(): + """ + UI fetches this to build sidebar dynamically. + """ + return group_tasks() + + # ------------------------------------------------- + # HEALTH + # ------------------------------------------------- + @router.get("/ui/health") + async def ui_health(): + return { + "status": "ok", + "ui": "active", + "tasks": len(TASKS), + } + + return router \ No newline at end of file diff --git a/services/whisper/core/execution/context.py b/services/whisper/core/execution/context.py new file mode 100644 index 0000000000000000000000000000000000000000..807d40a92e490eceee27e4878fa1d5e5fe78d40e --- /dev/null +++ b/services/whisper/core/execution/context.py @@ -0,0 +1,155 @@ +""" +V11 Execution Context +--------------------- + +Central runtime object passed into every task. + +Responsibilities: +- Hold inputs +- Share memory between tasks +- Store outputs +- Track execution metadata +- Provide filesystem helpers +- Provide logging helpers +""" + +from __future__ import annotations + +import os +import uuid +import tempfile +from typing import Any, Dict, Optional + + +# ========================================================= +# Context Object +# ========================================================= + +class ExecutionContext: + """ + Standard runtime context used by ALL tasks. + + Every task receives: + async def run(ctx: ExecutionContext) + """ + + # ----------------------------------------------------- + # INIT + # ----------------------------------------------------- + def __init__( + self, + task_name: str, + inputs: Optional[Dict[str, Any]] = None, + workspace: Optional[str] = None, + ): + + self.task_name = task_name + self.job_id = str(uuid.uuid4()) + + self.inputs: Dict[str, Any] = inputs or {} + self.outputs: Dict[str, Any] = {} + self.memory: Dict[str, Any] = {} + + self.status: str = "created" + self.error: Optional[str] = None + + self.workspace = workspace or self._create_workspace() + + # ----------------------------------------------------- + # WORKSPACE + # ----------------------------------------------------- + def _create_workspace(self) -> str: + path = tempfile.mkdtemp(prefix="basyx_job_") + return path + + def path(self, filename: str) -> str: + """ + Safe workspace path helper + """ + return os.path.join(self.workspace, filename) + + # ----------------------------------------------------- + # INPUT HELPERS + # ----------------------------------------------------- + def get(self, key: str, default=None): + return self.inputs.get(key, default) + + def require(self, key: str): + if key not in self.inputs: + raise ValueError(f"Missing required input: {key}") + return self.inputs[key] + + # ----------------------------------------------------- + # OUTPUT HELPERS + # ----------------------------------------------------- + def set_output(self, key: str, value: Any): + self.outputs[key] = value + + def result(self) -> Dict[str, Any]: + return { + "job_id": self.job_id, + "task": self.task_name, + "status": self.status, + "outputs": self.outputs, + "error": self.error, + } + + # ----------------------------------------------------- + # MEMORY (cross-task sharing) + # ----------------------------------------------------- + def remember(self, key: str, value: Any): + """ + Save value for downstream tasks. + """ + self.memory[key] = value + + def recall(self, key: str, default=None): + return self.memory.get(key, default) + + # ----------------------------------------------------- + # STATUS MANAGEMENT + # ----------------------------------------------------- + def mark_running(self): + self.status = "running" + + def mark_complete(self): + self.status = "completed" + + def mark_failed(self, error: Exception | str): + self.status = "failed" + self.error = str(error) + + # ----------------------------------------------------- + # LOGGING + # ----------------------------------------------------- + def log(self, message: str): + print(f"[{self.task_name} | {self.job_id}] {message}") + + # ----------------------------------------------------- + # SERIALIZATION + # ----------------------------------------------------- + def to_dict(self): + return { + "job_id": self.job_id, + "task_name": self.task_name, + "inputs": self.inputs, + "outputs": self.outputs, + "memory": self.memory, + "status": self.status, + "error": self.error, + "workspace": self.workspace, + } + + +# ========================================================= +# Context Factory +# ========================================================= + +def create_context(task_name: str, inputs: Dict[str, Any]) -> ExecutionContext: + """ + Standardized factory used by executor. + """ + return ExecutionContext( + task_name=task_name, + inputs=inputs, + ) \ No newline at end of file diff --git a/services/whisper/core/execution/executor.py b/services/whisper/core/execution/executor.py new file mode 100644 index 0000000000000000000000000000000000000000..c93bd5041b62ad070adfc3fe628652c615daed7b --- /dev/null +++ b/services/whisper/core/execution/executor.py @@ -0,0 +1,166 @@ +""" +BASYX V11 EXECUTOR +------------------ + +Central task execution engine. + +Responsibilities: +- Load task from registry +- Create execution context +- Execute task safely +- Capture outputs +- Handle failures +- Support chaining +""" + +from __future__ import annotations + +import asyncio +import inspect +from typing import Dict, Any, List + +from core.execution.context import create_context, ExecutionContext +from core.registry.loader import get_task_map + + +# ========================================================= +# TASK CACHE +# ========================================================= + +TASK_MAP = get_task_map() + + +# ========================================================= +# INTERNAL EXECUTION +# ========================================================= + +async def _run_task( + task_name: str, + inputs: Dict[str, Any], +) -> Dict[str, Any]: + """ + Execute a single task safely. + """ + + if task_name not in TASK_MAP: + raise ValueError(f"Unknown task: {task_name}") + + task = TASK_MAP[task_name] + + ctx: ExecutionContext = create_context( + task_name=task_name, + inputs=inputs, + ) + + ctx.mark_running() + ctx.log("Starting task") + + try: + + # --------------------------------------------- + # Execute task + # --------------------------------------------- + result = task.run + + if inspect.iscoroutinefunction(result): + await result(ctx) + else: + await asyncio.to_thread(result, ctx) + + ctx.mark_complete() + ctx.log("Task completed") + + except Exception as e: + ctx.mark_failed(e) + ctx.log(f"Task failed: {e}") + + return ctx.result() + + +# ========================================================= +# PUBLIC EXECUTOR +# ========================================================= + +async def execute_task( + task_name: str, + inputs: Dict[str, Any], +) -> Dict[str, Any]: + """ + Main entrypoint used by API + UI. + """ + + return await _run_task(task_name, inputs) + + +# ========================================================= +# PIPELINE EXECUTION (CHAINED TASKS) +# ========================================================= + +async def execute_pipeline( + tasks: List[Dict[str, Any]] +) -> List[Dict[str, Any]]: + """ + Execute tasks sequentially. + + Example: + [ + {"task": "transcribe", "inputs": {...}}, + {"task": "subtitles"}, + {"task": "render"} + ] + """ + + results = [] + shared_memory = {} + + for step in tasks: + + name = step["task"] + inputs = step.get("inputs", {}) + + # Inject memory from previous step + inputs["memory"] = shared_memory + + result = await _run_task(name, inputs) + + results.append(result) + + if result["status"] != "completed": + break + + # propagate outputs + shared_memory.update(result.get("outputs", {})) + + return results + + +# ========================================================= +# PARALLEL EXECUTION +# ========================================================= + +async def execute_parallel( + tasks: List[Dict[str, Any]] +) -> List[Dict[str, Any]]: + """ + Run multiple tasks concurrently. + """ + + coroutines = [ + _run_task(t["task"], t.get("inputs", {})) + for t in tasks + ] + + return await asyncio.gather(*coroutines) + + +# ========================================================= +# REGISTRY HOT RELOAD (DEV MODE) +# ========================================================= + +def reload_tasks(): + """ + Reload registry without restarting server. + Useful during development. + """ + global TASK_MAP + TASK_MAP = get_task_map() \ No newline at end of file diff --git a/services/whisper/core/registry/loader.py b/services/whisper/core/registry/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..4769dc19d0ebe685427696b0670c3b7254bec6dd --- /dev/null +++ b/services/whisper/core/registry/loader.py @@ -0,0 +1,45 @@ +from core.registry.tasks_registry import TASKS + + +# ------------------------------------------------ +# Return categorized tasks (UI Builder uses this) +# ------------------------------------------------ +def get_tasks(): + return TASKS + + +# ------------------------------------------------ +# Flatten tasks across categories +# ------------------------------------------------ +def get_all_tasks(): + + tasks = [] + + for category, items in TASKS.items(): + for task in items: + task_copy = dict(task) + task_copy["category"] = category + tasks.append(task_copy) + + return tasks + + +# ------------------------------------------------ +# Execution map +# id -> callable +# ------------------------------------------------ +def get_task_map(): + + task_map = {} + + for category, items in TASKS.items(): + for task in items: + + if "handler" not in task: + raise RuntimeError( + f"Task '{task['id']}' missing handler" + ) + + task_map[task["id"]] = task["handler"] + + return task_map \ No newline at end of file diff --git a/services/whisper/core/registry/task_model.py b/services/whisper/core/registry/task_model.py new file mode 100644 index 0000000000000000000000000000000000000000..22635b4ea372611dc9d00b829099095d05303ff0 --- /dev/null +++ b/services/whisper/core/registry/task_model.py @@ -0,0 +1,20 @@ +from pydantic import BaseModel, Field +from typing import Callable, Dict, Any + + +class TaskDefinition(BaseModel): + name: str + category: str + description: str + + handler: Callable + + inputs: Dict[str, str] = Field(default_factory=dict) + outputs: Dict[str, str] = Field(default_factory=dict) + + ui_schema: Dict[str, Any] = Field(default_factory=dict) + + autonomous: bool = True + + class Config: + arbitrary_types_allowed = True \ No newline at end of file diff --git a/services/whisper/core/registry/tasks_registry.py b/services/whisper/core/registry/tasks_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..c77ff9a133ece4328aa0105cc94aefae6cf67969 --- /dev/null +++ b/services/whisper/core/registry/tasks_registry.py @@ -0,0 +1,198 @@ +""" +BASYX V11 — Dynamic Task Registry +--------------------------------- + +Single Source Of Truth for: + +• UI generation +• API routes +• Executor routing +• Documentation builder +• Autonomous Brain planning +""" + +# ========================================================= +# ANALYSIS TASKS +# ========================================================= + +from publisher.tasks.transcribe import run as transcribe +from publisher.tasks.subtitles import run as subtitles +from publisher.tasks.highlights import run as highlights +from publisher.tasks.viral_score import run as viral_score +from publisher.tasks.strategy import run as strategy + + +# ========================================================= +# RENDER TASKS +# ========================================================= + +from publisher.tasks.render import run as render +from publisher.tasks.generate_thumbnail import run as generate_thumbnail +from publisher.tasks.generate_metadata import run as generate_metadata + + +# ========================================================= +# PUBLISH TASKS +# ========================================================= + +from publisher.tasks.publish_tiktok import run as publish_tiktok +from publisher.tasks.publish_youtube import run as publish_youtube +from publisher.tasks.publish_instagram import run as publish_instagram + + +# ========================================================= +# SYSTEM TASKS +# ========================================================= + +from publisher.tasks.batch_runner import run as batch_runner +from publisher.tasks.autonomous_mode import run as autonomous_mode + + +# ========================================================= +# TASK REGISTRY +# ========================================================= + +TASKS = { + + # ----------------------------------------------------- + # ANALYSIS + # ----------------------------------------------------- + "analysis": [ + + { + "id": "transcribe", + "name": "Transcribe", + "description": "Generate transcript from video/audio", + "inputs": ["file", "url_input"], + "output": "json", + "handler": transcribe, + }, + + { + "id": "subtitles", + "name": "Subtitles", + "description": "Generate SRT subtitles", + "inputs": ["file", "url_input"], + "output": "file", + "handler": subtitles, + }, + + { + "id": "highlights", + "name": "Highlights", + "description": "Detect viral highlight segments", + "inputs": ["file"], + "output": "json", + "handler": highlights, + }, + + { + "id": "viral-score", + "name": "Viral Score", + "description": "AI virality prediction", + "inputs": ["file"], + "output": "json", + "handler": viral_score, + }, + + { + "id": "strategy", + "name": "Strategy", + "description": "Content strategy generation", + "inputs": ["file"], + "output": "json", + "handler": strategy, + }, + ], + + # ----------------------------------------------------- + # RENDER + # ----------------------------------------------------- + "render": [ + + { + "id": "render", + "name": "Render Video", + "description": "Render final short-form video", + "inputs": ["file", "url_input"], + "output": "video", + "video_output": True, + "handler": render, + }, + + { + "id": "generate-thumbnail", + "name": "Thumbnail", + "description": "Generate AI thumbnail", + "inputs": ["file"], + "output": "image", + "handler": generate_thumbnail, + }, + + { + "id": "generate-metadata", + "name": "Metadata", + "description": "Generate captions, hashtags, titles", + "inputs": ["file"], + "output": "json", + "handler": generate_metadata, + }, + ], + + # ----------------------------------------------------- + # PUBLISH + # ----------------------------------------------------- + "publish": [ + + { + "id": "publish-tiktok", + "name": "Publish TikTok", + "description": "Upload video to TikTok", + "inputs": ["file"], + "output": "json", + "handler": publish_tiktok, + }, + + { + "id": "publish-youtube", + "name": "Publish YouTube", + "description": "Upload YouTube Short", + "inputs": ["file"], + "output": "json", + "handler": publish_youtube, + }, + + { + "id": "publish-instagram", + "name": "Publish Instagram", + "description": "Upload Instagram Reel", + "inputs": ["file"], + "output": "json", + "handler": publish_instagram, + }, + ], + + # ----------------------------------------------------- + # SYSTEM + # ----------------------------------------------------- + "system": [ + + { + "id": "batch", + "name": "Batch Processor", + "description": "Execute batch pipeline", + "inputs": [], + "output": "json", + "handler": batch_runner, + }, + + { + "id": "autonomous", + "name": "Autonomous Mode", + "description": "Start autonomous AI publisher", + "inputs": [], + "output": "json", + "handler": autonomous_mode, + }, + ], +} \ No newline at end of file diff --git a/services/whisper/ingestion/__init__.py b/services/whisper/ingestion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b477625143bbc699f5a65c22b8fda00c6666f92 --- /dev/null +++ b/services/whisper/ingestion/__init__.py @@ -0,0 +1 @@ +# Initialize package \ No newline at end of file diff --git a/services/whisper/ingestion/base64_loader.py b/services/whisper/ingestion/base64_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..cfdd9f1b490200de65adbc17378ebd443ca1ce0f --- /dev/null +++ b/services/whisper/ingestion/base64_loader.py @@ -0,0 +1,19 @@ +import base64 +import tempfile + + +def decode_base64(data): + + header, encoded = data.split(",", 1) + + binary = base64.b64decode(encoded) + + tmp = tempfile.NamedTemporaryFile( + delete=False, + suffix=".mp4" + ) + + tmp.write(binary) + tmp.close() + + return tmp.name \ No newline at end of file diff --git a/services/whisper/ingestion/classifiers.py b/services/whisper/ingestion/classifiers.py new file mode 100644 index 0000000000000000000000000000000000000000..980539adb98a3702e660790661c256db97acec95 --- /dev/null +++ b/services/whisper/ingestion/classifiers.py @@ -0,0 +1,29 @@ +import os +from urllib.parse import urlparse + +def classify_source(source: str): + + if not source: + raise Exception("Empty source") + + if os.path.exists(source): + return "local" + + if source.endswith(".mp4"): + return "direct" + + domain = urlparse(source).netloc.lower() + + if "youtube" in domain or "youtu.be" in domain: + return "youtube" + + if "tiktok" in domain: + return "tiktok" + + if "instagram" in domain: + return "instagram" + + if "facebook" in domain: + return "facebook" + + return "unknown" \ No newline at end of file diff --git a/services/whisper/ingestion/downloader.py b/services/whisper/ingestion/downloader.py new file mode 100644 index 0000000000000000000000000000000000000000..8b24300cb71d2ceee308e4c9b74992a594151105 --- /dev/null +++ b/services/whisper/ingestion/downloader.py @@ -0,0 +1,18 @@ +import requests +import tempfile + + +def download_file(url): + + r = requests.get(url, stream=True, timeout=120) + + r.raise_for_status() + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") + + for chunk in r.iter_content(1024 * 1024): + tmp.write(chunk) + + tmp.close() + + return tmp.name \ No newline at end of file diff --git a/services/whisper/ingestion/normalizer.py b/services/whisper/ingestion/normalizer.py new file mode 100644 index 0000000000000000000000000000000000000000..4c905497f2d1f8a08422005e433377528dd55124 --- /dev/null +++ b/services/whisper/ingestion/normalizer.py @@ -0,0 +1,24 @@ +import subprocess +import uuid +import os + +def normalize_video(path): + + output = f"jobs/norm_{uuid.uuid4()}.mp4" + + cmd = [ + "ffmpeg", + "-y", + "-i", path, + "-vf", "scale=1080:-2", + "-c:v", "libx264", + "-preset", "veryfast", + "-crf", "23", + "-c:a", "aac", + "-movflags", "+faststart", + output, + ] + + subprocess.run(cmd, check=True) + + return output \ No newline at end of file diff --git a/services/whisper/ingestion/resolver.py b/services/whisper/ingestion/resolver.py new file mode 100644 index 0000000000000000000000000000000000000000..e7a44d739d69802bae6804af22581069887fda7f --- /dev/null +++ b/services/whisper/ingestion/resolver.py @@ -0,0 +1,127 @@ +import os +import uuid +import requests +from pathlib import Path +from typing import Optional, Union +from fastapi import UploadFile + +# ============================== +# STORAGE CONFIG +# ============================== + +BASE_DIR = Path(__file__).resolve().parents[1] +UPLOAD_DIR = str(BASE_DIR / "jobs" / "uploads") +os.makedirs(UPLOAD_DIR, exist_ok=True) + +CHUNK_SIZE = 1024 * 1024 # 1MB streaming for large media + + +# ============================== +# CORE RESOLVER +# ============================== + +def resolve_input( + source: Optional[str] = None, + upload: Optional[UploadFile] = None, + raw_bytes: Optional[bytes] = None +) -> str: + """ + Universal ingestion layer for all pipeline systems. + + Supports: + - UploadFile (FastAPI / Gradio) + - URL download (http/https) + - Local filesystem path + - Raw bytes input (future automation nodes) + """ + + # ---------------------------------------- + # CASE 1: UploadFile (Gradio / FastAPI) + # ---------------------------------------- + if upload is not None: + filename = f"{uuid.uuid4()}_{upload.filename or 'upload.mp4'}" + path = os.path.join(UPLOAD_DIR, filename) + + with open(path, "wb") as f: + while True: + chunk = upload.file.read(CHUNK_SIZE) + if not chunk: + break + f.write(chunk) + + return path + + # ---------------------------------------- + # CASE 2: Raw bytes (automation / webhook) + # ---------------------------------------- + if raw_bytes is not None: + filename = f"{uuid.uuid4()}.mp4" + path = os.path.join(UPLOAD_DIR, filename) + + with open(path, "wb") as f: + f.write(raw_bytes) + + return path + + # ---------------------------------------- + # CASE 3: URL input (YouTube, TikTok, direct mp4) + # ---------------------------------------- + if source and source.startswith(("http://", "https://")): + + filename = f"{uuid.uuid4()}.mp4" + path = os.path.join(UPLOAD_DIR, filename) + + headers = { + "User-Agent": "Mozilla/5.0 (compatible; BasyxBot/1.0)" + } + + with requests.get(source, stream=True, headers=headers, timeout=60) as r: + r.raise_for_status() + + with open(path, "wb") as f: + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + f.write(chunk) + + return path + + # ---------------------------------------- + # CASE 4: Local file path + # ---------------------------------------- + if source and os.path.exists(source): + return source + + # ---------------------------------------- + # INVALID INPUT HANDLING + # ---------------------------------------- + raise ValueError( + "resolve_input failed: no valid source, upload, or raw_bytes provided" + ) + + +# ============================== +# OPTIONAL HELPERS (V11 READY) +# ============================== + +def detect_input_type(source: str) -> str: + """ + Lightweight classifier for routing decisions upstream. + """ + + if source.startswith(("http://", "https://")): + return "url" + + if os.path.exists(source): + return "file" + + return "unknown" + + +def normalize_source(source: str) -> str: + """ + Cleans input strings for downstream consistency. + """ + if not source: + return source + + return source.strip() diff --git a/services/whisper/ingestion/social.py b/services/whisper/ingestion/social.py new file mode 100644 index 0000000000000000000000000000000000000000..00af0f1188ccc305752d37488809991b336233ec --- /dev/null +++ b/services/whisper/ingestion/social.py @@ -0,0 +1,21 @@ +import tempfile +import yt_dlp + + +def download_social(url): + + output = tempfile.NamedTemporaryFile( + delete=False, + suffix=".mp4" + ).name + + ydl_opts = { + "outtmpl": output, + "format": "mp4", + "quiet": True + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + + return output \ No newline at end of file diff --git a/services/whisper/main.py b/services/whisper/main.py new file mode 100644 index 0000000000000000000000000000000000000000..3d315cfa8be4859c913412a1d15931a2f91ff507 --- /dev/null +++ b/services/whisper/main.py @@ -0,0 +1,361 @@ +from fastapi import FastAPI, UploadFile, File, Form, Request +from fastapi.responses import FileResponse, JSONResponse +from contextlib import asynccontextmanager +import os +import uuid +import asyncio +from pathlib import Path +import gradio as gr + +# ============================== +# LOGGER + QUEUE +# ============================== +from utils.logger import logger +from utils.job_queue import start_worker, create_job, get_job +from ingestion.resolver import resolve_input + +# ============================== +# AUTH SYSTEM (HARDENED IMPORT) +# ============================== +try: + from auth.routes import router as auth_router + from auth.database import Base, engine + AUTH_ENABLED = True +except Exception as e: + logger.error(f"[AUTH BOOT FAILED] {e}") + AUTH_ENABLED = False + +# ============================== +# CORE PIPELINE +# ============================== +from utils.transcription import transcribe_video +from utils.srt import generate_srt +from utils.render import render_subtitles +from utils.highlights import detect_highlights +from utils.viral_scorer import score_clip +from utils.director import rewrite_script, viral_score +from utils.engagement import simulate_retention +from utils.platform import adapt_platform +from utils.persona import predict_audience +from utils.clipper import create_clips +from utils.autonomous_engine import run_autonomous_engine + +# ============================== +# PUBLISHER +# ============================== +from publisher.publisher_ai import autonomous_loop +from publisher.scheduler_engine import init_scheduler +from publisher.platform_dispatcher import dispatch_publish +from publisher.bulk import execute as bulk_execute +from publisher.metadata_engine import generate_metadata +from publisher.thumbnail_engine import generate_thumbnail + +# ============================== +# SAFE bcrypt SHIELD (DO NOT FAIL BOOT) +# ============================== +try: + import bcrypt +except Exception as e: + logger.warning(f"[bcrypt warning ignored] {e}") + +# ============================== +# INIT +# ============================== +BASE_DIR = Path(__file__).resolve().parent +UPLOAD_DIR = str(BASE_DIR / "jobs") +os.makedirs(UPLOAD_DIR, exist_ok=True) + + +# ========================================================= +# SAFE DB INITIALIZATION +# ========================================================= +def init_database_safe(): + if not AUTH_ENABLED: + logger.warning("Auth disabled - skipping DB init") + return + + try: + Base.metadata.create_all(bind=engine) + logger.info("Database initialized successfully") + except Exception as e: + logger.error(f"Database init failed (non-fatal): {e}") + + +# ========================================================= +# LIFECYCLE +# ========================================================= +@asynccontextmanager +async def lifespan(app: FastAPI): + + logger.info("Starting Basyx Whisper V10.1") + + # DB init (NON-FATAL) + init_database_safe() + + # workers (must not block boot) + try: + start_worker() + except Exception as e: + logger.error(f"Worker failed: {e}") + + # scheduler + try: + init_scheduler() + except Exception as e: + logger.error(f"Scheduler failed: {e}") + + # autonomous engine (isolated task) + try: + asyncio.create_task(autonomous_loop()) + except Exception as e: + logger.error(f"Autonomous engine failed: {e}") + + yield + + logger.info("Shutdown complete") + + +# ========================================================= +# APP +# ========================================================= +app = FastAPI( + title="Basyx Whisper V10.1 Autonomous Operator", + lifespan=lifespan, +) + +# AUTH ROUTER (only if available) +if AUTH_ENABLED: + app.include_router(auth_router) + + +# ========================================================= +# TASKS +# ========================================================= +VALID_TASKS = { + "autonomous", + "auto-publish", + "publish", + "bulk-publish", + "generate-metadata", + "generate-thumbnail", + "schedule-post", + "transcribe", + "subtitles", + "render", + "highlights", + "viral-score", + "strategy", + "batch", + "clips", +} + + +def normalize_task(task: str): + task = task.lower().replace("_", "-") + if task not in VALID_TASKS: + raise Exception(f"Unknown task: {task}") + return task + + +# ========================================================= +# SAFE INPUT RESOLVER +# ========================================================= +async def safe_resolve(file, source): + try: + if not file and not source: + return None + + upload_file = file if isinstance(file, UploadFile) else None + + return await asyncio.to_thread(resolve_input, source, upload_file) + + except Exception as e: + logger.error(f"resolve_input failed: {e}") + return None + + +# ========================================================= +# EXECUTION ENGINE +# ========================================================= +async def execute_task(video_path, task, payload=None, webhook=None): + + payload = payload or {} + + if task == "bulk-publish": + return await bulk_execute(payload), None + + if task not in ["bulk-publish", "schedule-post"] and not video_path: + return {"error": "No valid input resolved"}, None + + if task == "autonomous": + return await asyncio.to_thread(run_autonomous_engine, video_path), None + + if task == "auto-publish": + auto = await asyncio.to_thread(run_autonomous_engine, video_path) + return await dispatch_publish(variants=auto.get("all_variants", [])), None + + if task == "publish": + return await dispatch_publish(video_path=video_path, payload=payload), None + + if task == "generate-metadata": + return generate_metadata(video_path), None + + if task == "generate-thumbnail": + output_path = os.path.join(UPLOAD_DIR, f"{uuid.uuid4()}.jpg") + thumb = generate_thumbnail(video_path, output=output_path) + return {"thumbnail": thumb}, output_path + + if task == "batch": + job_id = create_job(video_path, webhook=webhook) + return {"status": "queued", "job_id": job_id}, None + + if task == "transcribe": + words = await asyncio.to_thread(transcribe_video, video_path) + return {"words": words}, None + + if task == "subtitles": + words = await asyncio.to_thread(transcribe_video, video_path) + return {"srt": generate_srt(words)}, None + + if task == "render": + words = await asyncio.to_thread(transcribe_video, video_path) + srt = generate_srt(words) + + output = os.path.join(UPLOAD_DIR, f"{uuid.uuid4()}_render.mp4") + + await asyncio.to_thread(render_subtitles, video_path, srt, output) + + return {"status": "render_complete"}, output + + if task == "highlights": + words = await asyncio.to_thread(transcribe_video, video_path) + highlights = detect_highlights(words) or [] + clips = create_clips(video_path, highlights) + + return {"clips_created": len(clips)}, (clips[0] if clips else None) + + if task == "clips": + words = await asyncio.to_thread(transcribe_video, video_path) + highlights = detect_highlights(words) or [] + return {"clips": create_clips(video_path, highlights)}, None + + if task == "viral-score": + words = await asyncio.to_thread(transcribe_video, video_path) + segments = detect_highlights(words) or [] + return {"scores": [score_clip(s) for s in segments]}, None + + if task == "strategy": + words = await asyncio.to_thread(transcribe_video, video_path) + + script = rewrite_script(words) + persona = predict_audience(words) + curve = simulate_retention(words) + + return { + "hook": script["hook"], + "persona": persona, + "viral_score": viral_score(curve), + "platforms": { + "tiktok": adapt_platform(script, "tiktok"), + "reels": adapt_platform(script, "reels"), + }, + }, None + + return {"error": "Task execution failed"}, None + + +# ========================================================= +# ROUTER +# ========================================================= +@app.post("/execute/{task_name}") +async def execute_router( + request: Request, + task_name: str, + file: UploadFile = File(None), + url_input: str = Form(None), + source: str = Form(None), + webhook: str = Form(None), +): + + try: + task = normalize_task(task_name) + payload = {} + + if request.headers.get("content-type", "").startswith("application/json"): + payload = await request.json() + + video_path = await safe_resolve(file, url_input or source) + + result, output = await execute_task(video_path, task, payload, webhook) + + if output and isinstance(output, str) and os.path.exists(output): + return FileResponse(output) + + return {"task": task, "result": result} + + except Exception as e: + logger.exception(e) + return JSONResponse({"error": str(e)}, status_code=500) + + +# ========================================================= +# HEALTH +# ========================================================= +@app.get("/api/health") +def health(): + return { + "status": "online", + "version": "V10.1", + "auth_enabled": AUTH_ENABLED + } + + +@app.get("/api/status/{job_id}") +def status(job_id: str): + return get_job(job_id) or {"error": "Job not found"} + + +# ========================================================= +# GRADIO UI +# ========================================================= +async def ui_handler(video, task, webhook, url_input): + + source = url_input or video + video_path = await safe_resolve(video, source) + + result, output = await execute_task( + video_path, + normalize_task(task), + {}, + webhook, + ) + + return str(result), output + + +with gr.Blocks() as demo: + + gr.Markdown("# 🚀 Basyx Whisper V10.1 Stable Operator") + + video_input = gr.Video() + url_input = gr.Textbox(label="Video URL") + + task_dropdown = gr.Dropdown( + choices=list(VALID_TASKS), + value="autonomous", + ) + + webhook_input = gr.Textbox(label="Webhook") + + run_btn = gr.Button("Execute") + + output_box = gr.Textbox() + video_output = gr.Video() + + run_btn.click( + ui_handler, + [video_input, task_dropdown, webhook_input, url_input], + [output_box, video_output], + ) + +app = gr.mount_gradio_app(app, demo, path="/") diff --git a/services/whisper/publisher/__init__.py b/services/whisper/publisher/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bec290187d7cb803cdaf40915b67a33220b15297 --- /dev/null +++ b/services/whisper/publisher/__init__.py @@ -0,0 +1 @@ +# Publisher package \ No newline at end of file diff --git a/services/whisper/publisher/account_manager.py b/services/whisper/publisher/account_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..3519dd1360823f2d2a14851e335cf40261b413f4 --- /dev/null +++ b/services/whisper/publisher/account_manager.py @@ -0,0 +1,38 @@ +# publisher/account_manager.py + +from publisher.oauth.storage import load_tokens + +SUPPORTED = [ + "youtube", + "tiktok", + "meta" +] + + +def connected_accounts(user_id): + + accounts = [] + + for p in SUPPORTED: + token = load_tokens(user_id, p) + if token: + accounts.append({ + "platform": p, + "token": token + }) + + return accounts + + +def choose_platforms(strategy, accounts): + + if strategy == "all": + return accounts + + if strategy == "short_video": + return [ + a for a in accounts + if a["platform"] in ["youtube", "tiktok", "meta"] + ] + + return accounts[:1] \ No newline at end of file diff --git a/services/whisper/publisher/ai/gemini_client.py b/services/whisper/publisher/ai/gemini_client.py new file mode 100644 index 0000000000000000000000000000000000000000..62727277ad088b98af9e3e6af3dd808ade4ae139 --- /dev/null +++ b/services/whisper/publisher/ai/gemini_client.py @@ -0,0 +1,83 @@ +import os +import time +import random +import logging + +logger = logging.getLogger("gemini-client") + +# ========================= +# MODEL POOL (Gemini 3 Era) +# ========================= + +PRIMARY_MODELS = [ + "gemini-3.1-pro", + "gemini-3.1-flash", +] + +FALLBACK_MODELS = [ + "gemini-2.5-flash", + "gemini-3.1-flash-lite", +] + +# ========================= +# SIMPLE QUOTA TRACKER +# ========================= +_model_fail_count = { + "gemini-3.1-pro": 0, + "gemini-3.1-flash": 0, +} + + +MAX_FAILS = 3 + + +# ========================= +# CORE MODEL RESOLVER +# ========================= + +def _pick_model(): + """ + Select best available model with fallback logic. + """ + for m in PRIMARY_MODELS: + if _model_fail_count.get(m, 0) < MAX_FAILS: + return m + + return random.choice(FALLBACK_MODELS) + + +# ========================= +# MAIN CLIENT INTERFACE +# ========================= + +def get_model(): + """ + Public entrypoint used by publisher_ai. + Returns active Gemini model name. + """ + model = _pick_model() + logger.info(f"[Gemini] Selected model: {model}") + return model + + +def safe_generate(prompt: str, client_callable): + """ + Wrapper for Gemini calls with automatic fallback. + """ + + last_error = None + + for _ in range(3): + model = _pick_model() + + try: + result = client_callable(model, prompt) + return result + + except Exception as e: + last_error = e + _model_fail_count[model] = _model_fail_count.get(model, 0) + 1 + logger.warning(f"[Gemini FAIL] {model}: {str(e)}") + time.sleep(0.5) + + raise RuntimeError(f"All Gemini models failed: {last_error}") \ No newline at end of file diff --git a/services/whisper/publisher/bulk.py b/services/whisper/publisher/bulk.py new file mode 100644 index 0000000000000000000000000000000000000000..42665bd9d5bc0c8c90b317a330c4541e551444c5 --- /dev/null +++ b/services/whisper/publisher/bulk.py @@ -0,0 +1,187 @@ +""" +bulk.py +V9 Autonomous Publisher Engine + +Purpose: +-------- +Handles BULK publishing across multiple platforms. + +Supports: +- TikTok +- Reels (Instagram) +- YouTube Shorts +- Facebook +- Any future platform adapter + +Design: +------- +Input -> Normalize -> Dispatch -> Execute -> Collect Results + +Production Features: +-------------------- +✔ async concurrency +✔ retry system +✔ per-platform isolation +✔ failure tolerance +✔ structured logging +✔ scheduler-compatible +✔ autonomous engine ready +""" + +import asyncio +import traceback +from typing import Dict, List, Any + +# Platform adapters +from publisher.platforms.tiktok import publish_tiktok +from publisher.platforms.reels import publish_reels +from publisher.platforms.shorts import publish_shorts +from publisher.platforms.facebook import publish_facebook + + +# ===================================================== +# PLATFORM REGISTRY +# ===================================================== + +PLATFORM_MAP = { + "tiktok": publish_tiktok, + "reels": publish_reels, + "shorts": publish_shorts, + "facebook": publish_facebook, +} + + +# ===================================================== +# CONFIG +# ===================================================== + +MAX_CONCURRENT_POSTS = 5 +MAX_RETRIES = 2 + + +# ===================================================== +# HELPERS +# ===================================================== + +async def execute_with_retry(func, payload: Dict, retries=MAX_RETRIES): + """ + Safe execution wrapper with retries. + """ + + attempt = 0 + + while attempt <= retries: + try: + result = await func(payload) + return { + "status": "success", + "result": result, + } + + except Exception as e: + attempt += 1 + + if attempt > retries: + return { + "status": "failed", + "error": str(e), + "trace": traceback.format_exc(), + } + + await asyncio.sleep(2) + + +# ===================================================== +# SINGLE JOB EXECUTOR +# ===================================================== + +async def process_job(job: Dict[str, Any]): + """ + Expected job format: + + { + "platform": "tiktok", + "video_url": "...", + "caption": "...", + "hashtags": [], + "thumbnail": "...", + "schedule_time": optional + } + """ + + platform = job.get("platform") + + if platform not in PLATFORM_MAP: + return { + "status": "failed", + "error": f"Unsupported platform: {platform}", + } + + publisher = PLATFORM_MAP[platform] + + return await execute_with_retry(publisher, job) + + +# ===================================================== +# BULK ENGINE +# ===================================================== + +async def bulk_publish(jobs: List[Dict[str, Any]]): + """ + Main bulk execution engine. + """ + + semaphore = asyncio.Semaphore(MAX_CONCURRENT_POSTS) + + results = [] + + async def limited_job(job): + async with semaphore: + return await process_job(job) + + tasks = [limited_job(job) for job in jobs] + + completed = await asyncio.gather(*tasks, return_exceptions=False) + + results.extend(completed) + + return summarize_results(results) + + +# ===================================================== +# SUMMARY +# ===================================================== + +def summarize_results(results: List[Dict]): + success = sum(1 for r in results if r["status"] == "success") + failed = len(results) - success + + return { + "status": "completed", + "total_jobs": len(results), + "successful": success, + "failed": failed, + "results": results, + } + + +# ===================================================== +# FASTAPI ENTRYPOINT +# ===================================================== + +async def execute(payload: Dict): + """ + Universal endpoint handler + + POST /execute/bulk_publish + """ + + jobs = payload.get("jobs") + + if not jobs: + return { + "status": "error", + "message": "No jobs provided", + } + + return await bulk_publish(jobs) \ No newline at end of file diff --git a/services/whisper/publisher/hashtags.py b/services/whisper/publisher/hashtags.py new file mode 100644 index 0000000000000000000000000000000000000000..f629dd11f3ba34e6e87a53153f32f1e5d9a3aea0 --- /dev/null +++ b/services/whisper/publisher/hashtags.py @@ -0,0 +1,19 @@ +from publisher.ai.gemini_client import get_model + + +def generate_hashtags(video_path: str): + + model = get_model() + + prompt = """ + Generate 20 viral hashtags for short-form content. + Return comma-separated only. + """ + + res = model.generate_content(prompt) + + tags = res.text.replace("\n", "").strip() + + return { + "hashtags": tags + } \ No newline at end of file diff --git a/services/whisper/publisher/metadata_engine.py b/services/whisper/publisher/metadata_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..2701d104905703171186c0c98d95b56e28635e91 --- /dev/null +++ b/services/whisper/publisher/metadata_engine.py @@ -0,0 +1,24 @@ +from publisher.ai.gemini_client import get_model + + +def generate_metadata(video_path: str): + + model = get_model() + + prompt = f""" + Generate viral short-form video metadata. + + Return JSON: + title + description + hook + audience + """ + + response = model.generate_content(prompt) + + text = response.text.strip() + + return { + "metadata": text + } \ No newline at end of file diff --git a/services/whisper/publisher/models.py b/services/whisper/publisher/models.py new file mode 100644 index 0000000000000000000000000000000000000000..5e79623c345cd1a385a13efe5b721f319b51a1bf --- /dev/null +++ b/services/whisper/publisher/models.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel +from typing import List, Optional + + +class PublishPayload(BaseModel): + + video_path: str + + title: str + description: str + + hashtags: List[str] + + thumbnail: Optional[str] = None + schedule_time: Optional[int] = None + + platforms: List[str] \ No newline at end of file diff --git a/services/whisper/publisher/oauth/providers/google.py b/services/whisper/publisher/oauth/providers/google.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1c26907089a06151c177d2d194077673313cb1 --- /dev/null +++ b/services/whisper/publisher/oauth/providers/google.py @@ -0,0 +1,52 @@ +# publisher/oauth/providers/google.py + +import httpx +import os +from ..storage import save_tokens + +CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID") +CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET") + +REDIRECT_URI = "https://your-domain.com/oauth/callback/google" + +AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth" +TOKEN_URL = "https://oauth2.googleapis.com/token" + + +def authorization_url(state): + + scope = ( + "https://www.googleapis.com/auth/youtube.upload " + "https://www.googleapis.com/auth/userinfo.profile" + ) + + return ( + f"{AUTH_URL}" + f"?client_id={CLIENT_ID}" + f"&redirect_uri={REDIRECT_URI}" + f"&response_type=code" + f"&scope={scope}" + f"&access_type=offline" + f"&state={state}" + ) + + +async def exchange_code(code): + + async with httpx.AsyncClient() as client: + r = await client.post( + TOKEN_URL, + data={ + "code": code, + "client_id": CLIENT_ID, + "client_secret": CLIENT_SECRET, + "redirect_uri": REDIRECT_URI, + "grant_type": "authorization_code", + }, + ) + + return r.json() + + +def store(user_id, tokens): + save_tokens(user_id, "google", tokens) \ No newline at end of file diff --git a/services/whisper/publisher/oauth/providers/meta.py b/services/whisper/publisher/oauth/providers/meta.py new file mode 100644 index 0000000000000000000000000000000000000000..117e59090b973283a312424d021358175fe27202 --- /dev/null +++ b/services/whisper/publisher/oauth/providers/meta.py @@ -0,0 +1,42 @@ +# publisher/oauth/providers/meta.py + +import httpx +import os +from ..storage import save_tokens + +APP_ID = os.getenv("META_APP_ID") +APP_SECRET = os.getenv("META_APP_SECRET") + +REDIRECT_URI = "https://your-domain.com/oauth/callback/meta" + + +def authorization_url(state): + + return ( + "https://www.facebook.com/v19.0/dialog/oauth" + f"?client_id={APP_ID}" + f"&redirect_uri={REDIRECT_URI}" + "&scope=pages_manage_posts,pages_read_engagement," + "instagram_content_publish" + f"&state={state}" + ) + + +async def exchange_code(code): + + async with httpx.AsyncClient() as client: + r = await client.get( + "https://graph.facebook.com/v19.0/oauth/access_token", + params={ + "client_id": APP_ID, + "redirect_uri": REDIRECT_URI, + "client_secret": APP_SECRET, + "code": code, + }, + ) + + return r.json() + + +def store(user_id, tokens): + save_tokens(user_id, "meta", tokens) \ No newline at end of file diff --git a/services/whisper/publisher/oauth/providers/tiktok.py b/services/whisper/publisher/oauth/providers/tiktok.py new file mode 100644 index 0000000000000000000000000000000000000000..bb16727a88226ec77dedc69f352b8f26763f8ead --- /dev/null +++ b/services/whisper/publisher/oauth/providers/tiktok.py @@ -0,0 +1,45 @@ +# publisher/oauth/providers/tiktok.py + +import httpx +import os +from ..storage import save_tokens + +CLIENT_KEY = os.getenv("TIKTOK_CLIENT_KEY") +CLIENT_SECRET = os.getenv("TIKTOK_CLIENT_SECRET") + +REDIRECT_URI = "https://your-domain.com/oauth/callback/tiktok" + + +def authorization_url(state): + + scope = "user.info.basic,video.upload" + + return ( + "https://www.tiktok.com/v2/auth/authorize/" + f"?client_key={CLIENT_KEY}" + f"&response_type=code" + f"&scope={scope}" + f"&redirect_uri={REDIRECT_URI}" + f"&state={state}" + ) + + +async def exchange_code(code): + + async with httpx.AsyncClient() as client: + r = await client.post( + "https://open.tiktokapis.com/v2/oauth/token/", + data={ + "client_key": CLIENT_KEY, + "client_secret": CLIENT_SECRET, + "code": code, + "grant_type": "authorization_code", + "redirect_uri": REDIRECT_URI, + }, + ) + + return r.json() + + +def store(user_id, tokens): + save_tokens(user_id, "tiktok", tokens) \ No newline at end of file diff --git a/services/whisper/publisher/oauth/providers/youtube.py b/services/whisper/publisher/oauth/providers/youtube.py new file mode 100644 index 0000000000000000000000000000000000000000..6c6f2ca063752921660875797171ec5f14a30fb5 --- /dev/null +++ b/services/whisper/publisher/oauth/providers/youtube.py @@ -0,0 +1,8 @@ +# publisher/oauth/providers/youtube.py + +from .google import authorization_url, exchange_code +from ..storage import save_tokens + + +def store(user_id, tokens): + save_tokens(user_id, "youtube", tokens) \ No newline at end of file diff --git a/services/whisper/publisher/oauth/router.py b/services/whisper/publisher/oauth/router.py new file mode 100644 index 0000000000000000000000000000000000000000..9b65e3719eef2cc5d535f465c178b5c6eaf8d783 --- /dev/null +++ b/services/whisper/publisher/oauth/router.py @@ -0,0 +1,47 @@ +# publisher/oauth/router.py + +from fastapi import APIRouter, Request +from fastapi.responses import RedirectResponse + +from .sessions import create_session, get_user +from .providers import google, meta, tiktok, youtube + +router = APIRouter() + +PROVIDERS = { + "google": google, + "meta": meta, + "tiktok": tiktok, + "youtube": youtube +} + + +@router.get("/connect/{provider}") +async def connect(provider: str, user_id: str): + + if provider not in PROVIDERS: + return {"error": "provider not supported"} + + state = create_session(user_id) + + url = PROVIDERS[provider].authorization_url(state) + + return RedirectResponse(url) + + +@router.get("/callback/{provider}") +async def callback(provider: str, request: Request): + + if provider not in PROVIDERS: + return {"error": "provider not supported"} + + state = request.query_params.get("state") + code = request.query_params.get("code") + + user_id = get_user(state) + + tokens = await PROVIDERS[provider].exchange_code(code) + + PROVIDERS[provider].store(user_id, tokens) + + return {"status": f"{provider} connected"} \ No newline at end of file diff --git a/services/whisper/publisher/oauth/sessions.py b/services/whisper/publisher/oauth/sessions.py new file mode 100644 index 0000000000000000000000000000000000000000..4291d29179ed7056a4afdf90dc7659fa74e6605c --- /dev/null +++ b/services/whisper/publisher/oauth/sessions.py @@ -0,0 +1,29 @@ +# publisher/oauth/sessions.py + +import secrets +import time + +_sessions = {} + + +def create_session(user_id): + state = secrets.token_urlsafe(32) + _sessions[state] = { + "user_id": user_id, + "created": time.time() + } + return state + + +def get_user(state): + session = _sessions.get(state) + if not session: + return None + return session["user_id"] + + +def cleanup(expiry=600): + now = time.time() + for k in list(_sessions.keys()): + if now - _sessions[k]["created"] > expiry: + del _sessions[k] \ No newline at end of file diff --git a/services/whisper/publisher/oauth/storage.py b/services/whisper/publisher/oauth/storage.py new file mode 100644 index 0000000000000000000000000000000000000000..429af5913f2fe294c8c408233e6137c2a5aad4aa --- /dev/null +++ b/services/whisper/publisher/oauth/storage.py @@ -0,0 +1,44 @@ +# publisher/oauth/storage.py + +import os +import json +from pathlib import Path +from cryptography.fernet import Fernet + +STORAGE_DIR = Path("oauth_tokens") +STORAGE_DIR.mkdir(exist_ok=True) + +KEY_PATH = STORAGE_DIR / "secret.key" + + +def load_key(): + if not KEY_PATH.exists(): + key = Fernet.generate_key() + KEY_PATH.write_bytes(key) + return KEY_PATH.read_bytes() + + +fernet = Fernet(load_key()) + + +def _file(user_id, provider): + return STORAGE_DIR / f"{user_id}_{provider}.json" + + +def save_tokens(user_id: str, provider: str, data: dict): + encrypted = fernet.encrypt(json.dumps(data).encode()) + _file(user_id, provider).write_bytes(encrypted) + + +def load_tokens(user_id: str, provider: str): + f = _file(user_id, provider) + if not f.exists(): + return None + decrypted = fernet.decrypt(f.read_bytes()) + return json.loads(decrypted.decode()) + + +def delete_tokens(user_id: str, provider: str): + f = _file(user_id, provider) + if f.exists(): + f.unlink() \ No newline at end of file diff --git a/services/whisper/publisher/platform_dispatcher.py b/services/whisper/publisher/platform_dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..48ad6aaa13e0e8ece47b3776ada3dd5e94dc2d23 --- /dev/null +++ b/services/whisper/publisher/platform_dispatcher.py @@ -0,0 +1,48 @@ +from importlib import import_module +import logging + +logger = logging.getLogger("platform-dispatcher") + + +def _safe_import(module_path, fn_name): + try: + module = import_module(module_path) + return getattr(module, fn_name) + except Exception as e: + logger.warning(f"[Dispatcher] Missing {module_path}: {str(e)}") + return None + + +# lazy-loaded publishers (prevents boot crash) + +def dispatch_publish(video_path=None, payload=None, variants=None): + + payload = payload or {} + results = {} + + youtube = _safe_import("publisher.platforms.youtube", "publish_youtube") + tiktok = _safe_import("publisher.platforms.tiktok", "publish_tiktok") + reels = _safe_import("publisher.platforms.reels", "publish_reels") + shorts = _safe_import("publisher.platforms.shorts", "publish_shorts") + facebook = _safe_import("publisher.platforms.facebook", "publish_facebook") + + async def run(): + + if youtube: + results["youtube"] = await youtube({"video_path": video_path, **payload}) + + if tiktok: + results["tiktok"] = await tiktok({"video_path": video_path, **payload}) + + if reels: + results["reels"] = await reels({"video_path": video_path, **payload}) + + if shorts: + results["shorts"] = await shorts({"video_path": video_path, **payload}) + + if facebook: + results["facebook"] = await facebook({"video_path": video_path, **payload}) + + return results + + return run() \ No newline at end of file diff --git a/services/whisper/publisher/platforms/__init__.py b/services/whisper/publisher/platforms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5662f3cfd1380a213cfd3a2466756fc6191f9342 --- /dev/null +++ b/services/whisper/publisher/platforms/__init__.py @@ -0,0 +1 @@ +# Platforms package \ No newline at end of file diff --git a/services/whisper/publisher/platforms/auth.py b/services/whisper/publisher/platforms/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..4fe8a84c54cb457f143b218fa5b92aace5efb3aa --- /dev/null +++ b/services/whisper/publisher/platforms/auth.py @@ -0,0 +1,38 @@ +import os + +# ========================= +# SIMPLE ENV AUTH LOADER +# ========================= + +def load_env_var(key: str, default=None): + """ + Safe environment variable loader. + Works without python-dotenv dependency. + """ + return os.environ.get(key, default) + + +def get_token(platform: str): + """ + Returns stored token for platform. + No external dependency version. + """ + + key_map = { + "tiktok": "TIKTOK_TOKEN", + "youtube": "YOUTUBE_TOKEN", + "facebook": "FACEBOOK_TOKEN", + "reels": "META_TOKEN", + } + + env_key = key_map.get(platform) + + if not env_key: + raise ValueError(f"Unsupported platform: {platform}") + + token = load_env_var(env_key) + + if not token: + raise ValueError(f"Missing token for {platform} ({env_key})") + + return token \ No newline at end of file diff --git a/services/whisper/publisher/platforms/base.py b/services/whisper/publisher/platforms/base.py new file mode 100644 index 0000000000000000000000000000000000000000..5346aaf83b132a93ab908e651f6d971cb45de4fd --- /dev/null +++ b/services/whisper/publisher/platforms/base.py @@ -0,0 +1,15 @@ +import os + +def validate_payload(payload): + + if "video_path" not in payload: + raise Exception("video_path required") + + if not os.path.exists(payload["video_path"]): + raise Exception("Video missing") + + payload.setdefault("caption", "") + payload.setdefault("hashtags", []) + payload.setdefault("thumbnail", None) + + return payload \ No newline at end of file diff --git a/services/whisper/publisher/platforms/errors.py b/services/whisper/publisher/platforms/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..053523e29eab094cf189b6ac7181c159091e32af --- /dev/null +++ b/services/whisper/publisher/platforms/errors.py @@ -0,0 +1,14 @@ +class PublisherError(Exception): + pass + + +class PlatformAuthError(PublisherError): + pass + + +class PlatformUploadError(PublisherError): + pass + + +class PlatformRateLimit(PublisherError): + pass \ No newline at end of file diff --git a/services/whisper/publisher/platforms/facebook.py b/services/whisper/publisher/platforms/facebook.py new file mode 100644 index 0000000000000000000000000000000000000000..3c50a88ebf691187d20d51a83748dc44aa940415 --- /dev/null +++ b/services/whisper/publisher/platforms/facebook.py @@ -0,0 +1,28 @@ +# publisher/platforms/facebook.py + +import httpx +from .auth import get_token +from .base import validate_payload + + +async def publish_facebook(payload): + + payload = validate_payload(payload) + + page_id = get_token("FACEBOOK_PAGE_ID") + token = get_token("META_ACCESS_TOKEN") + + async with httpx.AsyncClient(timeout=600) as client: + + with open(payload["video_path"], "rb") as f: + + res = await client.post( + f"https://graph-video.facebook.com/{page_id}/videos", + data={ + "description": payload["caption"], + "access_token": token, + }, + files={"source": f}, + ) + + return {"platform": "facebook", "status": "published"} \ No newline at end of file diff --git a/services/whisper/publisher/platforms/http.py b/services/whisper/publisher/platforms/http.py new file mode 100644 index 0000000000000000000000000000000000000000..2551d337551846ae86b9093722bca0d6ec1cdf92 --- /dev/null +++ b/services/whisper/publisher/platforms/http.py @@ -0,0 +1,38 @@ +import httpx +import asyncio + + +async def request( + method, + url, + headers=None, + data=None, + json=None, + files=None, + retries=3, +): + + for attempt in range(retries): + + try: + async with httpx.AsyncClient(timeout=120) as client: + + r = await client.request( + method, + url, + headers=headers, + data=data, + json=json, + files=files, + ) + + r.raise_for_status() + + return r.json() + + except Exception as e: + + if attempt == retries - 1: + raise + + await asyncio.sleep(2 ** attempt) \ No newline at end of file diff --git a/services/whisper/publisher/platforms/reels.py b/services/whisper/publisher/platforms/reels.py new file mode 100644 index 0000000000000000000000000000000000000000..059bab8695a7e3183920cb73f19981d99565a094 --- /dev/null +++ b/services/whisper/publisher/platforms/reels.py @@ -0,0 +1,39 @@ +# publisher/platforms/reels.py + +import httpx +from .auth import get_token +from .base import validate_payload + + +async def publish_reels(payload): + + payload = validate_payload(payload) + + token = get_token("META_ACCESS_TOKEN") + ig_id = get_token("INSTAGRAM_ACCOUNT_ID") + + async with httpx.AsyncClient() as client: + + # Create media container + create = await client.post( + f"https://graph.facebook.com/v19.0/{ig_id}/media", + data={ + "video_url": payload["video_path"], + "caption": payload["caption"], + "access_token": token, + "media_type": "REELS", + }, + ) + + container = create.json()["id"] + + # Publish + publish = await client.post( + f"https://graph.facebook.com/v19.0/{ig_id}/media_publish", + data={ + "creation_id": container, + "access_token": token, + }, + ) + + return {"platform": "reels", "status": "published"} \ No newline at end of file diff --git a/services/whisper/publisher/platforms/shorts.py b/services/whisper/publisher/platforms/shorts.py new file mode 100644 index 0000000000000000000000000000000000000000..7435f1aa74047f22f8799232c7cb549bc55c4f29 --- /dev/null +++ b/services/whisper/publisher/platforms/shorts.py @@ -0,0 +1,37 @@ +from googleapiclient.discovery import build +from googleapiclient.http import MediaFileUpload +from .base import validate_payload +from .auth import get_token + + +async def publish_shorts(payload: dict): + + payload = validate_payload(payload) + + youtube = build( + "youtube", + "v3", + developerKey=get_token("YOUTUBE_API_KEY"), + ) + + request = youtube.videos().insert( + part="snippet,status", + body={ + "snippet": { + "title": payload["caption"][:90], + "description": payload["caption"], + "tags": payload["hashtags"], + "categoryId": "22", + }, + "status": {"privacyStatus": "public"}, + }, + media_body=MediaFileUpload(payload["video_path"]), + ) + + response = request.execute() + + return { + "platform": "shorts", + "video_id": response["id"], + "status": "published", + } \ No newline at end of file diff --git a/services/whisper/publisher/platforms/tiktok.py b/services/whisper/publisher/platforms/tiktok.py new file mode 100644 index 0000000000000000000000000000000000000000..a97a0cff8713e7dfbf76419c2718d5b9ac1625fa --- /dev/null +++ b/services/whisper/publisher/platforms/tiktok.py @@ -0,0 +1,34 @@ +# publisher/platforms/tiktok.py + +import httpx +from .base import validate_payload +from .auth import get_token + + +async def publish_tiktok(payload: dict): + + payload = validate_payload(payload) + + token = get_token("TIKTOK_ACCESS_TOKEN") + + async with httpx.AsyncClient(timeout=600) as client: + + # Step 1 — create upload session + init = await client.post( + "https://open.tiktokapis.com/v2/post/publish/video/init/", + headers={"Authorization": f"Bearer {token}"}, + json={ + "post_info": { + "title": payload["caption"], + "privacy_level": "PUBLIC", + } + }, + ) + + upload_url = init.json()["data"]["upload_url"] + + # Step 2 — upload video + with open(payload["video_path"], "rb") as f: + await client.put(upload_url, content=f) + + return {"platform": "tiktok", "status": "published"} \ No newline at end of file diff --git a/services/whisper/publisher/platforms/youtube.py b/services/whisper/publisher/platforms/youtube.py new file mode 100644 index 0000000000000000000000000000000000000000..77a05346e70a84987b7c8b8dbd613ef424009d22 --- /dev/null +++ b/services/whisper/publisher/platforms/youtube.py @@ -0,0 +1,40 @@ +import logging + +logger = logging.getLogger("youtube-publisher") + + +# ========================= +# YOUTUBE PUBLISHER (V10 SAFE) +# ========================= + +async def publish_youtube(payload: dict): + """ + Production-safe YouTube publishing interface. + + Expected payload: + { + "video_path": str, + "title": str, + "description": str, + "tags": list[str], + "schedule_time": optional + } + """ + + video_path = payload.get("video_path") + + if not video_path: + raise ValueError("video_path is required") + + logger.info(f"[YouTube] Publishing video: {video_path}") + + # NOTE: + # No API dependency here yet (SDK layer should be injected via OAuth system) + # This prevents startup crashes on missing credentials. + + return { + "status": "queued", + "platform": "youtube", + "video": video_path, + "message": "YouTube publish request accepted (SDK layer pending OAuth connection)" + } \ No newline at end of file diff --git a/services/whisper/publisher/publisher.py b/services/whisper/publisher/publisher.py new file mode 100644 index 0000000000000000000000000000000000000000..eb9fdbfbd367d01971d7b7e1201e4274b2ffced6 --- /dev/null +++ b/services/whisper/publisher/publisher.py @@ -0,0 +1,52 @@ +import requests + +from .metadata import generate_metadata +from .hashtags import generate_hashtags +from .thumbnail import generate_thumbnail +from .scheduler import schedule_post +from .router import publish_all + + +async def run_publisher(payload: dict): + + source = payload["source"] + webhook = payload.get("webhook") + platforms = payload.get( + "platforms", + ["tiktok", "reels", "shorts", "facebook"] + ) + + # 1. Metadata + metadata = await generate_metadata(source) + + # 2. Hashtags + hashtags = await generate_hashtags(metadata) + + # 3. Thumbnail + thumbnail = await generate_thumbnail(source) + + # 4. Schedule + schedule_time = schedule_post(payload) + + # 5. Publish + results = await publish_all( + source=source, + platforms=platforms, + metadata=metadata, + hashtags=hashtags, + thumbnail=thumbnail, + schedule_time=schedule_time, + ) + + response = { + "status": "completed", + "metadata": metadata, + "hashtags": hashtags, + "thumbnail": thumbnail, + "results": results, + } + + if webhook: + requests.post(webhook, json=response) + + return response \ No newline at end of file diff --git a/services/whisper/publisher/publisher_ai.py b/services/whisper/publisher/publisher_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..71624ac2565fa190118c7fcb71828c215df22145 --- /dev/null +++ b/services/whisper/publisher/publisher_ai.py @@ -0,0 +1,213 @@ +import asyncio +import logging +from typing import Optional, Dict, Any + +logger = logging.getLogger("publisher_ai") + + +# ===================================================== +# EXTERNAL DEPENDENCIES (lazy-safe imports) +# ===================================================== + +try: + from publisher.scheduler_engine import get_next_job +except Exception: + get_next_job = None + +try: + from publisher.platform_dispatcher import dispatch_publish +except Exception: + dispatch_publish = None + + +# ===================================================== +# AUTONOMOUS PROCESSOR CORE +# ===================================================== + +async def process_job(job: Dict[str, Any]) -> Dict[str, Any]: + """ + Executes a single publishing job. + This is the atomic unit of the autonomous system. + """ + + logger.info(f"[AI] Processing job: {job.get('id', 'unknown')}") + + job_type = job.get("type") + payload = job.get("payload", {}) + + if job_type == "publish": + if not dispatch_publish: + raise RuntimeError("dispatch_publish not available") + + return await dispatch_publish( + video_path=payload.get("video_path"), + payload=payload + ) + + if job_type == "auto-publish": + # already pre-processed variants expected + return await dispatch_publish( + variants=payload.get("variants", []) + ) + + if job_type == "bulk": + # bulk jobs are handled upstream + return {"status": "forwarded_bulk"} + + return {"status": "ignored", "reason": "unknown_job_type"} + + +# ===================================================== +# AUTONOMOUS LOOP (FIXED - NO REQUIRED ARGUMENTS) +# ===================================================== + +async def autonomous_loop(seed_video_path: Optional[str] = None): + """ + V10 Autonomous Publisher Brain Loop + + FIXES: + - no required positional arguments + - safe startup execution + - scheduler-driven architecture + - continuous polling worker + """ + + logger.info("[V10] Autonomous Publisher Brain started") + + if seed_video_path: + logger.info(f"[V10] Seed video detected (optional): {seed_video_path}") + + # optional warm-up behavior (non-blocking) + if seed_video_path: + asyncio.create_task(_warmup_seed(seed_video_path)) + + # ================================================= + # MAIN LOOP + # ================================================= + while True: + try: + + # 1. Pull job from scheduler/queue + job = None + + if get_next_job: + job = await get_next_job() + + # 2. Idle state handling + if not job: + await asyncio.sleep(3) + continue + + logger.info(f"[V10] Job received: {job.get('id')}") + + # 3. Process job + result = await process_job(job) + + logger.info(f"[V10] Job completed: {job.get('id')}") + + # 4. Optional webhook callback + webhook = job.get("webhook") + if webhook: + asyncio.create_task(_send_webhook(webhook, result)) + + except Exception as e: + logger.exception(f"[V10] Loop error: {str(e)}") + await asyncio.sleep(2) + + +# ===================================================== +# OPTIONAL WARMUP PIPELINE +# ===================================================== + +async def _warmup_seed(video_path: str): + """ + Optional: runs once at startup if seed video exists + """ + + try: + logger.info(f"[V10] Warmup processing seed video: {video_path}") + + if not dispatch_publish: + logger.warning("dispatch_publish not available during warmup") + return + + await dispatch_publish( + video_path=video_path, + payload={"mode": "warmup"} + ) + + except Exception as e: + logger.exception(f"[V10] Warmup failed: {str(e)}") + + +# ===================================================== +# WEBHOOK HANDLER +# ===================================================== + +async def _send_webhook(url: str, data: dict): + """ + Lightweight webhook sender (no external dependency required) + """ + + try: + import json + import urllib.request + + payload = json.dumps(data).encode("utf-8") + + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"} + ) + + urllib.request.urlopen(req, timeout=5) + + except Exception as e: + logger.warning(f"[V10] Webhook failed: {str(e)}") + + # ===================================================== +# PUBLIC ENTRYPOINT (REQUIRED BY MAIN APP) +# ===================================================== + +import asyncio + + +async def start_autonomous_brain(): + """ + Unified startup entry for Autonomous Publisher. + Safe background loop. + """ + + while True: + try: + # call your existing brain runner here + await asyncio.to_thread(run_autonomous_engine) + + except Exception as e: + print("Autonomous brain error:", e) + + # prevent CPU burn + await asyncio.sleep(30) + + import asyncio + + +async def start_brain(): + """ + V11 Standard Publisher Entry Point + """ + + while True: + try: + # 🔁 Replace this with your real engine function + # Example options: + # await asyncio.to_thread(run_autonomous_engine) + # await asyncio.to_thread(autonomous_loop) + # await asyncio.to_thread(run_brain) + + await asyncio.sleep(10) + + except Exception as e: + print("[Publisher Brain Error]", e) + await asyncio.sleep(10) \ No newline at end of file diff --git a/services/whisper/publisher/router.py b/services/whisper/publisher/router.py new file mode 100644 index 0000000000000000000000000000000000000000..06d143c0a0e08a8c5c0125a9c4c423113427a3c7 --- /dev/null +++ b/services/whisper/publisher/router.py @@ -0,0 +1,28 @@ +from publisher.platforms.tiktok import TikTokPublisher +from publisher.platforms.reels import ReelsPublisher +from publisher.platforms.shorts import ShortsPublisher +from publisher.platforms.facebook import FacebookPublisher + + +PUBLISHERS = { + "tiktok": TikTokPublisher(), + "reels": ReelsPublisher(), + "shorts": ShortsPublisher(), + "facebook": FacebookPublisher(), +} + + +async def publish(payload): + + results = {} + + for platform in payload.platforms: + + publisher = PUBLISHERS.get(platform) + + if not publisher: + continue + + results[platform] = await publisher.safe_publish(payload) + + return results \ No newline at end of file diff --git a/services/whisper/publisher/scheduler_engine.py b/services/whisper/publisher/scheduler_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..b0933d1d8d75578ebf076df782d91266470f7b00 --- /dev/null +++ b/services/whisper/publisher/scheduler_engine.py @@ -0,0 +1,109 @@ +import asyncio +import logging +from datetime import datetime, timedelta + +logger = logging.getLogger("scheduler-engine") + +# ========================= +# INTERNAL SCHEDULER STATE +# ========================= + +_scheduler_running = False +_tasks = [] # in-memory fallback (can later swap to Redis/DB) + + +# ========================= +# INIT ENTRYPOINT (FIX) +# ========================= + +def init_scheduler(): + """ + Called by main.py on startup. + Safe, idempotent scheduler bootstrap. + """ + global _scheduler_running + + if _scheduler_running: + logger.info("[Scheduler] Already running") + return + + _scheduler_running = True + logger.info("🚀 Scheduler Engine V10 initialized") + + +# ========================= +# CORE SCHEDULER API +# ========================= + +def schedule_post(payload: dict): + """ + Adds a post to the queue. + Expected payload: + { + "video_path": str, + "platform": str, + "publish_at": datetime ISO string + } + """ + + job = { + "id": f"job_{len(_tasks)+1}", + "payload": payload, + "status": "queued", + "created_at": datetime.utcnow().isoformat() + } + + _tasks.append(job) + + logger.info(f"[Scheduler] Job queued: {job['id']}") + + return job + + +# ========================= +# WORKER LOOP +# ========================= + +async def _worker_loop(): + """ + Background scheduler processor. + """ + + logger.info("[Scheduler] Worker loop started") + + while True: + try: + now = datetime.utcnow() + + for job in _tasks: + if job["status"] != "queued": + continue + + publish_time = job["payload"].get("publish_at") + + if not publish_time: + continue + + publish_time = datetime.fromisoformat(publish_time) + + if now >= publish_time: + logger.info(f"[Scheduler] Executing {job['id']}") + + # mark as done (actual publish handled elsewhere) + job["status"] = "ready" + + except Exception as e: + logger.error(f"[Scheduler Error] {str(e)}") + + await asyncio.sleep(5) + + +# ========================= +# OPTIONAL START LOOP +# ========================= + +def start_scheduler_loop(): + """ + Optional explicit background runner. + """ + asyncio.create_task(_worker_loop()) \ No newline at end of file diff --git a/services/whisper/publisher/tasks/auto_publish.py b/services/whisper/publisher/tasks/auto_publish.py new file mode 100644 index 0000000000000000000000000000000000000000..22a3831f07474471180b4f4d2fa321198b3be920 --- /dev/null +++ b/services/whisper/publisher/tasks/auto_publish.py @@ -0,0 +1,145 @@ +import asyncio +from datetime import datetime +import uuid + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + if isinstance(context, dict): + return { + "input_file": context.get("input_file"), + "url_input": context.get("url_input"), + "params": context.get("params", {}) or {}, + "platforms": context.get("platforms", ["tiktok", "reels"]) + } + + return { + "input_file": getattr(context, "input_file", None), + "url_input": getattr(context, "url_input", None), + "params": getattr(context, "params", {}) or {}, + "platforms": getattr(context, "platforms", ["tiktok", "reels"]) + } + + +# ------------------------------------------------- +# SAFE EXECUTOR WRAPPER (registry-first) +# ------------------------------------------------- + +async def run_task(task_name, payload): + """ + Uses V11 registry executor if available. + Falls back safely if not. + """ + + try: + from core.execution.executor import execute_task as registry_execute + + return await registry_execute(task_name, payload) + + except Exception: + return { + "status": "failed", + "task": task_name, + "message": "registry executor unavailable" + } + + +# ------------------------------------------------- +# MAIN AUTO-PUBLISH PIPELINE +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + # ------------------------------------------------- + # STEP 1 — TRANSCRIBE + # ------------------------------------------------- + transcript = await run_task("transcribe", ctx) + + if transcript.get("status") != "success": + return { + "status": "error", + "stage": "transcribe", + "detail": transcript + } + + # ------------------------------------------------- + # STEP 2 — STRATEGY GENERATION + # ------------------------------------------------- + strategy = await run_task("strategy", { + "text": transcript + }) + + # ------------------------------------------------- + # STEP 3 — METADATA GENERATION + # ------------------------------------------------- + metadata = await run_task("generate-metadata", { + "strategy": strategy + }) + + # ------------------------------------------------- + # STEP 4 — VARIANT BUILDING + # ------------------------------------------------- + platforms = ctx["platforms"] + + variants = [] + + for platform in platforms: + + variants.append({ + "platform": platform, + "content": strategy, + "metadata": metadata + }) + + # ------------------------------------------------- + # STEP 5 — PUBLISH + # ------------------------------------------------- + publish_results = [] + + for v in variants: + + result = await run_task("publish", { + "platform": v["platform"], + "content": v["content"], + "metadata": v["metadata"] + }) + + publish_results.append({ + "platform": v["platform"], + "result": result + }) + + # ------------------------------------------------- + # FINAL RESPONSE + # ------------------------------------------------- + + return { + "status": "success", + "pipeline": "auto_publish_v11", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "platforms": platforms, + "variants_created": len(variants), + "publish_results": publish_results + } + + + except Exception as e: + + return { + "status": "error", + "pipeline": "auto_publish_v11", + "message": str(e), + "stage": "auto_publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/autonomous_mode.py b/services/whisper/publisher/tasks/autonomous_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..27e98e44f822415a9acaab58ae4c8e56fbcd82ba --- /dev/null +++ b/services/whisper/publisher/tasks/autonomous_mode.py @@ -0,0 +1,106 @@ +import asyncio +from dataclasses import dataclass + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +@dataclass +class Context: + input_file: any = None + url_input: str = None + params: dict = None + + +def normalize_context(ctx): + if isinstance(ctx, dict): + return Context( + input_file=ctx.get("input_file"), + url_input=ctx.get("url_input"), + params=ctx.get("params", {}) or {} + ) + return ctx + + +# ------------------------------------------------- +# PIPELINE ORCHESTRATOR +# ------------------------------------------------- + +async def run(context): + """ + Autonomous pipeline: + 1. Transcribe + 2. Extract highlights + 3. Compute viral score + 4. Generate strategy + """ + + context = normalize_context(context) + + try: + # ------------------------------------------------- + # STEP 1 — TRANSCRIPTION + # ------------------------------------------------- + from publisher.tasks.transcribe import run as transcribe_task + + transcript = await transcribe_task(context) + + if transcript.get("status") != "success": + return { + "status": "error", + "stage": "transcribe", + "detail": transcript + } + + # ------------------------------------------------- + # STEP 2 — HIGHLIGHTS + # ------------------------------------------------- + from publisher.tasks.highlights import run as highlights_task + + highlights = await highlights_task(context) + + # ------------------------------------------------- + # STEP 3 — VIRAL SCORE + # ------------------------------------------------- + from publisher.tasks.viral_score import run as viral_task + + viral = await viral_task(context) + + # ------------------------------------------------- + # STEP 4 — STRATEGY GENERATION + # ------------------------------------------------- + from publisher.tasks.strategy import run as strategy_task + + strategy = await strategy_task(context) + + # ------------------------------------------------- + # AGGREGATED OUTPUT + # ------------------------------------------------- + + segments = transcript.get("segments", []) + full_text = " ".join([s["text"] for s in segments]) + + return { + "status": "success", + "pipeline": "autonomous-v11", + "summary": { + "segments": len(segments), + "highlights": highlights.get("count", 0), + "viral_score": viral.get("viral_score", 0), + }, + "transcript": { + "text": full_text, + "segments": segments + }, + "highlights": highlights.get("highlights", []), + "viral": viral, + "strategy": strategy + } + + except Exception as e: + return { + "status": "error", + "message": str(e), + "stage": "autonomous_pipeline_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/batch.py b/services/whisper/publisher/tasks/batch.py new file mode 100644 index 0000000000000000000000000000000000000000..1095eb5e179507c2d76462ea7d54b8f58eda3520 --- /dev/null +++ b/services/whisper/publisher/tasks/batch.py @@ -0,0 +1,173 @@ +import asyncio +import uuid +from datetime import datetime + +# Optional queue integration (safe fallback if not present) +try: + from utils.job_queue import create_job +except Exception: + create_job = None + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + if isinstance(context, dict): + return { + "items": context.get("items", []), + "webhook": context.get("webhook"), + "mode": context.get("mode", "sequential") + } + return { + "items": getattr(context, "items", []), + "webhook": getattr(context, "webhook", None), + "mode": getattr(context, "mode", "sequential") + } + + +# ------------------------------------------------- +# SINGLE TASK EXECUTOR WRAPPER +# ------------------------------------------------- + +async def execute_single(task_name, payload): + """ + Uses registry executor if available, otherwise returns structured fallback. + """ + + try: + from core.execution.executor import execute_task + + return await execute_task( + task_name, + payload + ) + + except Exception as e: + return { + "task": task_name, + "status": "failed", + "error": str(e) + } + + +# ------------------------------------------------- +# MAIN BATCH RUNNER +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + items = ctx["items"] + + if not items: + return { + "status": "error", + "message": "Batch requires 'items' list" + } + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + results = [] + failed = 0 + + # ------------------------------------------------- + # MODE: SEQUENTIAL EXECUTION + # ------------------------------------------------- + + if ctx["mode"] == "sequential": + + for i, item in enumerate(items): + + task_name = item.get("task") + payload = item.get("payload", {}) + + if not task_name: + results.append({ + "index": i, + "status": "skipped", + "reason": "missing task" + }) + continue + + result = await execute_single(task_name, payload) + + if isinstance(result, dict) and result.get("status") == "failed": + failed += 1 + + results.append({ + "index": i, + "task": task_name, + "result": result + }) + + # ------------------------------------------------- + # MODE: PARALLEL EXECUTION + # ------------------------------------------------- + + elif ctx["mode"] == "parallel": + + async def run_item(i, item): + task_name = item.get("task") + payload = item.get("payload", {}) + + if not task_name: + return { + "index": i, + "status": "skipped" + } + + result = await execute_single(task_name, payload) + + return { + "index": i, + "task": task_name, + "result": result + } + + results = await asyncio.gather( + *[run_item(i, item) for i, item in enumerate(items)] + ) + + else: + return { + "status": "error", + "message": f"Unsupported mode: {ctx['mode']}" + } + + # ------------------------------------------------- + # JOB QUEUE INTEGRATION (OPTIONAL) + # ------------------------------------------------- + + job_id = None + if create_job: + try: + job_id = create_job( + video_path=None, + webhook=ctx["webhook"], + metadata={ + "batch_id": batch_id, + "total": len(items), + "failed": failed + } + ) + except Exception: + job_id = None + + # ------------------------------------------------- + # FINAL RESPONSE + # ------------------------------------------------- + + return { + "status": "success", + "batch_id": batch_id, + "job_id": job_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "mode": ctx["mode"], + "total": len(items), + "failed": failed, + "results": results + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/batch_runner.py b/services/whisper/publisher/tasks/batch_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..1173ebca1bb7acdbe757e65080cd3bd34bbfe122 --- /dev/null +++ b/services/whisper/publisher/tasks/batch_runner.py @@ -0,0 +1,168 @@ +import uuid +import asyncio +from datetime import datetime + +from utils.logger import logger +from utils.job_queue import create_job, get_job + + +# ===================================================== +# VALIDATION +# ===================================================== + +def normalize_payload(payload: dict | None): + payload = payload or {} + + items = payload.get("items", []) + + if not isinstance(items, list): + raise ValueError("batch items must be a list") + + normalized = [] + + for i, item in enumerate(items): + if not isinstance(item, dict): + raise ValueError(f"batch item {i} must be an object") + + normalized.append({ + "id": item.get("id", str(uuid.uuid4())), + "task": item.get("task"), + "video_path": item.get("video_path"), + "source": item.get("source"), + "payload": item.get("payload", {}), + "webhook": item.get("webhook"), + }) + + if not normalized: + raise ValueError("batch cannot be empty") + + return normalized + + +# ===================================================== +# SAFE TASK EXECUTION WRAPPER +# ===================================================== + +async def execute_single(task_executor, item, index: int): + + try: + logger.info(f"[BATCH] Executing item {index} → {item['task']}") + + result = await task_executor( + item["task"], + { + **(item.get("payload") or {}), + "video_path": item.get("video_path"), + "source": item.get("source"), + }, + item.get("webhook"), + ) + + return { + "index": index, + "id": item["id"], + "task": item["task"], + "status": "success", + "result": result, + } + + except Exception as e: + + logger.exception(f"[BATCH ERROR] item {index}") + + return { + "index": index, + "id": item["id"], + "task": item.get("task"), + "status": "failed", + "error": str(e), + } + + +# ===================================================== +# CONCURRENCY CONTROLLER +# ===================================================== + +async def run_concurrent(tasks, executor, max_concurrency: int = 3): + + semaphore = asyncio.Semaphore(max_concurrency) + + async def bound(item, index): + async with semaphore: + return await execute_single(executor, item, index) + + return await asyncio.gather( + *[bound(item, i) for i, item in enumerate(tasks)] + ) + + +# ===================================================== +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ===================================================== + +async def run(payload: dict | None = None, context: dict | None = None): + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + items = normalize_payload(payload) + + logger.info(f"[BATCH] Starting batch_id={batch_id}, items={len(items)}") + + # ------------------------------------------------- + # EXECUTOR HOOK (inject from registry context) + # ------------------------------------------------- + + def executor(task_name, task_payload, webhook=None): + """ + This is intentionally abstract so it can plug into: + - registry executor + - legacy executor + - FastAPI layer + """ + + from core.execution.executor import execute_task + + return execute_task(task_name, task_payload, webhook) + + # ------------------------------------------------- + # RUN BATCH + # ------------------------------------------------- + + results = await run_concurrent(items, executor) + + success_count = len([r for r in results if r["status"] == "success"]) + failed_count = len(results) - success_count + + # ------------------------------------------------- + # OUTPUT CONTRACT + # ------------------------------------------------- + + return { + "status": "completed", + "task": "batch_runner", + + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + "total": len(items), + "success": success_count, + "failed": failed_count, + + "results": results, + } + + except Exception as e: + + logger.exception("[BATCH FATAL ERROR]") + + return { + "status": "error", + "task": "batch_runner", + "batch_id": batch_id, + "message": str(e), + "stage": "batch_execution_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/generate_metadata.py b/services/whisper/publisher/tasks/generate_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..ecac82d17a5e1da99db55e9d1fe7ba658c1d3843 --- /dev/null +++ b/services/whisper/publisher/tasks/generate_metadata.py @@ -0,0 +1,200 @@ +import asyncio +from datetime import datetime +import uuid + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + """ + Accepts registry dict or legacy object input. + """ + + if isinstance(context, dict): + return { + "strategy": context.get("strategy", {}), + "transcript": context.get("transcript", {}), + "platform": context.get("platform", "tiktok"), + "video_path": context.get("video_path") + } + + return { + "strategy": getattr(context, "strategy", {}), + "transcript": getattr(context, "transcript", {}), + "platform": getattr(context, "platform", "tiktok"), + "video_path": getattr(context, "video_path", None) + } + + +# ------------------------------------------------- +# SAFE TEXT EXTRACTOR +# ------------------------------------------------- + +def extract_text(transcript): + """ + Extracts usable text from transcript structure safely. + """ + + if isinstance(transcript, dict): + segments = transcript.get("segments", []) + if segments: + return " ".join([s.get("text", "") for s in segments]) + + if isinstance(transcript, str): + return transcript + + return "" + + +# ------------------------------------------------- +# METADATA GENERATOR CORE +# ------------------------------------------------- + +def build_metadata(text, strategy, platform): + """ + Deterministic metadata generator (no external API required). + """ + + hook = "" + if isinstance(strategy, dict): + hook = strategy.get("hook", "") + + if not hook: + hook = text[:120] + "..." if text else "Discover powerful insights in this video." + + title = hook[:70].strip() + + description = ( + f"{hook}\n\n" + f"Watch till the end for key insights.\n" + f"Optimized for {platform}." + ) + + tags = [ + "content", + "viral", + "shorts", + platform, + "ai generated", + "social media" + ] + + hashtags = [ + "#ViralContent", + "#ContentCreator", + "#Shorts", + f"#{platform.capitalize()}", + "#AIContent" + ] + + return { + "title": title, + "description": description, + "tags": tags, + "hashtags": hashtags + } + + +# ------------------------------------------------- +# OPTIONAL LLM ENHANCEMENT (SAFE WRAPPER) +# ------------------------------------------------- + +async def enhance_with_llm(base_metadata, context): + """ + Optional enhancement layer. + Never breaks pipeline if API missing. + """ + + try: + import os + + if not os.getenv("GEMINI_API_KEY"): + return base_metadata + + # Lazy import to avoid startup crashes + import google.generativeai as genai + + genai.configure(api_key=os.environ["GEMINI_API_KEY"]) + + model = genai.GenerativeModel("gemini-1.5-flash") + + prompt = f""" +Improve this social media metadata for virality: + +TITLE: {base_metadata['title']} +DESCRIPTION: {base_metadata['description']} +TAGS: {base_metadata['tags']} +HASHTAGS: {base_metadata['hashtags']} + +Return STRICT JSON with: +title, description, tags, hashtags +""" + + response = await model.generate_content_async(prompt) + + import json + cleaned = response.text.strip().replace("```json", "").replace("```", "") + data = json.loads(cleaned) + + return data + + except Exception: + return base_metadata + + +# ------------------------------------------------- +# MAIN TASK ENTRYPOINT +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + strategy = ctx["strategy"] + transcript = ctx["transcript"] + platform = ctx["platform"] + + text = extract_text(transcript) + + # ------------------------------------------------- + # BASE METADATA + # ------------------------------------------------- + + base_metadata = build_metadata(text, strategy, platform) + + # ------------------------------------------------- + # OPTIONAL ENHANCEMENT + # ------------------------------------------------- + + final_metadata = await enhance_with_llm(base_metadata, ctx) + + # ------------------------------------------------- + # RESPONSE + # ------------------------------------------------- + + return { + "status": "success", + "task": "generate-metadata", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "platform": platform, + "metadata": final_metadata + } + + except Exception as e: + + return { + "status": "error", + "task": "generate-metadata", + "batch_id": batch_id, + "message": str(e), + "stage": "metadata_generation_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/generate_thumbnail.py b/services/whisper/publisher/tasks/generate_thumbnail.py new file mode 100644 index 0000000000000000000000000000000000000000..8d93022e4a532fbca1cb1c0ea032146acf4b19d5 --- /dev/null +++ b/services/whisper/publisher/tasks/generate_thumbnail.py @@ -0,0 +1,155 @@ +import os +import uuid +import asyncio +from datetime import datetime + +from PIL import Image, ImageDraw, ImageFont + + +# ------------------------------------------------- +# SAFE OUTPUT DIRECTORY +# ------------------------------------------------- + +OUTPUT_DIR = "jobs/thumbnails" +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + + if isinstance(context, dict): + return { + "title": context.get("title"), + "hook": context.get("hook"), + "strategy": context.get("strategy", {}), + "text": context.get("text", ""), + "video_path": context.get("video_path") + } + + return { + "title": getattr(context, "title", None), + "hook": getattr(context, "hook", None), + "strategy": getattr(context, "strategy", {}), + "text": getattr(context, "text", ""), + "video_path": getattr(context, "video_path", None) + } + + +# ------------------------------------------------- +# TEXT EXTRACTOR +# ------------------------------------------------- + +def extract_text(ctx): + if ctx.get("hook"): + return ctx["hook"] + + if isinstance(ctx.get("strategy"), dict): + hook = ctx["strategy"].get("hook") + if hook: + return hook + + return ctx.get("text") or "Create engaging content that stands out." + + +# ------------------------------------------------- +# SAFE FONT LOADER +# ------------------------------------------------- + +def load_font(size): + """ + Tries system fonts safely. + Falls back to default PIL font if unavailable. + """ + + try: + return ImageFont.truetype("arial.ttf", size) + except Exception: + return ImageFont.load_default() + + +# ------------------------------------------------- +# THUMBNAIL GENERATOR CORE +# ------------------------------------------------- + +def build_thumbnail(text, width=1280, height=720): + + img = Image.new("RGB", (width, height), color=(10, 10, 10)) + draw = ImageDraw.Draw(img) + + # Accent style bar + draw.rectangle([0, 0, 20, height], fill=(232, 255, 71)) + + # Title text + font_large = load_font(64) + font_small = load_font(36) + + wrapped_text = text[:120] + + draw.text( + (60, 200), + wrapped_text, + font=font_large, + fill=(232, 232, 232) + ) + + draw.text( + (60, 320), + "AI-Generated Content", + font=font_small, + fill=(136, 136, 136) + ) + + return img + + +# ------------------------------------------------- +# MAIN ENTRYPOINT +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + text = extract_text(ctx) + + # ------------------------------------------------- + # BUILD IMAGE (CPU SAFE) + # ------------------------------------------------- + + img = await asyncio.to_thread(build_thumbnail, text) + + file_name = f"{batch_id}_thumbnail.png" + output_path = os.path.join(OUTPUT_DIR, file_name) + + img.save(output_path) + + # ------------------------------------------------- + # RESPONSE + # ------------------------------------------------- + + return { + "status": "success", + "task": "generate-thumbnail", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "thumbnail_path": output_path + } + + except Exception as e: + + return { + "status": "error", + "task": "generate-thumbnail", + "batch_id": batch_id, + "message": str(e), + "stage": "thumbnail_generation_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/highlights.py b/services/whisper/publisher/tasks/highlights.py new file mode 100644 index 0000000000000000000000000000000000000000..88e861916aeeca68ed968b0e613ac3af9ab42cb4 --- /dev/null +++ b/services/whisper/publisher/tasks/highlights.py @@ -0,0 +1,192 @@ +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# UTILITIES +# ------------------------------------------------- + +def clamp(value, min_v=0, max_v=1): + return max(min_v, min(max_v, value)) + + +def safe_len(x): + try: + return len(x) + except Exception: + return 0 + + +# ------------------------------------------------- +# CORE HIGHLIGHT DETECTION ENGINE +# ------------------------------------------------- + +def detect_peak_density(words, window=12): + """ + Simple sliding window density heuristic. + Returns indices where speech intensity peaks. + """ + + if not words or len(words) < window: + return [] + + scores = [] + + for i in range(len(words) - window): + chunk = words[i:i + window] + + # heuristic: repetition + punctuation + trigger words + unique_ratio = len(set(chunk)) / window + + trigger_words = {"why", "how", "what", "secret", "hack", "never", "stop", "crazy", "insane"} + trigger_hits = sum(1 for w in chunk if str(w).lower() in trigger_words) + + score = (1 - unique_ratio) + (trigger_hits * 0.15) + + scores.append((i, score)) + + # sort by strongest signal + scores.sort(key=lambda x: x[1], reverse=True) + + # take top peaks (non-overlapping) + selected = [] + used = set() + + for idx, _ in scores: + if any(abs(idx - u) < window for u in used): + continue + selected.append(idx) + used.add(idx) + if len(selected) >= 8: # max clips + break + + return selected + + +def build_segments(words, indices, window=20): + """ + Convert peak indices into structured segments. + """ + + segments = [] + + for idx in indices: + start = max(0, idx - window // 2) + end = min(len(words), idx + window // 2) + + segment_words = words[start:end] + + if not segment_words: + continue + + segments.append({ + "id": str(uuid.uuid4()), + "start_index": start, + "end_index": end, + "words": segment_words, + "length": len(segment_words), + }) + + return segments + + +# ------------------------------------------------- +# FALLBACK MODE (NO SIGNAL DETECTED) +# ------------------------------------------------- + +def fallback_segments(words): + """ + Ensures highlights always exist even for weak input. + """ + + if not words: + return [] + + chunk_size = max(25, len(words) // 5) + + segments = [] + + for i in range(0, len(words), chunk_size): + chunk = words[i:i + chunk_size] + + segments.append({ + "id": str(uuid.uuid4()), + "start_index": i, + "end_index": i + len(chunk), + "words": chunk, + "length": len(chunk), + }) + + if len(segments) >= 5: + break + + return segments + + +# ------------------------------------------------- +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ------------------------------------------------- + +def run(context): + """ + Expected input: + { + "words": [...] + } + """ + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + words = [] + + if isinstance(context, dict): + words = context.get("words", []) + else: + words = getattr(context, "words", []) or [] + + # ------------------------------------------------- + # DETECT PEAKS + # ------------------------------------------------- + + peak_indices = detect_peak_density(words) + + if peak_indices: + segments = build_segments(words, peak_indices) + else: + segments = fallback_segments(words) + + # ------------------------------------------------- + # OUTPUT CONTRACT + # ------------------------------------------------- + + return { + "status": "success", + "task": "highlights", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core output for downstream clipper + "highlights": segments, + + # UI-friendly summary + "summary": { + "total_words": safe_len(words), + "segments_found": len(segments), + "peak_detection": bool(peak_indices), + } + } + + except Exception as e: + + return { + "status": "error", + "task": "highlights", + "batch_id": batch_id, + "message": str(e), + "stage": "highlight_detection_failed", + "highlights": [] + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish.py b/services/whisper/publisher/tasks/publish.py new file mode 100644 index 0000000000000000000000000000000000000000..c0113f609930bb9ae6562541ef57c0c59a185de8 --- /dev/null +++ b/services/whisper/publisher/tasks/publish.py @@ -0,0 +1,147 @@ +import asyncio +from datetime import datetime +import uuid + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + """ + Supports: + - dict input (registry executor) + - object input (legacy execution engine) + """ + + if isinstance(context, dict): + return { + "platform": context.get("platform", "tiktok"), + "content": context.get("content", {}), + "metadata": context.get("metadata", {}), + "video_path": context.get("video_path"), + "payload": context + } + + return { + "platform": getattr(context, "platform", "tiktok"), + "content": getattr(context, "content", {}), + "metadata": getattr(context, "metadata", {}), + "video_path": getattr(context, "video_path", None), + "payload": {} + } + + +# ------------------------------------------------- +# SAFE DISPATCH LAYER (registry-aware) +# ------------------------------------------------- + +async def safe_dispatch(platform, content, metadata, video_path=None): + """ + Uses platform_dispatcher if available. + Falls back to simulated response if missing. + """ + + try: + from publisher.platform_dispatcher import dispatch_publish + + return await dispatch_publish( + video_path=video_path, + payload={ + "platform": platform, + "content": content, + "metadata": metadata + } + ) + + except Exception as e: + return { + "status": "fallback_success", + "platform": platform, + "message": "dispatch fallback executed", + "error": str(e), + "simulated": True + } + + +# ------------------------------------------------- +# MAIN PUBLISH TASK +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + platform = ctx["platform"] + content = ctx["content"] + metadata = ctx["metadata"] + video_path = ctx["video_path"] + + # ------------------------------------------------- + # VALIDATION LAYER + # ------------------------------------------------- + + if not content: + return { + "status": "error", + "stage": "validation", + "message": "Missing content payload" + } + + # ------------------------------------------------- + # PLATFORM ROUTING + # ------------------------------------------------- + + supported_platforms = { + "tiktok", + "reels", + "youtube", + "shorts" + } + + if platform not in supported_platforms: + return { + "status": "error", + "stage": "validation", + "message": f"Unsupported platform: {platform}" + } + + # ------------------------------------------------- + # DISPATCH EXECUTION + # ------------------------------------------------- + + result = await safe_dispatch( + platform=platform, + content=content, + metadata=metadata, + video_path=video_path + ) + + # ------------------------------------------------- + # RESPONSE NORMALIZATION + # ------------------------------------------------- + + return { + "status": "success", + "task": "publish", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "platform": platform, + "result": result + } + + except Exception as e: + + return { + "status": "error", + "task": "publish", + "batch_id": batch_id, + "message": str(e), + "stage": "publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish_instagram.py b/services/whisper/publisher/tasks/publish_instagram.py new file mode 100644 index 0000000000000000000000000000000000000000..326e0ad65b702e0f35013423eaf96f22882e147e --- /dev/null +++ b/services/whisper/publisher/tasks/publish_instagram.py @@ -0,0 +1,153 @@ +import uuid +from datetime import datetime + + +# ===================================================== +# VALIDATION LAYER +# ===================================================== + +def validate_video(video_path: str | None): + if not video_path: + raise ValueError("video_path is required for Instagram publishing") + + return video_path + + +def normalize_payload(payload: dict | None): + payload = payload or {} + + return { + "caption": payload.get("caption", ""), + "hashtags": payload.get("hashtags", []), + + # Instagram constraints + "share_to_feed": bool(payload.get("share_to_feed", True)), + "collaborators": payload.get("collaborators", []), + "location_id": payload.get("location_id"), + + # IG-specific metadata controls + "disable_comments": bool(payload.get("disable_comments", False)), + "hide_likes": bool(payload.get("hide_likes", False)), + + # Reels compatibility (IG defaults) + "format": "reel", + "aspect_ratio": "9:16", + } + + +# ===================================================== +# INSTAGRAM GRAPH ADAPTER (ABSTRACT LAYER) +# ===================================================== + +class InstagramClient: + """ + Abstracted Instagram Graph API adapter. + + Production replacement: + - Meta Graph API / Instagram Content Publishing API + - OAuth token refresh layer + - container-based upload flow (creation → publish) + """ + + def __init__(self): + self.provider = "instagram" + + def create_container(self, video_path: str, metadata: dict): + """ + Step 1: create media container (simulation layer) + """ + + return { + "container_id": str(uuid.uuid4()), + "status": "created", + } + + def publish_container(self, container_id: str): + """ + Step 2: publish media container (simulation layer) + """ + + return { + "ig_media_id": str(uuid.uuid4()), + "permalink": f"https://instagram.com/p/{uuid.uuid4().hex[:11]}", + "status": "published" + } + + +# ===================================================== +# SAFE EXECUTION WRAPPER +# ===================================================== + +def safe_publish(client: InstagramClient, video_path: str, metadata: dict, retries: int = 2): + + last_error = None + + for _ in range(retries + 1): + + try: + container = client.create_container(video_path, metadata) + return client.publish_container(container["container_id"]) + + except Exception as e: + last_error = str(e) + + return { + "status": "failed", + "error": last_error + } + + +# ===================================================== +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ===================================================== + +def run(video_path: str | None = None, payload: dict | None = None, context: dict | None = None): + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + video_path = validate_video(video_path) + metadata = normalize_payload(payload) + + client = InstagramClient() + + result = safe_publish(client, video_path, metadata) + + # ================================================= + # OUTPUT CONTRACT (STRICT FOR UI + REGISTRY) + # ================================================= + + return { + "status": "success" if result.get("status") != "failed" else "error", + "task": "publish_instagram", + "platform": "instagram", + + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core outputs + "ig_media_id": result.get("ig_media_id"), + "permalink": result.get("permalink"), + + # UI-friendly metadata + "caption": metadata["caption"], + "aspect_ratio": metadata["aspect_ratio"], + + # full raw result + "result": result, + } + + except Exception as e: + + return { + "status": "error", + "task": "publish_instagram", + "platform": "instagram", + + "batch_id": batch_id, + "message": str(e), + "stage": "instagram_publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish_reels.py b/services/whisper/publisher/tasks/publish_reels.py new file mode 100644 index 0000000000000000000000000000000000000000..2c3aaad9605319f825efa24f0f4f301bccf9382f --- /dev/null +++ b/services/whisper/publisher/tasks/publish_reels.py @@ -0,0 +1,134 @@ +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# VALIDATION +# ------------------------------------------------- + +def validate_video(video_path: str | None): + if not video_path: + raise ValueError("video_path is required for Reels publishing") + + return video_path + + +def normalize_payload(payload: dict | None): + payload = payload or {} + + return { + "caption": payload.get("caption", ""), + "hashtags": payload.get("hashtags", []), + "share_to_feed": bool(payload.get("share_to_feed", True)), + "collaborators": payload.get("collaborators", []), + "location_id": payload.get("location_id"), + } + + +# ------------------------------------------------- +# REELS CLIENT (ADAPTER LAYER) +# ------------------------------------------------- + +class ReelsClient: + """ + Abstracted Meta/Instagram publishing adapter. + + Replace with: + - Meta Graph API / Instagram Content Publishing API + - OAuth token injection layer + """ + + def __init__(self): + self.provider = "instagram_reels" + + def upload_reel(self, video_path: str, metadata: dict): + """ + Simulated deterministic upload layer. + + Production replacement: + - POST /{ig-user-id}/media + - POST publish container + """ + + return { + "reel_id": str(uuid.uuid4()), + "media_id": str(uuid.uuid4()), + "permalink": f"https://instagram.com/reel/{uuid.uuid4().hex[:11]}", + "status": "published", + "caption": metadata["caption"], + } + + +# ------------------------------------------------- +# SAFE UPLOAD WRAPPER +# ------------------------------------------------- + +def safe_publish(client: ReelsClient, video_path: str, metadata: dict, retries: int = 2): + + last_error = None + + for _ in range(retries + 1): + + try: + return client.upload_reel(video_path, metadata) + + except Exception as e: + last_error = str(e) + + return { + "status": "failed", + "error": last_error + } + + +# ------------------------------------------------- +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ------------------------------------------------- + +def run(video_path: str | None = None, payload: dict | None = None, context: dict | None = None): + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + video_path = validate_video(video_path) + metadata = normalize_payload(payload) + + client = ReelsClient() + + result = safe_publish(client, video_path, metadata) + + # ------------------------------------------------- + # OUTPUT CONTRACT (STRICT FOR UI + REGISTRY) + # ------------------------------------------------- + + return { + "status": "success" if result.get("status") != "failed" else "error", + "task": "publish_reels", + "platform": "instagram_reels", + + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core outputs + "reel_id": result.get("reel_id"), + "media_id": result.get("media_id"), + "permalink": result.get("permalink"), + + # UI consumption + "result": result, + "caption": metadata["caption"], + } + + except Exception as e: + + return { + "status": "error", + "task": "publish_reels", + "platform": "instagram_reels", + "batch_id": batch_id, + "message": str(e), + "stage": "reels_publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish_shorts.py b/services/whisper/publisher/tasks/publish_shorts.py new file mode 100644 index 0000000000000000000000000000000000000000..6443a2e05ca4668f9c76f0c97379d0862f0c56a1 --- /dev/null +++ b/services/whisper/publisher/tasks/publish_shorts.py @@ -0,0 +1,137 @@ +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# VALIDATION LAYER +# ------------------------------------------------- + +def validate_video(video_path: str | None): + if not video_path: + raise ValueError("video_path is required for Shorts publishing") + + return video_path + + +def normalize_payload(payload: dict | None): + payload = payload or {} + + return { + "title": payload.get("title", "Shorts Video"), + "description": payload.get("description", ""), + "tags": payload.get("tags", []), + "privacy_status": payload.get("privacy_status", "public"), + "hashtags": payload.get("hashtags", []), + + # Shorts-specific optimization flags + "force_vertical": True, + "aspect_ratio": "9:16", + "max_duration_seconds": payload.get("max_duration_seconds", 60), + } + + +# ------------------------------------------------- +# SHORTS PLATFORM ADAPTER (ABSTRACT) +# ------------------------------------------------- + +class ShortsClient: + """ + Platform-agnostic Shorts uploader. + + Replace later with: + - YouTube Shorts (YouTube Data API) + - TikTok fallback adapter + - Instagram Reels cross-posting layer + """ + + def __init__(self): + self.provider = "shorts" + + def upload(self, video_path: str, metadata: dict): + + return { + "short_id": str(uuid.uuid4()), + "video_id": str(uuid.uuid4()), + "url": f"https://shorts.platform/watch/{uuid.uuid4().hex[:11]}", + "status": "published", + "aspect_ratio": metadata["aspect_ratio"], + "duration_limit": metadata["max_duration_seconds"] + } + + +# ------------------------------------------------- +# SAFE EXECUTION WRAPPER +# ------------------------------------------------- + +def safe_publish(client: ShortsClient, video_path: str, metadata: dict, retries: int = 2): + + last_error = None + + for _ in range(retries + 1): + + try: + return client.upload(video_path, metadata) + + except Exception as e: + last_error = str(e) + + return { + "status": "failed", + "error": last_error + } + + +# ------------------------------------------------- +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ------------------------------------------------- + +def run(video_path: str | None = None, payload: dict | None = None, context: dict | None = None): + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + video_path = validate_video(video_path) + metadata = normalize_payload(payload) + + client = ShortsClient() + + result = safe_publish(client, video_path, metadata) + + # ------------------------------------------------- + # OUTPUT CONTRACT (STRICT FOR UI + REGISTRY) + # ------------------------------------------------- + + return { + "status": "success" if result.get("status") != "failed" else "error", + "task": "publish_shorts", + "platform": "shorts", + + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core outputs + "short_id": result.get("short_id"), + "video_id": result.get("video_id"), + "url": result.get("url"), + + # UI-friendly fields + "aspect_ratio": metadata["aspect_ratio"], + "max_duration_seconds": metadata["max_duration_seconds"], + + # full result + "result": result, + } + + except Exception as e: + + return { + "status": "error", + "task": "publish_shorts", + "platform": "shorts", + "batch_id": batch_id, + "message": str(e), + "stage": "shorts_publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish_tiktok.py b/services/whisper/publisher/tasks/publish_tiktok.py new file mode 100644 index 0000000000000000000000000000000000000000..31b9cd35beda233af72d53799a42772d42dc5ec8 --- /dev/null +++ b/services/whisper/publisher/tasks/publish_tiktok.py @@ -0,0 +1,22 @@ +import shutil +from pathlib import Path + + +async def run(payload, ctx): + + video_path = payload["video_path"] + caption = payload["caption"] + + upload_dir = Path("published") + upload_dir.mkdir(exist_ok=True) + + destination = upload_dir / Path(video_path).name + + shutil.copy(video_path, destination) + + ctx.log(f"Published video with caption: {caption}") + + return { + "status": "published", + "path": str(destination) + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/publish_youtube.py b/services/whisper/publisher/tasks/publish_youtube.py new file mode 100644 index 0000000000000000000000000000000000000000..64033caa11d3d360a7ed366fd035d5fcd9932a94 --- /dev/null +++ b/services/whisper/publisher/tasks/publish_youtube.py @@ -0,0 +1,127 @@ +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# SAFE NORMALIZATION +# ------------------------------------------------- + +def normalize_payload(payload: dict | None): + payload = payload or {} + + return { + "title": payload.get("title", "Untitled Video"), + "description": payload.get("description", ""), + "tags": payload.get("tags", []), + "privacy_status": payload.get("privacy_status", "public"), + "category_id": payload.get("category_id", "22"), + } + + +def normalize_video(video_path: str | None): + if not video_path: + raise ValueError("video_path is required for YouTube publishing") + + return video_path + + +# ------------------------------------------------- +# MOCKABLE YOUTUBE CLIENT INTERFACE +# (replace with real API integration later) +# ------------------------------------------------- + +class YouTubeClient: + + def __init__(self): + # In production: inject OAuth client here + self.provider = "youtube" + + def upload(self, video_path, metadata): + """ + Deterministic stub for upload pipeline. + + Replace with: + - google-api-python-client + - or YouTube Data API v3 upload session + """ + + return { + "video_id": str(uuid.uuid4()), + "url": f"https://youtube.com/watch?v={uuid.uuid4().hex[:11]}", + "status": "uploaded", + "visibility": metadata["privacy_status"] + } + + +# ------------------------------------------------- +# RETRY WRAPPER (SAFE FOR NETWORK FAILURES) +# ------------------------------------------------- + +def safe_upload(client, video_path, metadata, retries=2): + + last_error = None + + for attempt in range(retries + 1): + + try: + return client.upload(video_path, metadata) + + except Exception as e: + last_error = str(e) + + return { + "status": "failed", + "error": last_error + } + + +# ------------------------------------------------- +# MAIN ENTRYPOINT (REGISTRY-COMPATIBLE) +# ------------------------------------------------- + +def run(video_path: str | None = None, payload: dict | None = None, context: dict | None = None): + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + video_path = normalize_video(video_path) + metadata = normalize_payload(payload) + + client = YouTubeClient() + + result = safe_upload(client, video_path, metadata) + + # ------------------------------------------------- + # OUTPUT CONTRACT (IMPORTANT FOR REGISTRY + UI) + # ------------------------------------------------- + + return { + "status": "success" if result.get("status") != "failed" else "error", + "task": "publish_youtube", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core output + "platform": "youtube", + "result": result, + + # UI-friendly fields + "video_url": result.get("url"), + "video_id": result.get("video_id"), + + # publishing metadata echo + "metadata": metadata + } + + except Exception as e: + + return { + "status": "error", + "task": "publish_youtube", + "batch_id": batch_id, + "message": str(e), + "stage": "youtube_publish_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/render.py b/services/whisper/publisher/tasks/render.py new file mode 100644 index 0000000000000000000000000000000000000000..c6ca8e358a29df8a3c15ffa570039d82a28acf7c --- /dev/null +++ b/services/whisper/publisher/tasks/render.py @@ -0,0 +1,181 @@ +import os +import uuid +import asyncio +import subprocess +from datetime import datetime + + +# ------------------------------------------------- +# SAFE OUTPUT DIRECTORY +# ------------------------------------------------- + +OUTPUT_DIR = "jobs/renders" +os.makedirs(OUTPUT_DIR, exist_ok=True) + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + + if isinstance(context, dict): + return { + "video_path": context.get("video_path"), + "srt": context.get("srt"), + "subtitles": context.get("subtitles") + } + + return { + "video_path": getattr(context, "video_path", None), + "srt": getattr(context, "srt", None), + "subtitles": getattr(context, "subtitles", None) + } + + +# ------------------------------------------------- +# SRT RESOLVER +# ------------------------------------------------- + +def resolve_srt(ctx): + """ + Accepts: + - raw SRT string + - file path + - None + """ + + srt = ctx.get("srt") + + if not srt: + return None + + if isinstance(srt, str) and os.path.exists(srt): + with open(srt, "r", encoding="utf-8") as f: + return f.read() + + return srt if isinstance(srt, str) else None + + +# ------------------------------------------------- +# SAFE FFMPEG RENDER ENGINE +# ------------------------------------------------- + +def run_ffmpeg(video_path, srt_path, output_path): + + cmd = [ + "ffmpeg", + "-y", + "-i", video_path, + ] + + # Subtitle overlay (only if available) + if srt_path and os.path.exists(srt_path): + cmd += [ + "-vf", + f"subtitles={srt_path}" + ] + + cmd += [ + "-c:v", "libx264", + "-preset", "veryfast", + "-crf", "23", + "-c:a", "aac", + "-b:a", "128k", + output_path + ] + + process = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + if process.returncode != 0: + raise RuntimeError(process.stderr) + + return output_path + + +# ------------------------------------------------- +# MAIN ENTRYPOINT +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + video_path = ctx.get("video_path") + srt_data = resolve_srt(ctx) + + if not video_path or not os.path.exists(video_path): + return { + "status": "error", + "task": "render", + "message": "Missing or invalid video_path", + "stage": "validation" + } + + # ------------------------------------------------- + # TEMP SRT FILE HANDLING + # ------------------------------------------------- + + srt_path = None + + if srt_data: + srt_path = os.path.join(OUTPUT_DIR, f"{batch_id}.srt") + + with open(srt_path, "w", encoding="utf-8") as f: + f.write(srt_data) + + output_path = os.path.join( + OUTPUT_DIR, + f"{batch_id}_render.mp4" + ) + + # ------------------------------------------------- + # FFMPEG EXECUTION (THREAD SAFE) + # ------------------------------------------------- + + await asyncio.to_thread( + run_ffmpeg, + video_path, + srt_path, + output_path + ) + + # ------------------------------------------------- + # CLEANUP OPTIONAL + # ------------------------------------------------- + + if srt_path and os.path.exists(srt_path): + os.remove(srt_path) + + # ------------------------------------------------- + # RESPONSE + # ------------------------------------------------- + + return { + "status": "success", + "task": "render", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + "output_path": output_path + } + + except Exception as e: + + return { + "status": "error", + "task": "render", + "batch_id": batch_id, + "message": str(e), + "stage": "render_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/strategy.py b/services/whisper/publisher/tasks/strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..72604d6531a63f5430b2ba710975ec696cebca1f --- /dev/null +++ b/services/whisper/publisher/tasks/strategy.py @@ -0,0 +1,150 @@ +import asyncio +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# SAFE DEFAULTS +# ------------------------------------------------- + +DEFAULT_PERSONA = { + "age_range": "18-34", + "interests": ["content creation", "social media growth"], + "behavior": "scroll-heavy short-form consumption" +} + + +# ------------------------------------------------- +# CONTEXT NORMALIZER +# ------------------------------------------------- + +def normalize_context(context): + + if isinstance(context, dict): + return { + "words": context.get("words") or context.get("transcript") or [], + "video_path": context.get("video_path"), + } + + return { + "words": getattr(context, "words", None) or getattr(context, "transcript", None) or [], + "video_path": getattr(context, "video_path", None) + } + + +# ------------------------------------------------- +# SIMPLE VIRAL HEURISTICS ENGINE +# (replaces fragile LLM dependency assumptions) +# ------------------------------------------------- + +def compute_hook(words): + if not words: + return "Create content that hooks attention in the first 3 seconds." + + # crude heuristic: pick first 12 words + text = " ".join(words if isinstance(words, list) else []) + return text.split(".")[0][:120] + + +def compute_viral_score(words): + if not words: + return 50 + + length = len(words) + + # heuristic scoring model (deterministic) + score = min(95, 40 + (length / 50)) + + return round(score, 2) + + +def detect_platform_fit(score): + if score >= 80: + return ["tiktok", "reels", "youtube-shorts"] + if score >= 60: + return ["tiktok", "reels"] + return ["reels"] + + +# ------------------------------------------------- +# RETENTION CURVE SIMULATOR +# ------------------------------------------------- + +def simulate_retention_curve(words): + + if not words: + return [1.0, 0.7, 0.5, 0.3] + + n = len(words) + + return [ + 1.0, + max(0.7, 1 - (n * 0.001)), + max(0.4, 1 - (n * 0.002)), + max(0.2, 1 - (n * 0.003)), + ] + + +# ------------------------------------------------- +# MAIN STRATEGY ENGINE +# ------------------------------------------------- + +async def run(context): + + ctx = normalize_context(context) + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + words = ctx.get("words") or [] + + # ------------------------------------------------- + # CORE STRATEGY OUTPUTS + # ------------------------------------------------- + + hook = compute_hook(words) + viral_score = compute_viral_score(words) + platforms = detect_platform_fit(viral_score) + retention_curve = simulate_retention_curve(words) + + # ------------------------------------------------- + # STRUCTURED RESPONSE (CRITICAL FOR REGISTRY) + # ------------------------------------------------- + + result = { + "status": "success", + "task": "strategy", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core outputs + "hook": hook, + "viral_score": viral_score, + "platforms": platforms, + + # structured sub-blocks (UI + publisher consumption) + "persona": DEFAULT_PERSONA, + + "retention_curve": retention_curve, + + "strategy": { + "recommended_length_sec": min(60, max(15, len(words) // 3)), + "hook_strength": "high" if viral_score > 75 else "medium", + "distribution_priority": platforms + } + } + + return result + + except Exception as e: + + return { + "status": "error", + "task": "strategy", + "batch_id": batch_id, + "message": str(e), + "stage": "strategy_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/tasks/subtitles.py b/services/whisper/publisher/tasks/subtitles.py new file mode 100644 index 0000000000000000000000000000000000000000..edfd45e7764053ec27b1102faf74eb9de0b6177d --- /dev/null +++ b/services/whisper/publisher/tasks/subtitles.py @@ -0,0 +1,131 @@ +""" +BASYX V11 — Subtitles Generator +Creates SRT subtitles from transcription +""" + +import os +import tempfile +import asyncio +from faster_whisper import WhisperModel +import httpx + + +# -------------------------------------------------- +# GLOBAL MODEL (shared) +# -------------------------------------------------- + +MODEL = WhisperModel( + model_size_or_path="base", + device="cpu", + compute_type="int8" +) + + +# -------------------------------------------------- +# HELPERS +# -------------------------------------------------- + +async def save_upload(file): + suffix = os.path.splitext(file.filename)[-1] + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) + tmp.write(await file.read()) + tmp.close() + return tmp.name + + +async def download_url(url: str): + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") + + async with httpx.AsyncClient(timeout=300) as client: + async with client.stream("GET", url) as r: + r.raise_for_status() + async for chunk in r.aiter_bytes(): + tmp.write(chunk) + + tmp.close() + return tmp.name + + +def format_time(seconds: float): + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int((seconds - int(seconds)) * 1000) + + return f"{h:02}:{m:02}:{s:02},{ms:03}" + + +def build_srt(segments): + lines = [] + + for i, seg in enumerate(segments, start=1): + lines.append(str(i)) + lines.append( + f"{format_time(seg.start)} --> {format_time(seg.end)}" + ) + lines.append(seg.text.strip()) + lines.append("") + + return "\n".join(lines) + + +# -------------------------------------------------- +# MAIN TASK +# -------------------------------------------------- + +async def run(context): + + media_path = None + + try: + # ---------------- INPUT ---------------- + + if context.input_file: + media_path = await save_upload(context.input_file) + + elif context.url_input: + media_path = await download_url(context.url_input) + + else: + return { + "status": "error", + "message": "No input provided" + } + + # ---------------- TRANSCRIBE ---------------- + + segments, info = await asyncio.to_thread( + MODEL.transcribe, + media_path, + beam_size=5 + ) + + # ---------------- BUILD SRT ---------------- + + srt_text = build_srt(list(segments)) + + # Save file + srt_path = tempfile.NamedTemporaryFile( + delete=False, + suffix=".srt" + ).name + + with open(srt_path, "w", encoding="utf-8") as f: + f.write(srt_text) + + return { + "status": "success", + "language": info.language, + "srt_path": srt_path, + "preview": srt_text[:1000] + } + + except Exception as e: + return { + "status": "error", + "message": str(e) + } + + finally: + if media_path and os.path.exists(media_path): + os.remove(media_path) \ No newline at end of file diff --git a/services/whisper/publisher/tasks/transcribe.py b/services/whisper/publisher/tasks/transcribe.py new file mode 100644 index 0000000000000000000000000000000000000000..c2a5d4c89fda2c95dd40e7033adaa62808123198 --- /dev/null +++ b/services/whisper/publisher/tasks/transcribe.py @@ -0,0 +1,130 @@ +""" +BASYX V11 — Transcribe Task +Production Version +""" + +from faster_whisper import WhisperModel +import tempfile +import os +import httpx +import asyncio + +# -------------------------------------------------- +# GLOBAL MODEL (LOAD ONCE) +# -------------------------------------------------- + +MODEL = WhisperModel( + model_size_or_path="base", + device="cpu", + compute_type="int8" +) + + +# -------------------------------------------------- +# HELPERS +# -------------------------------------------------- + +async def save_upload(file): + """Save uploaded file to temp path""" + suffix = os.path.splitext(file.filename)[-1] + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) + tmp.write(await file.read()) + tmp.close() + + return tmp.name + + +async def download_url(url: str): + """Download media from URL safely""" + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") + + async with httpx.AsyncClient(timeout=300) as client: + async with client.stream("GET", url) as r: + r.raise_for_status() + async for chunk in r.aiter_bytes(): + tmp.write(chunk) + + tmp.close() + return tmp.name + + +def build_segments(segments): + """Normalize whisper segments""" + results = [] + + for seg in segments: + results.append({ + "start": round(seg.start, 2), + "end": round(seg.end, 2), + "text": seg.text.strip() + }) + + return results + + +# -------------------------------------------------- +# MAIN TASK ENTRYPOINT +# -------------------------------------------------- + +async def run(context): + """ + Expected context: + context.input_file + context.url_input + """ + + media_path = None + + try: + # ---------------------------- + # INPUT RESOLUTION + # ---------------------------- + + if context.input_file: + media_path = await save_upload(context.input_file) + + elif context.url_input: + media_path = await download_url(context.url_input) + + else: + return { + "status": "error", + "message": "No file or URL provided" + } + + # ---------------------------- + # TRANSCRIPTION + # ---------------------------- + + segments, info = await asyncio.to_thread( + MODEL.transcribe, + media_path, + beam_size=5 + ) + + segment_list = build_segments(segments) + + full_text = " ".join(s["text"] for s in segment_list) + + # ---------------------------- + # OUTPUT + # ---------------------------- + + return { + "status": "success", + "language": info.language, + "duration": info.duration, + "segments": segment_list, + "text": full_text + } + + except Exception as e: + return { + "status": "error", + "message": str(e) + } + + finally: + if media_path and os.path.exists(media_path): + os.remove(media_path) \ No newline at end of file diff --git a/services/whisper/publisher/tasks/viral_score.py b/services/whisper/publisher/tasks/viral_score.py new file mode 100644 index 0000000000000000000000000000000000000000..aed6223e0a82a71d49ee827cc5afa42eaf0252b7 --- /dev/null +++ b/services/whisper/publisher/tasks/viral_score.py @@ -0,0 +1,211 @@ +import uuid +from datetime import datetime + + +# ------------------------------------------------- +# CORE VIRAL SCORING ENGINE (DETERMINISTIC) +# ------------------------------------------------- + +def clamp(value, min_v=0, max_v=100): + return max(min_v, min(max_v, value)) + + +def base_score(words): + """ + Base score derived from transcript length and density. + """ + if not words: + return 45.0 + + length = len(words) + + # optimal short-form zone: 80–250 words + if 80 <= length <= 250: + return 78.0 + if 250 < length <= 400: + return 70.0 + if length < 80: + return 60.0 + + return 55.0 + + +def hook_boost(words): + """ + Early attention heuristic: first 10–15 words impact score. + """ + if not words: + return 0 + + first_chunk = words[:15] if isinstance(words, list) else [] + + # crude signal: presence of question / trigger words + trigger_words = {"why", "how", "what", "you", "stop", "never", "secret", "hack"} + + hits = sum(1 for w in first_chunk if w.lower() in trigger_words) + + return hits * 4 # max ~20 boost + + +def retention_penalty(words): + """ + Penalize overly long or low-density content. + """ + if not words: + return 5 + + length = len(words) + + if length > 500: + return 15 + if length > 350: + return 10 + if length < 50: + return 8 + + return 3 + + +def engagement_density(words): + """ + Measures repetition + punchy structure signals. + """ + if not words: + return 0 + + unique = len(set(words)) + total = len(words) + + if total == 0: + return 0 + + ratio = unique / total + + # lower repetition = better clarity + return clamp(ratio * 25, 0, 25) + + +# ------------------------------------------------- +# SEGMENT SCORING (FOR CLIPS) +# ------------------------------------------------- + +def score_segment(segment): + """ + Segment can be: + - list of words + - dict with 'words' + """ + if isinstance(segment, dict): + words = segment.get("words", []) + else: + words = segment or [] + + score = ( + base_score(words) + + hook_boost(words) + + engagement_density(words) + - retention_penalty(words) + ) + + return { + "score": round(clamp(score), 2), + "length": len(words) if words else 0, + "signal": "strong" if score > 80 else "medium" if score > 60 else "weak" + } + + +# ------------------------------------------------- +# MAIN ENTRYPOINT (REGISTRY COMPATIBLE) +# ------------------------------------------------- + +def run(context): + """ + Expected input: + { + "words": [...], + "segments": [...] + } + """ + + batch_id = str(uuid.uuid4()) + started_at = datetime.utcnow().isoformat() + + try: + + words = None + segments = [] + + if isinstance(context, dict): + words = context.get("words", []) + segments = context.get("segments", []) or [] + else: + words = getattr(context, "words", []) or [] + segments = getattr(context, "segments", []) or [] + + # ------------------------------------------------- + # GLOBAL SCORE + # ------------------------------------------------- + + score = ( + base_score(words) + + hook_boost(words) + + engagement_density(words) + - retention_penalty(words) + ) + + score = round(clamp(score), 2) + + # ------------------------------------------------- + # SEGMENT SCORES + # ------------------------------------------------- + + segment_scores = [] + for seg in segments: + segment_scores.append(score_segment(seg)) + + # fallback: if no segments provided, treat full video as one + if not segment_scores: + segment_scores = [score_segment(words)] + + # ------------------------------------------------- + # OUTPUT CONTRACT + # ------------------------------------------------- + + return { + "status": "success", + "task": "viral_score", + "batch_id": batch_id, + "started_at": started_at, + "completed_at": datetime.utcnow().isoformat(), + + # core + "viral_score": score, + + # breakdown + "breakdown": { + "base_score": round(base_score(words), 2), + "hook_boost": round(hook_boost(words), 2), + "engagement_density": round(engagement_density(words), 2), + "retention_penalty": round(retention_penalty(words), 2), + }, + + # UI + publisher consumption + "segments": segment_scores, + + # classification + "classification": ( + "high-viral" if score >= 80 + else "medium-viral" if score >= 60 + else "low-viral" + ) + } + + except Exception as e: + + return { + "status": "error", + "task": "viral_score", + "batch_id": batch_id, + "message": str(e), + "stage": "viral_score_failed" + } \ No newline at end of file diff --git a/services/whisper/publisher/thumbnail.py b/services/whisper/publisher/thumbnail.py new file mode 100644 index 0000000000000000000000000000000000000000..af830a8a8384b6630b05f098a648078191c35989 --- /dev/null +++ b/services/whisper/publisher/thumbnail.py @@ -0,0 +1,19 @@ +from publisher.ai.gemini_client import get_model + + +def generate_thumbnail(video_path): + + model = get_model() + + prompt = """ + Suggest BEST thumbnail concept: + - facial emotion + - text overlay + - color style + """ + + result = model.generate_content(prompt) + + return { + "thumbnail_strategy": result.text + } \ No newline at end of file diff --git a/services/whisper/publisher/thumbnail_engine.py b/services/whisper/publisher/thumbnail_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..bdb5e95287b8987a06ce62ef4c59d1f117724f7f --- /dev/null +++ b/services/whisper/publisher/thumbnail_engine.py @@ -0,0 +1,25 @@ +# publisher/thumbnail_engine.py + +from PIL import Image, ImageDraw, ImageFont +from pathlib import Path + + +def generate_thumbnail(text, output): + + img = Image.new("RGB", (1080,1080), "black") + + draw = ImageDraw.Draw(img) + + font_path = Path(__file__).resolve().parents[1] / "fonts" / "TikTok-Bold.ttf" + try: + font = ImageFont.truetype(str(font_path), 80) + except Exception: + font = ImageFont.load_default() + + draw.text((80,400), text[:60], font=font, fill="white") + + Path(output).parent.mkdir(exist_ok=True) + + img.save(output) + + return output diff --git a/services/whisper/publisher/token_refresher.py b/services/whisper/publisher/token_refresher.py new file mode 100644 index 0000000000000000000000000000000000000000..de75337c23140c37f52f55c8f1e029d1f4dfed37 --- /dev/null +++ b/services/whisper/publisher/token_refresher.py @@ -0,0 +1,28 @@ +# publisher/token_refresher.py + +import httpx +from publisher.oauth.storage import save_tokens + + +async def refresh_google(user_id, token): + + if "refresh_token" not in token: + return token + + async with httpx.AsyncClient() as client: + r = await client.post( + "https://oauth2.googleapis.com/token", + data={ + "client_id": token["client_id"], + "client_secret": token["client_secret"], + "refresh_token": token["refresh_token"], + "grant_type": "refresh_token", + }, + ) + + new = r.json() + token.update(new) + + save_tokens(user_id, "google", token) + + return token \ No newline at end of file diff --git a/services/whisper/requirements.txt b/services/whisper/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f37afc004a7915db082675f6b7f9108dbe7280ec --- /dev/null +++ b/services/whisper/requirements.txt @@ -0,0 +1,71 @@ +# ========================= +# CORE API +# ========================= +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +python-multipart>=0.0.9 + +# ========================= +# UI +# ========================= +gradio>=4.0.0 + +# ========================= +# VIDEO PIPELINE +# ========================= +faster-whisper==1.0.3 +moviepy==1.0.3 +ffmpeg-python>=0.2.0 +imageio>=2.34.0 +imageio-ffmpeg>=0.4.9 +pysrt>=1.1.2 + +# ========================= +# NUMERICS / DATA +# ========================= +numpy>=1.26.0 +pydantic>=2.6.0 + +# ========================= +# YOUTUBE / SOCIAL INGESTION +# ========================= +yt-dlp>=2025.1.0 + +# ========================= +# HTTP / NETWORK +# ========================= +requests>=2.31.0 +httplib2>=0.22.0 + +# ========================= +# GOOGLE YOUTUBE + PLATFORM APIs +# ========================= +google-api-python-client>=2.118.0 +google-auth>=2.29.0 +google-auth-oauthlib>=1.2.0 +google-auth-httplib2>=0.2.0 + +# ========================= +# GEMINI AI +# ========================= +google-generativeai>=0.7.2 + +# ========================= +# AUTH SYSTEM (JWT + SECURITY) +# ========================= +python-jose[cryptography]>=3.3.0 + +# FIXED BCRYPT STACK (CRITICAL STABILITY FIX) +passlib[bcrypt]>=1.7.4 +email-validator + +# ========================= +# DATABASE (SQLALCHEMY + POSTGRES) +# ========================= +sqlalchemy>=2.0.25 +psycopg2-binary>=2.9.9 + +# ========================= +# OPTIONAL STABILITY LAYER +# ========================= +aiofiles>=23.2.1 \ No newline at end of file diff --git a/services/whisper/start.sh b/services/whisper/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..64994f81258302bc79d593c8667abed8606293c6 --- /dev/null +++ b/services/whisper/start.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "Starting Fast-Whisper API..." + +uvicorn main:app --host 0.0.0.0 --port 7860 \ No newline at end of file diff --git a/services/whisper/static/app.js b/services/whisper/static/app.js new file mode 100644 index 0000000000000000000000000000000000000000..155b598b75d56570d8a1ff98a1700b8592a93b18 --- /dev/null +++ b/services/whisper/static/app.js @@ -0,0 +1,230 @@ +/* ========================================================= + BASYX V11 UI ENGINE + Production-grade frontend runtime +========================================================= */ + +const API_BASE = ""; + +/* ========================================================= + STATE +========================================================= */ + +const state = { + currentTask: "transcribe", + loading: false, +}; + +/* ========================================================= + SAFE ELEMENT ACCESS +========================================================= */ + +function el(id) { + const node = document.getElementById(id); + if (!node) console.warn(`[UI] Missing element: ${id}`); + return node; +} + +/* ========================================================= + NAVIGATION SYSTEM +========================================================= */ + +function nav(task) { + state.currentTask = task; + + document.querySelectorAll(".section").forEach((s) => + s.classList.remove("active") + ); + + const page = el(task); + if (page) page.classList.add("active"); + + document.querySelectorAll(".nav-item").forEach((n) => + n.classList.remove("active") + ); + + if (event && event.target) { + event.target.classList.add("active"); + } + + location.hash = task; +} + +/* Restore on reload */ +window.addEventListener("load", () => { + const hash = location.hash.replace("#", ""); + if (hash && el(hash)) nav(hash); +}); + +/* ========================================================= + UI LOADING STATE +========================================================= */ + +function setLoading(task, isLoading) { + const btn = el(`btn-${task}`); + if (!btn) return; + + btn.disabled = isLoading; + btn.innerText = isLoading ? "Processing..." : "Execute"; +} + +/* ========================================================= + ERROR HANDLING +========================================================= */ + +async function safeFetch(url, options, retries = 2) { + try { + const res = await fetch(url, options); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`HTTP ${res.status}: ${text}`); + } + + return res; + } catch (err) { + if (retries > 0) { + await new Promise((r) => setTimeout(r, 800)); + return safeFetch(url, options, retries - 1); + } + throw err; + } +} + +/* ========================================================= + RESPONSE PARSER (JSON / BLOB / VIDEO) +========================================================= */ + +async function parseResponse(res, task) { + const contentType = res.headers.get("content-type") || ""; + + // VIDEO OR FILE OUTPUT + if ( + contentType.includes("video") || + contentType.includes("octet-stream") + ) { + const blob = await res.blob(); + return { + type: "blob", + url: URL.createObjectURL(blob), + }; + } + + // JSON OUTPUT + const data = await res.json(); + return { + type: "json", + data, + }; +} + +/* ========================================================= + INPUT COLLECTION +========================================================= */ + +function collectInput(task) { + const file = el(`file-${task}`)?.files?.[0]; + const url = el(`url-${task}`)?.value?.trim(); + + const fd = new FormData(); + + if (file) fd.append("file", file); + if (url) fd.append("url_input", url); + + return fd; +} + +/* ========================================================= + CORE EXECUTION +========================================================= */ + +async function run(task) { + if (state.loading) return; + + state.loading = true; + setLoading(task, true); + + try { + const fd = collectInput(task); + + const res = await safeFetch(`${API_BASE}/execute/${task}`, { + method: "POST", + body: fd, + }); + + const parsed = await parseResponse(res, task); + + renderOutput(task, parsed); + } catch (err) { + console.error(err); + renderError(task, err.message); + } finally { + state.loading = false; + setLoading(task, false); + } +} + +/* ========================================================= + OUTPUT RENDERING +========================================================= */ + +function renderOutput(task, result) { + const outputBox = el(`result-${task}`); + const videoBox = el(`video-${task}`); + + if (result.type === "blob") { + if (videoBox) { + videoBox.src = result.url; + videoBox.style.display = "block"; + } + return; + } + + if (outputBox) { + outputBox.textContent = JSON.stringify(result.data, null, 2); + } +} + +/* ========================================================= + ERROR UI +========================================================= */ + +function renderError(task, message) { + const outputBox = el(`result-${task}`); + + if (outputBox) { + outputBox.textContent = `ERROR:\n${message}`; + outputBox.style.color = "#ff4d4d"; + } +} + +/* ========================================================= + HEALTH CHECK (optional future UI use) +========================================================= */ + +async function checkHealth() { + try { + const res = await fetch("/health"); + return await res.json(); + } catch { + return { status: "offline" }; + } +} + +/* ========================================================= + MOBILE SIDEBAR TOGGLE +========================================================= */ + +function toggleSidebar() { + const sidebar = document.querySelector(".sidebar"); + if (!sidebar) return; + + sidebar.classList.toggle("open"); +} + +/* Close sidebar on nav click (mobile UX) */ +document.addEventListener("click", (e) => { + if (e.target.classList.contains("nav-item")) { + const sidebar = document.querySelector(".sidebar"); + if (sidebar) sidebar.classList.remove("open"); + } +}); \ No newline at end of file diff --git a/services/whisper/static/index.html b/services/whisper/static/index.html new file mode 100644 index 0000000000000000000000000000000000000000..40615d735074bdbb0f1c28e9cae703e4d2e3abdf --- /dev/null +++ b/services/whisper/static/index.html @@ -0,0 +1,278 @@ + + + + + + +Basyx V11 Content OS + + + + + + + + + + +
+ + +
+

+ AUTONOMOUS CONTENT OPERATOR +

+
+ + +
+
+

TRANSCRIBE

+ + + +
+ 

+

+
+

+  

+    
SUBTITLES

+    
+    
+    
+  

+  

+

+
+

+  

+    
RENDER

+    
+    
+    
+  

+  
+

+
+

+  

+    
HIGHLIGHTS

+    
+    
+  

+  

+

+
+

+  

+    
VIRAL SCORE

+    
+    
+  

+  

+

+
+

+  

+    
STRATEGY

+    
+    
+  

+  

+

+
+
+
+
+
\ No newline at end of file
diff --git a/services/whisper/static/style.css b/services/whisper/static/style.css
new file mode 100644
index 0000000000000000000000000000000000000000..55c26a4edbbaf5f1d4f9790fd8eac05f2934c102
--- /dev/null
+++ b/services/whisper/static/style.css
@@ -0,0 +1,338 @@
+/* =========================================================
+   BASYX V11 UI SYSTEM
+   Industrial Dark OS Theme
+========================================================= */
+
+:root {
+  --void: #080808;
+  --surface: #0f0f0f;
+  --panel: #161616;
+  --panel-2: #121212;
+
+  --border: #232323;
+  --border-2: #2c2c2c;
+
+  --accent: #e8ff47;
+  --accent-dim: #b8cc30;
+
+  --text: #e8e8e8;
+  --text-dim: #888888;
+
+  --muted: #555555;
+  --danger: #ef4444;
+  --warn: #f59e0b;
+  --ok: #22c55e;
+
+  --radius: 6px;
+
+  --font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono";
+  --font-display: Bebas Neue, sans-serif;
+  --font-body: system-ui, -apple-system, Segoe UI, Roboto;
+}
+
+/* =========================================================
+   BASE RESET
+========================================================= */
+
+* {
+  box-sizing: border-box;
+}
+
+html, body {
+  margin: 0;
+  padding: 0;
+  background: var(--void);
+  color: var(--text);
+  font-family: var(--font-body);
+  height: 100%;
+}
+
+h1, h2, h3 {
+  margin: 0;
+}
+
+/* =========================================================
+   SIDEBAR
+========================================================= */
+
+.sidebar {
+  position: fixed;
+  left: 0;
+  top: 0;
+  width: 220px;
+  height: 100vh;
+  background: #0b0b0b;
+  border-right: 1px solid var(--border);
+  padding: 16px;
+  overflow-y: auto;
+  transition: transform 0.25s ease;
+}
+
+.brand {
+  font-family: var(--font-display);
+  font-size: 28px;
+  color: var(--accent);
+  margin-bottom: 18px;
+  letter-spacing: 1px;
+}
+
+.nav-group-title {
+  font-size: 11px;
+  color: var(--text-dim);
+  margin: 16px 0 6px;
+  letter-spacing: 1px;
+}
+
+.nav-item {
+  padding: 10px 12px;
+  cursor: pointer;
+  color: var(--text-dim);
+  border-left: 2px solid transparent;
+  transition: all 0.2s ease;
+  border-radius: 4px;
+}
+
+.nav-item:hover {
+  color: var(--text);
+  background: rgba(255, 255, 255, 0.03);
+}
+
+.nav-item.active {
+  border-left: 2px solid var(--accent);
+  color: var(--accent);
+  background: rgba(232, 255, 71, 0.06);
+}
+
+/* =========================================================
+   MAIN LAYOUT
+========================================================= */
+
+.main {
+  margin-left: 220px;
+  padding: 24px;
+}
+
+/* =========================================================
+   SECTIONS
+========================================================= */
+
+.section {
+  display: none;
+  animation: fadeIn 0.25s ease;
+}
+
+.section.active {
+  display: block;
+}
+
+@keyframes fadeIn {
+  from { opacity: 0; transform: translateY(6px); }
+  to { opacity: 1; transform: translateY(0); }
+}
+
+/* =========================================================
+   PANELS
+========================================================= */
+
+.panel {
+  background: var(--panel);
+  border: 1px solid var(--border);
+  padding: 18px;
+  border-radius: var(--radius);
+  margin-bottom: 16px;
+}
+
+.panel h2 {
+  font-family: var(--font-display);
+  letter-spacing: 1px;
+  color: var(--accent);
+  margin-bottom: 12px;
+}
+
+/* =========================================================
+   INPUT SYSTEM
+========================================================= */
+
+input, textarea, select {
+  width: 100%;
+  padding: 10px 12px;
+  margin-top: 8px;
+  background: #111;
+  border: 1px solid var(--border);
+  border-radius: 4px;
+  color: var(--text);
+  outline: none;
+}
+
+input:focus {
+  border-color: var(--accent);
+}
+
+/* =========================================================
+   BUTTONS
+========================================================= */
+
+button {
+  padding: 12px 14px;
+  border: none;
+  cursor: pointer;
+  background: var(--accent);
+  color: #000;
+  font-weight: 700;
+  border-radius: 4px;
+  transition: all 0.2s ease;
+  margin-top: 10px;
+}
+
+button:hover {
+  filter: brightness(1.05);
+}
+
+button:disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+}
+
+/* =========================================================
+   PRE / OUTPUT
+========================================================= */
+
+pre {
+  background: #0b0b0b;
+  border: 1px solid var(--border);
+  padding: 12px;
+  border-radius: var(--radius);
+  overflow: auto;
+  color: var(--text);
+  font-family: var(--font-mono);
+}
+
+/* =========================================================
+   VIDEO OUTPUT
+========================================================= */
+
+video {
+  width: 100%;
+  border-radius: var(--radius);
+  border: 1px solid var(--border);
+  background: #000;
+}
+
+/* =========================================================
+   CARDS (future UI expansion)
+========================================================= */
+
+.card-grid {
+  display: grid;
+  grid-template-columns: repeat(3, 1fr);
+  gap: 12px;
+}
+
+.card {
+  background: var(--panel);
+  border: 1px solid var(--border);
+  padding: 14px;
+  border-radius: var(--radius);
+  cursor: pointer;
+  transition: 0.2s;
+}
+
+.card:hover {
+  border-color: var(--accent);
+  transform: translateY(-2px);
+}
+
+/* =========================================================
+   STATUS BADGES
+========================================================= */
+
+.badge {
+  display: inline-block;
+  padding: 4px 8px;
+  font-size: 12px;
+  border-radius: 4px;
+  border: 1px solid var(--border);
+}
+
+.badge.ok {
+  color: var(--ok);
+  border-color: var(--ok);
+}
+
+.badge.warn {
+  color: var(--warn);
+  border-color: var(--warn);
+}
+
+.badge.error {
+  color: var(--danger);
+  border-color: var(--danger);
+}
+
+/* =========================================================
+   STAT CARDS
+========================================================= */
+
+.stat-grid {
+  display: grid;
+  grid-template-columns: repeat(3, 1fr);
+  gap: 12px;
+  margin: 12px 0;
+}
+
+.stat-card {
+  background: var(--panel);
+  border: 1px solid var(--border);
+  padding: 14px;
+  border-radius: var(--radius);
+}
+
+.stat-label {
+  font-size: 12px;
+  color: var(--text-dim);
+}
+
+.stat-value {
+  font-size: 34px;
+  font-family: var(--font-display);
+  color: var(--accent);
+}
+
+/* =========================================================
+   LOADING STATE
+========================================================= */
+
+.loading {
+  opacity: 0.6;
+  pointer-events: none;
+}
+
+/* =========================================================
+   MOBILE
+========================================================= */
+
+@media (max-width: 768px) {
+
+  .sidebar {
+    transform: translateX(-100%);
+    position: fixed;
+    z-index: 50;
+  }
+
+  .sidebar.open {
+    transform: translateX(0);
+  }
+
+  .main {
+    margin-left: 0;
+    padding: 16px;
+  }
+
+  .card-grid {
+    grid-template-columns: 1fr;
+  }
+
+  .stat-grid {
+    grid-template-columns: 1fr;
+  }
+}
\ No newline at end of file
diff --git a/services/whisper/utils/__init__.py b/services/whisper/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..40c75472b2df38dce59c6250b63167298c0e06a2
--- /dev/null
+++ b/services/whisper/utils/__init__.py
@@ -0,0 +1 @@
+# Initialize package
diff --git a/services/whisper/utils/ab_generator.py b/services/whisper/utils/ab_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe35d963dcc587d1e64fba02b674ccf2db3b8416
--- /dev/null
+++ b/services/whisper/utils/ab_generator.py
@@ -0,0 +1,11 @@
+from .strategist import rewrite_hook
+
+def generate_variants(segment):
+
+    text = " ".join([w["text"] for w in segment])
+
+    return [
+        rewrite_hook(text),
+        f"🔥 {text}",
+        f"Did you know? {text}",
+    ]
\ No newline at end of file
diff --git a/services/whisper/utils/auto_editor.py b/services/whisper/utils/auto_editor.py
new file mode 100644
index 0000000000000000000000000000000000000000..88fed30e92620253ff2c03663103d7c0aff7bbbb
--- /dev/null
+++ b/services/whisper/utils/auto_editor.py
@@ -0,0 +1,112 @@
+"""
+auto_editor.py
+---------------------------------------
+Automatic video pacing editor
+
+Features:
+- Removes silence automatically
+- Speeds up slow segments
+- Keeps speech natural
+- CPU optimized (HuggingFace FREE tier safe)
+
+Input:
+    input.mp4
+
+Output:
+    edited.mp4
+"""
+
+import subprocess
+import os
+import sys
+from pathlib import Path
+
+INPUT_VIDEO = "input.mp4"
+OUTPUT_VIDEO = "edited.mp4"
+
+
+def run_command(cmd):
+    """Run shell command safely"""
+    try:
+        subprocess.run(
+            cmd,
+            shell=True,
+            check=True
+        )
+    except subprocess.CalledProcessError as e:
+        print("Command failed:", e)
+        sys.exit(1)
+
+
+def check_input():
+    if not os.path.exists(INPUT_VIDEO):
+        print(f"❌ Missing file: {INPUT_VIDEO}")
+        sys.exit(1)
+
+
+def install_auto_editor():
+    """
+    Ensures auto-editor exists.
+    Required because HF containers reset.
+    """
+    print("Installing auto-editor...")
+    run_command("pip install --no-cache-dir auto-editor")
+
+
+def auto_edit():
+    """
+    Main editing step.
+    Removes silence + improves pacing.
+    """
+
+    cmd = f"""
+    auto-editor "{INPUT_VIDEO}"
+        --margin 0.2s
+        --silent-speed 99999
+        --video-speed 1
+        --audio-normalize peak
+        --export mp4
+        --output "{OUTPUT_VIDEO}"
+    """
+
+    print("Running auto-editor...")
+    run_command(cmd)
+
+
+def optimize_output():
+    """
+    Re-encode for social media compatibility.
+    """
+
+    temp = "optimized.mp4"
+
+    cmd = f"""
+    ffmpeg -y -i "{OUTPUT_VIDEO}"
+        -vf scale=1080:1920:force_original_aspect_ratio=decrease,pad=1080:1920:(ow-iw)/2:(oh-ih)/2
+        -c:v libx264
+        -preset veryfast
+        -crf 23
+        -c:a aac
+        -b:a 128k
+        "{temp}"
+    """
+
+    print("Optimizing output...")
+    run_command(cmd)
+
+    os.replace(temp, OUTPUT_VIDEO)
+
+
+def main():
+    print("===== AUTO EDITOR START =====")
+
+    check_input()
+    install_auto_editor()
+    auto_edit()
+    optimize_output()
+
+    print("✅ Editing complete:", OUTPUT_VIDEO)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/services/whisper/utils/autonomous_engine.py b/services/whisper/utils/autonomous_engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..00dca39c9898a87b1f0506556474dcc71d8d5c95
--- /dev/null
+++ b/services/whisper/utils/autonomous_engine.py
@@ -0,0 +1,97 @@
+"""
+V8 Autonomous Viral Engine
+--------------------------
+Fully automated pipeline for viral video generation.
+"""
+
+import os
+from utils.transcription import transcribe_video
+from utils.highlights import detect_highlights
+from utils.clipper import create_clips
+from utils.srt import generate_srt
+from utils.render import render_subtitles
+from utils.variations import generate_hooks
+from utils.pacing import pacing_engine
+from utils.viral_scorer import score_clip
+
+
+# =====================================================
+# CORE AUTONOMOUS PIPELINE
+# =====================================================
+
+def run_autonomous_engine(video_path):
+    """
+    One-call system → full viral pipeline
+    """
+
+    print("[V8 ENGINE] Starting autonomous pipeline...")
+
+    # 1. TRANSCRIBE
+    words = transcribe_video(video_path)
+
+    # 2. DETECT HIGHLIGHTS
+    highlights = detect_highlights(words)
+
+    if not highlights:
+        return {
+            "status": "failed",
+            "reason": "No highlights detected"
+        }
+
+    # 3. AUTO CLIP GENERATION
+    clips = create_clips(video_path, highlights)
+
+    if not clips:
+        return {
+            "status": "failed",
+            "reason": "Clip generation failed"
+        }
+
+    outputs = []
+
+    # 4. PROCESS EACH CLIP AUTONOMOUSLY
+    for i, clip in enumerate(clips):
+
+        try:
+            clip_words = transcribe_video(clip)
+
+            # 5. CAPTIONS
+            srt = generate_srt(clip_words)
+
+            # 6. PACING OPTIMIZATION (NEW V8)
+            paced_clip = pacing_engine(clip, clip_words)
+
+            # 7. HOOK VARIATIONS
+            hooks = generate_hooks(clip_words)
+
+            # 8. RENDER OUTPUT
+            output_path = clip.replace(".mp4", f"_v8_{i}.mp4")
+
+            final_video = render_subtitles(
+                paced_clip,
+                srt,
+                output_path
+            )
+
+            # 9. VIRAL SCORING
+            score = score_clip(clip_words)
+
+            outputs.append({
+                "clip": final_video,
+                "score": score,
+                "hooks": hooks[:3],
+                "index": i
+            })
+
+        except Exception as e:
+            print(f"[V8 ENGINE] Clip {i} failed:", str(e))
+            continue
+
+    # 10. SORT BY VIRAL SCORE
+    outputs = sorted(outputs, key=lambda x: x["score"], reverse=True)
+
+    return {
+        "status": "completed",
+        "best_clip": outputs[0] if outputs else None,
+        "all_variants": outputs
+    }
\ No newline at end of file
diff --git a/services/whisper/utils/batch_queue.py b/services/whisper/utils/batch_queue.py
new file mode 100644
index 0000000000000000000000000000000000000000..4155e344dfc28f95be640273a7ed1b301cdec644
--- /dev/null
+++ b/services/whisper/utils/batch_queue.py
@@ -0,0 +1,90 @@
+import threading
+from queue import Queue
+import uuid
+import time
+
+from .job_queue import jobs, update, notify_webhook
+from .transcription import transcribe_video
+from .render import render_video
+from .highlights import detect_highlights
+
+
+batch_queue = Queue()
+
+
+def create_batch_job(video_path, webhook=None):
+
+    job_id = str(uuid.uuid4())
+
+    jobs[job_id] = {
+        "id": job_id,
+        "status": "queued",
+        "progress": 0,
+        "clips": [],
+        "video": video_path,
+        "webhook": webhook,
+    }
+
+    batch_queue.put(job_id)
+
+    return job_id
+
+
+def worker():
+
+    while True:
+
+        job_id = batch_queue.get()
+        job = jobs[job_id]
+
+        try:
+
+            update(job_id, status="processing", progress=5)
+
+            # 1. TRANSCRIBE
+            words = transcribe_video(job["video"])
+
+            update(job_id, progress=30)
+
+            # 2. DETECT HIGHLIGHTS
+            highlights = detect_highlights(words)
+
+            update(job_id, progress=50)
+
+            outputs = []
+
+            # 3. RENDER MULTIPLE CLIPS
+            for i, segment in enumerate(highlights):
+
+                start = segment[0]["start"]
+                end = segment[-1]["end"]
+
+                clip_path = render_video(job["video"], words)
+
+                outputs.append(clip_path)
+
+                update(job_id, progress=50 + int((i+1)/len(highlights)*40))
+
+            # 4. FINALIZE
+            update(job_id,
+                   status="completed",
+                   progress=100,
+                   clips=outputs)
+
+            notify_webhook(job_id)
+
+        except Exception as e:
+
+            update(job_id,
+                   status="failed",
+                   error=str(e))
+
+            notify_webhook(job_id)
+
+        batch_queue.task_done()
+
+
+def start_batch_worker():
+
+    t = threading.Thread(target=worker, daemon=True)
+    t.start()
\ No newline at end of file
diff --git a/services/whisper/utils/broll.py b/services/whisper/utils/broll.py
new file mode 100644
index 0000000000000000000000000000000000000000..377300581b6a5e525f9a67bcfa3e37bc778819ab
--- /dev/null
+++ b/services/whisper/utils/broll.py
@@ -0,0 +1,221 @@
+"""
+broll.py
+---------------------------------------
+AI B-Roll Injection System (V8)
+
+Purpose:
+- Detect topics in transcript
+- Map topics → generic stock B-roll assets
+- Overlay or replace segments
+- Improve retention & visual variety
+
+Works in CPU-only environments.
+No external API dependency required.
+"""
+
+import os
+import random
+import subprocess
+
+
+# =====================================================
+# STOCK B-ROLL LIBRARY (LOCAL FALLBACK)
+# =====================================================
+
+DEFAULT_BROLL = {
+    "money": "assets/broll/money.mp4",
+    "success": "assets/broll/success.mp4",
+    "business": "assets/broll/business.mp4",
+    "phone": "assets/broll/phone.mp4",
+    "tech": "assets/broll/tech.mp4",
+    "people": "assets/broll/people.mp4",
+    "talking": "assets/broll/talking.mp4",
+    "default": "assets/broll/default.mp4",
+}
+
+
+# =====================================================
+# TOPIC DETECTION
+# =====================================================
+
+def detect_topic(text):
+    """
+    Simple keyword-based topic classifier.
+    Lightweight (no ML dependency).
+    """
+
+    text = text.lower()
+
+    if any(w in text for w in ["money", "rich", "income", "profit"]):
+        return "money"
+
+    if any(w in text for w in ["business", "startup", "company"]):
+        return "business"
+
+    if any(w in text for w in ["phone", "mobile", "iphone", "android"]):
+        return "phone"
+
+    if any(w in text for w in ["tech", "ai", "software", "computer"]):
+        return "tech"
+
+    if any(w in text for w in ["success", "win", "achieve"]):
+        return "success"
+
+    if any(w in text for w in ["people", "person", "man", "woman"]):
+        return "people"
+
+    if any(w in text for w in ["talk", "speak", "say"]):
+        return "talking"
+
+    return "default"
+
+
+# =====================================================
+# SEGMENT ANALYZER
+# =====================================================
+
+def extract_segments(words, segment_length=8):
+    """
+    Converts transcript words into grouped segments.
+    """
+
+    segments = []
+    buffer = []
+
+    for w in words:
+        buffer.append(w)
+
+        if len(buffer) >= segment_length:
+            segments.append(buffer)
+            buffer = []
+
+    if buffer:
+        segments.append(buffer)
+
+    return segments
+
+
+# =====================================================
+# B-ROLL MATCHING ENGINE
+# =====================================================
+
+def match_broll(segment):
+    """
+    Map transcript segment → B-roll video
+    """
+
+    text = " ".join([w["word"] for w in segment])
+    topic = detect_topic(text)
+
+    return DEFAULT_BROLL.get(topic, DEFAULT_BROLL["default"])
+
+
+# =====================================================
+# B-ROLL INSERTION (FFMPEG OVERLAY STRATEGY)
+# =====================================================
+
+def overlay_broll(base_video, broll_video, output_path, start_time, duration):
+    """
+    Overlays B-roll using ffmpeg.
+    Lightweight crossfade approach.
+    """
+
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", base_video,
+        "-i", broll_video,
+        "-filter_complex",
+        f"[1:v]scale=1080:1920,format=rgba[ov];"
+        f"[0:v][ov]overlay=enable='between(t,{start_time},{start_time+duration})'",
+        "-c:v", "libx264",
+        "-preset", "ultrafast",
+        "-c:a", "copy",
+        output_path
+    ]
+
+    subprocess.run(cmd, check=True)
+
+
+# =====================================================
+# MAIN PIPELINE
+# =====================================================
+
+def insert_broll(video_path, words=None):
+    """
+    Full B-roll injection pipeline
+    """
+
+    if not words:
+        # fallback: return original video
+        return video_path
+
+    segments = extract_segments(words)
+
+    current_video = video_path
+    outputs = []
+
+    for i, segment in enumerate(segments):
+
+        broll = match_broll(segment)
+
+        output_file = f"broll_output_{i}.mp4"
+
+        start_time = segment[0]["start"]
+        duration = segment[-1]["end"] - start_time
+
+        try:
+            overlay_broll(
+                current_video,
+                broll,
+                output_file,
+                start_time,
+                duration
+            )
+
+            current_video = output_file
+            outputs.append(output_file)
+
+        except Exception as e:
+            print(f"[BROLL ERROR] Segment {i}: {e}")
+            continue
+
+    return outputs[-1] if outputs else video_path
+
+
+# =====================================================
+# ADVANCED VERSION (V8 EXTENSION)
+# =====================================================
+
+def smart_broll_engine(words, hook_boost=True):
+    """
+    Enhanced version:
+    - prioritizes hook segments
+    - increases emotional pacing
+    """
+
+    segments = extract_segments(words)
+
+    prioritized = []
+
+    for seg in segments:
+
+        text = " ".join([w["word"] for w in seg]).lower()
+
+        score = 0
+
+        if any(k in text for k in ["you", "this", "stop", "now"]):
+            score += 2
+
+        if hook_boost and len(seg) < 5:
+            score += 1
+
+        prioritized.append((score, seg))
+
+    prioritized.sort(reverse=True, key=lambda x: x[0])
+
+    final_video = None
+
+    for _, seg in prioritized:
+        final_video = insert_broll(final_video or "input.mp4", seg)
+
+    return final_video
\ No newline at end of file
diff --git a/services/whisper/utils/caption_director.py b/services/whisper/utils/caption_director.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc7aab54eaa246c93c8dd1afa8c30c1d39f7b86d
--- /dev/null
+++ b/services/whisper/utils/caption_director.py
@@ -0,0 +1,231 @@
+"""
+caption_director.py
+---------------------------------------
+AI Caption Intelligence System
+
+Responsibilities:
+- Convert transcript → styled caption segments
+- Decide emphasis words
+- Break text into readable chunks
+- Optimize for TikTok/Reels retention
+- Support hook-style captions
+
+INPUT:
+    words = [
+        {"word": "hello", "start": 0.2, "end": 0.5},
+        ...
+    ]
+
+OUTPUT:
+    caption blocks:
+    [
+        {
+            "text": "THIS IS CRAZY",
+            "start": 0.2,
+            "end": 2.1,
+            "style": "hook"
+        }
+    ]
+"""
+
+import re
+
+
+# -----------------------------
+# CONFIG
+# -----------------------------
+
+MAX_WORDS_PER_CAPTION = 6
+HOOK_KEYWORDS = [
+    "listen", "wait", "you", "this", "crazy",
+    "insane", "important", "stop", "secret"
+]
+
+
+# -----------------------------
+# UTIL: CLEAN TEXT
+# -----------------------------
+
+def clean_word(word):
+    return re.sub(r"[^a-zA-Z0-9']", "", word).lower()
+
+
+# -----------------------------
+# DETECT EMPHASIS
+# -----------------------------
+
+def is_emphasis(word):
+    w = clean_word(word)
+    return w in HOOK_KEYWORDS or len(word) > 8
+
+
+# -----------------------------
+# GROUP WORDS INTO CAPTIONS
+# -----------------------------
+
+def group_words(words):
+    captions = []
+    buffer = []
+
+    for w in words:
+        buffer.append(w)
+
+        if len(buffer) >= MAX_WORDS_PER_CAPTION:
+            captions.append(buffer)
+            buffer = []
+
+    if buffer:
+        captions.append(buffer)
+
+    return captions
+
+
+# -----------------------------
+# BUILD CAPTION BLOCK
+# -----------------------------
+
+def build_caption_block(group):
+    text = []
+    start = group[0]["start"]
+    end = group[-1]["end"]
+
+    emphasis_count = 0
+
+    for w in group:
+        word = w["word"]
+
+        if is_emphasis(word):
+            text.append(word.upper())
+            emphasis_count += 1
+        else:
+            text.append(word)
+
+    caption_text = " ".join(text)
+
+    style = "hook" if emphasis_count > 0 else "normal"
+
+    return {
+        "text": caption_text,
+        "start": start,
+        "end": end,
+        "style": style
+    }
+
+
+# -----------------------------
+# MAIN DIRECTOR
+# -----------------------------
+
+def caption_director(words):
+    """
+    Main caption intelligence engine
+    """
+
+    if not words:
+        return []
+
+    grouped = group_words(words)
+
+    captions = []
+
+    for group in grouped:
+        captions.append(build_caption_block(group))
+
+    return captions
+
+
+# -----------------------------
+# HOOK CAPTION GENERATOR
+# -----------------------------
+
+def generate_hook_caption(words):
+    """
+    Extracts first high-impact caption
+    """
+
+    for w in words[:20]:
+        if is_emphasis(w["word"]):
+            return {
+                "text": w["word"].upper(),
+                "start": w["start"],
+                "end": w["end"],
+                "style": "hook"
+            }
+
+    return None
+
+
+# -----------------------------
+# AUTO CAPTION PIPELINE
+# -----------------------------
+
+def auto_captions(words):
+    """
+    Full pipeline:
+    - detect hook
+    - generate captions
+    """
+
+    captions = caption_director(words)
+
+    hook = generate_hook_caption(words)
+
+    if hook:
+        captions.insert(0, hook)
+
+    return captions
+
+
+# -----------------------------
+# STYLE DECISION ENGINE
+# -----------------------------
+
+def decide_style(caption):
+    text = caption["text"]
+
+    if caption["style"] == "hook":
+        return "large_bold_center"
+
+    if len(text) > 40:
+        return "small_multi_line"
+
+    if text.isupper():
+        return "emphasis"
+
+    return "standard"
+
+
+# -----------------------------
+# EXPORT HELPERS
+# -----------------------------
+
+def format_for_render(captions):
+    """
+    Converts captions into render-friendly format
+    """
+
+    formatted = []
+
+    for c in captions:
+        formatted.append({
+            "text": c["text"],
+            "start": c["start"],
+            "end": c["end"],
+            "style": decide_style(c)
+        })
+
+    return formatted
+
+
+# -----------------------------
+# PUBLIC API
+# -----------------------------
+
+def process_captions(words):
+    """
+    Full external API
+    """
+
+    captions = auto_captions(words)
+
+    return format_for_render(captions)
\ No newline at end of file
diff --git a/services/whisper/utils/caption_seo.py b/services/whisper/utils/caption_seo.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee9ae85129368464a6b2e90e0e852fe61d70a80d
--- /dev/null
+++ b/services/whisper/utils/caption_seo.py
@@ -0,0 +1,7 @@
+def generate_caption(words):
+
+    keywords = list(set([w["text"].lower() for w in words[:10]]))
+
+    tags = " ".join([f"#{k}" for k in keywords[:5]])
+
+    return f"{' '.join(keywords[:8]).capitalize()}...\n\n{tags}"
\ No newline at end of file
diff --git a/services/whisper/utils/clipper.py b/services/whisper/utils/clipper.py
new file mode 100644
index 0000000000000000000000000000000000000000..554bce316083dc5461ff086d781f84440c02cda9
--- /dev/null
+++ b/services/whisper/utils/clipper.py
@@ -0,0 +1,130 @@
+from moviepy.editor import VideoFileClip
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+# =====================================================
+# SEGMENT NORMALIZER (CRITICAL V8 FIX)
+# =====================================================
+
+def normalize_segments(segments):
+    """
+    Accepts:
+      - dict segments: {"start": x, "end": y}
+      - list segments: [{"start":x,"end":y}, ...]
+      - tuple/list segments: [(start,end), ...]
+
+    Returns:
+      - clean list of dicts
+    """
+
+    if not segments:
+        return []
+
+    normalized = []
+
+    # Case 1: single dict
+    if isinstance(segments, dict):
+        segments = [segments]
+
+    for s in segments:
+
+        # dict format (preferred)
+        if isinstance(s, dict):
+            if "start" in s and "end" in s:
+                normalized.append({
+                    "start": float(s["start"]),
+                    "end": float(s["end"])
+                })
+            continue
+
+        # list/tuple format
+        if isinstance(s, (list, tuple)) and len(s) >= 2:
+            try:
+                normalized.append({
+                    "start": float(s[0]),
+                    "end": float(s[1])
+                })
+            except Exception:
+                continue
+
+    return normalized
+
+
+# =====================================================
+# CORE CLIP GENERATOR (SAFE VERSION)
+# =====================================================
+
+def create_clip(video_path, start, end, index):
+    """
+    Creates a single clip safely with validation
+    """
+
+    try:
+        start = float(start)
+        end = float(end)
+
+        if end <= start:
+            logger.warning(f"Invalid segment skipped: {start}-{end}")
+            return None
+
+        clip = VideoFileClip(video_path).subclip(start, end)
+
+        output = video_path.replace(
+            ".mp4",
+            f"_clip_{index}.mp4"
+        )
+
+        clip.write_videofile(
+            output,
+            codec="libx264",
+            audio_codec="aac",
+            preset="ultrafast",
+            threads=2,
+            logger=None  # prevents HF log spam
+        )
+
+        return output
+
+    except Exception as e:
+        logger.error(f"Clip creation failed: {str(e)}")
+        return None
+
+
+# =====================================================
+# BATCH CLIP ENGINE (V8 AUTOCUT CORE FIX)
+# =====================================================
+
+def create_clips(video_path, segments):
+    """
+    Main entry used by V8 Highlights / AutoClip engine
+    """
+
+    segments = normalize_segments(segments)
+
+    if not segments:
+        logger.warning("No valid segments found")
+        return []
+
+    outputs = []
+
+    for i, seg in enumerate(segments):
+
+        try:
+            out = create_clip(
+                video_path,
+                seg["start"],
+                seg["end"],
+                i
+            )
+
+            if out:
+                outputs.append(out)
+
+        except Exception as e:
+            logger.error(f"Segment {i} failed: {str(e)}")
+            continue
+
+    return outputs
\ No newline at end of file
diff --git a/services/whisper/utils/config.py b/services/whisper/utils/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..8622d3ccdb090ea288a4cab5babbc8139b29de13
--- /dev/null
+++ b/services/whisper/utils/config.py
@@ -0,0 +1,11 @@
+from pathlib import Path
+
+
+BASE_DIR = Path(__file__).resolve().parents[1]
+
+MODEL_SIZE = "base"
+
+FONT_PATH = str(BASE_DIR / "fonts" / "TikTok-Bold.ttf")
+
+OUTPUT_DIR = str(BASE_DIR / "outputs")
+TEMP_DIR = str(BASE_DIR / "temp")
diff --git a/services/whisper/utils/director.py b/services/whisper/utils/director.py
new file mode 100644
index 0000000000000000000000000000000000000000..b476af4d3dad578ca9fd7f2c84b8823d1a314580
--- /dev/null
+++ b/services/whisper/utils/director.py
@@ -0,0 +1,26 @@
+def rewrite_script(words):
+    """
+    Converts raw transcript → structured viral script
+    """
+
+    text = " ".join([w["text"] for w in words])
+
+    return {
+        "hook": f"Wait—{text[:60]}...",
+        "summary": text[:200],
+        "emotion": "high" if "!" in text else "medium"
+    }
+
+
+def viral_score(engagement_curve):
+    """
+    Evaluates full video potential
+    """
+
+    if not engagement_curve:
+        return 0
+
+    peak = max(engagement_curve)
+    avg = sum(engagement_curve) / len(engagement_curve)
+
+    return (peak * 0.7) + (avg * 0.3)
\ No newline at end of file
diff --git a/services/whisper/utils/emotion.py b/services/whisper/utils/emotion.py
new file mode 100644
index 0000000000000000000000000000000000000000..eed357ef3f50ffc05c3e75f5d97ec6221997ed34
--- /dev/null
+++ b/services/whisper/utils/emotion.py
@@ -0,0 +1,14 @@
+def emotion_score(words):
+
+    strong_words = [
+        "amazing", "crazy", "insane", "love", "hate",
+        "never", "always", "big", "huge", "shocking"
+    ]
+
+    score = 0
+
+    for w in words:
+        if w["text"].lower() in strong_words:
+            score += 2
+
+    return min(score, 100)
\ No newline at end of file
diff --git a/services/whisper/utils/engagement.py b/services/whisper/utils/engagement.py
new file mode 100644
index 0000000000000000000000000000000000000000..8063db1a370f66b09b95ce148aae9dbc416c9a6a
--- /dev/null
+++ b/services/whisper/utils/engagement.py
@@ -0,0 +1,17 @@
+def simulate_retention(words):
+
+    timeline = []
+    score = 100
+
+    for i, w in enumerate(words):
+
+        # drop-off logic
+        if i > len(words) * 0.7:
+            score -= 2
+
+        if w["text"].lower() in ["boring", "slow"]:
+            score -= 10
+
+        timeline.append(max(score, 0))
+
+    return timeline
\ No newline at end of file
diff --git a/services/whisper/utils/ffmpeg.py b/services/whisper/utils/ffmpeg.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0778e12bd3cce6574a9c7d91502d979920e0b75
--- /dev/null
+++ b/services/whisper/utils/ffmpeg.py
@@ -0,0 +1,14 @@
+import subprocess
+
+def extract_audio(video, audio):
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i", video,
+        "-vn",
+        "-acodec", "pcm_s16le",
+        "-ar", "16000",
+        "-ac", "1",
+        audio
+    ]
+    subprocess.run(cmd, check=True)
\ No newline at end of file
diff --git a/services/whisper/utils/highlights.py b/services/whisper/utils/highlights.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c17cf99b6b690a6d9941f8f97597124eaf56b5e
--- /dev/null
+++ b/services/whisper/utils/highlights.py
@@ -0,0 +1,35 @@
+def score_word(word):
+    """
+    Simple heuristic scoring:
+    - longer words slightly more important
+    - punctuation emphasis
+    """
+
+    score = len(word["text"]) * 0.1
+
+    if any(p in word["text"] for p in ["!", "?", "."]):
+        score += 1
+
+    return score
+
+
+def detect_highlights(words, threshold=0.8):
+    """
+    Groups words into highlight segments
+    """
+
+    highlights = []
+    buffer = []
+
+    for w in words:
+        if score_word(w) > threshold:
+            buffer.append(w)
+        else:
+            if buffer:
+                highlights.append(buffer)
+                buffer = []
+
+    if buffer:
+        highlights.append(buffer)
+
+    return highlights
\ No newline at end of file
diff --git a/services/whisper/utils/hook_detector.py b/services/whisper/utils/hook_detector.py
new file mode 100644
index 0000000000000000000000000000000000000000..254fa9bed943c861db78c4f03258b6c766ec66ef
--- /dev/null
+++ b/services/whisper/utils/hook_detector.py
@@ -0,0 +1,6 @@
+def detect_hook(words):
+    """
+    Finds strongest opening segment (first 3–7 seconds)
+    """
+
+    return [w for w in words if w["start"] <= 5]
\ No newline at end of file
diff --git a/services/whisper/utils/input_resolver.py b/services/whisper/utils/input_resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..97d809adb6d079861065c759e477facbacc34f43
--- /dev/null
+++ b/services/whisper/utils/input_resolver.py
@@ -0,0 +1,112 @@
+import os
+import uuid
+import requests
+import subprocess
+from urllib.parse import urlparse
+
+from utils.logger import logger
+
+DOWNLOAD_DIR = "jobs"
+os.makedirs(DOWNLOAD_DIR, exist_ok=True)
+
+
+# ---------------------------------------------------
+# URL DETECTION
+# ---------------------------------------------------
+
+def is_url(value: str):
+    try:
+        result = urlparse(value)
+        return result.scheme in ("http", "https")
+    except Exception:
+        return False
+
+
+def is_social_url(url: str):
+    domains = [
+        "youtube.com",
+        "youtu.be",
+        "tiktok.com",
+        "instagram.com",
+        "facebook.com",
+        "fb.watch",
+        "twitter.com",
+        "x.com"
+    ]
+    return any(d in url.lower() for d in domains)
+
+
+# ---------------------------------------------------
+# DIRECT FILE DOWNLOAD
+# ---------------------------------------------------
+
+def download_direct(url: str) -> str:
+    filename = f"{uuid.uuid4()}.mp4"
+    output = os.path.join(DOWNLOAD_DIR, filename)
+
+    logger.info(f"[INPUT] Direct download → {url}")
+
+    with requests.get(url, stream=True, timeout=120) as r:
+        r.raise_for_status()
+        with open(output, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+
+    logger.info(f"[INPUT] Saved → {output}")
+    return output
+
+
+# ---------------------------------------------------
+# SOCIAL MEDIA DOWNLOAD (yt-dlp)
+# ---------------------------------------------------
+
+def download_social(url: str) -> str:
+
+    filename = f"{uuid.uuid4()}.mp4"
+    output = os.path.join(DOWNLOAD_DIR, filename)
+
+    logger.info(f"[INPUT] Social download → {url}")
+
+    cmd = [
+        "yt-dlp",
+        "-f", "bestvideo+bestaudio/best",
+        "--merge-output-format", "mp4",
+        "-o", output,
+        url,
+    ]
+
+    subprocess.run(cmd, check=True)
+
+    if not os.path.exists(output):
+        raise Exception("yt-dlp download failed")
+
+    logger.info(f"[INPUT] Saved → {output}")
+    return output
+
+
+# ---------------------------------------------------
+# UNIVERSAL RESOLVER
+# ---------------------------------------------------
+
+def resolve_input(input_value):
+    """
+    Accepts:
+    - Upload path
+    - Direct URL
+    - YouTube/TikTok/Instagram/Facebook link
+    """
+
+    # Already local
+    if isinstance(input_value, str) and os.path.exists(input_value):
+        return input_value
+
+    # URL input
+    if isinstance(input_value, str) and is_url(input_value):
+
+        if is_social_url(input_value):
+            return download_social(input_value)
+
+        return download_direct(input_value)
+
+    raise Exception("Unsupported input type")
\ No newline at end of file
diff --git a/services/whisper/utils/job_queue.py b/services/whisper/utils/job_queue.py
new file mode 100644
index 0000000000000000000000000000000000000000..5135118af5985bc0003967a81633563359e47d13
--- /dev/null
+++ b/services/whisper/utils/job_queue.py
@@ -0,0 +1,194 @@
+import threading
+import uuid
+from queue import Queue
+import traceback
+import time
+
+from .logger import logger
+from .validators import validate_video
+from .transcription import transcribe_video
+from .director import rewrite_script, viral_score
+from .engagement import simulate_retention
+from .platform import adapt_platform
+from .persona import predict_audience
+from .clipper import create_clip
+
+
+# =====================================================
+# GLOBAL STATE
+# =====================================================
+
+jobs = {}
+queue = Queue()
+
+WORKER_STARTED = False
+
+
+# =====================================================
+# CREATE DIRECTOR JOB
+# =====================================================
+
+def create_job(video_path: str, webhook: str | None = None):
+
+    job_id = str(uuid.uuid4())
+
+    jobs[job_id] = {
+        "id": job_id,
+        "status": "queued",
+        "stage": "waiting",
+        "progress": 0,
+
+        "video": video_path,
+
+        # V7 INTELLIGENCE OUTPUTS
+        "viral_score": None,
+        "persona": None,
+        "hook": None,
+        "platforms": [],
+        "strategy_summary": None,
+
+        "clips": [],
+        "webhook": webhook,
+        "error": None,
+        "created_at": time.time(),
+    }
+
+    queue.put(job_id)
+
+    logger.info(f"[V7] Director job queued: {job_id}")
+
+    return job_id
+
+
+# =====================================================
+# GET JOB
+# =====================================================
+
+def get_job(job_id: str):
+    return jobs.get(job_id)
+
+
+# =====================================================
+# UPDATE HELPERS
+# =====================================================
+
+def update(job_id, **kwargs):
+    if job_id in jobs:
+        jobs[job_id].update(kwargs)
+
+
+# =====================================================
+# V7 AUTONOMOUS DIRECTOR WORKER
+# =====================================================
+
+def worker():
+
+    logger.info("[V7] Autonomous Viral Director started")
+
+    while True:
+
+        job_id = queue.get()
+        job = jobs[job_id]
+
+        try:
+
+            # -----------------------------
+            # 1. TRANSCRIPTION
+            # -----------------------------
+            update(job_id, status="processing", stage="transcribing", progress=10)
+            words = transcribe_video(job["video"])
+
+            # -----------------------------
+            # 2. SCRIPT RECONSTRUCTION
+            # -----------------------------
+            update(job_id, stage="rewriting narrative", progress=25)
+            script = rewrite_script(words)
+
+            # -----------------------------
+            # 3. AUDIENCE MODELING
+            # -----------------------------
+            persona = predict_audience(words)
+
+            # -----------------------------
+            # 4. ENGAGEMENT SIMULATION
+            # -----------------------------
+            update(job_id, stage="simulating audience", progress=45)
+            curve = simulate_retention(words)
+
+            v_score = viral_score(curve)
+
+            # -----------------------------
+            # 5. PLATFORM STRATEGY
+            # -----------------------------
+            update(job_id, stage="platform adaptation", progress=65)
+
+            tiktok = adapt_platform(script, "tiktok")
+            reels = adapt_platform(script, "reels")
+
+            platforms = ["tiktok", "reels"]
+
+            # -----------------------------
+            # 6. SINGLE BEST OUTPUT (DIRECTOR DECISION)
+            # -----------------------------
+            update(job_id, stage="rendering final cut", progress=85)
+
+            clip = create_clip(
+                job["video"],
+                words[0]["start"],
+                words[-1]["end"],
+                0
+            )
+
+            # -----------------------------
+            # 7. FINAL DIRECTOR OUTPUT
+            # -----------------------------
+            update(job_id,
+                   status="completed",
+                   stage="director finished",
+                   progress=100,
+
+                   viral_score=v_score,
+                   persona=persona,
+                   hook=script["hook"],
+                   platforms=platforms,
+
+                   strategy_summary={
+                       "curve_peak": max(curve),
+                       "avg_curve": sum(curve) / len(curve),
+                       "decision": "auto-selected best full narrative cut"
+                   },
+
+                   clips=[clip])
+
+            logger.info(f"[V7] Director output complete: {job_id}")
+
+        except Exception as e:
+
+            logger.error(traceback.format_exc())
+
+            update(job_id,
+                   status="failed",
+                   stage="error",
+                   error=str(e))
+
+        finally:
+            queue.task_done()
+
+
+# =====================================================
+# START WORKER (SINGLETON SAFE)
+# =====================================================
+
+def start_worker():
+
+    global WORKER_STARTED
+
+    if WORKER_STARTED:
+        return
+
+    WORKER_STARTED = True
+
+    t = threading.Thread(target=worker, daemon=True)
+    t.start()
+
+    logger.info("[V7] Worker initialized")
\ No newline at end of file
diff --git a/services/whisper/utils/jumpcut.py b/services/whisper/utils/jumpcut.py
new file mode 100644
index 0000000000000000000000000000000000000000..baa61f4e20c247688c350535d3bec560afac0c73
--- /dev/null
+++ b/services/whisper/utils/jumpcut.py
@@ -0,0 +1,226 @@
+"""
+jumpcut.py
+---------------------------------------
+Smart Jump Cut Engine (V8)
+
+Purpose:
+- Remove silence and filler pauses
+- Improve pacing for short-form video
+- Optimize retention curve
+- Create TikTok/Reels-style fast cuts
+
+Works fully on CPU (FFmpeg-based).
+No GPU required.
+"""
+
+import subprocess
+import os
+
+
+# =====================================================
+# CONFIG
+# =====================================================
+
+TEMP_SILENCE_FILE = "silence_detect.txt"
+OUTPUT_FILE = "jumpcut_output.mp4"
+
+
+# =====================================================
+# SILENCE DETECTION
+# =====================================================
+
+def detect_silence(video_path):
+    """
+    Uses ffmpeg silencedetect to find pauses.
+    """
+
+    cmd = [
+        "ffmpeg",
+        "-i", video_path,
+        "-af", "silencedetect=noise=-30dB:d=0.4",
+        "-f", "null",
+        "-"
+    ]
+
+    result = subprocess.run(cmd, stderr=subprocess.PIPE, text=True)
+
+    return result.stderr
+
+
+# =====================================================
+# PARSE SILENCE TIMESTAMPS
+# =====================================================
+
+def parse_silence(log):
+    """
+    Extract silence start/end timestamps
+    """
+
+    silences = []
+
+    start = None
+
+    for line in log.split("\n"):
+
+        if "silence_start" in line:
+            try:
+                start = float(line.split("silence_start:")[1].strip())
+            except:
+                continue
+
+        if "silence_end" in line and start is not None:
+            try:
+                end = float(line.split("silence_end:")[1].split("|")[0].strip())
+                silences.append((start, end))
+                start = None
+            except:
+                continue
+
+    return silences
+
+
+# =====================================================
+# BUILD FILTER (JUMP CUT LOGIC)
+# =====================================================
+
+def build_filter(silences, duration):
+    """
+    Converts silence ranges into ffmpeg trim filter
+    """
+
+    if not silences:
+        return None
+
+    segments = []
+    last_end = 0
+
+    for start, end in silences:
+
+        if start > last_end:
+            segments.append((last_end, start))
+
+        last_end = end
+
+    if last_end < duration:
+        segments.append((last_end, duration))
+
+    filters = []
+
+    for i, (start, end) in enumerate(segments):
+        filters.append(
+            f"[0:v]trim=start={start}:end={end},setpts=PTS-STARTPTS[v{i}];"
+            f"[0:a]atrim=start={start}:end={end},asetpts=PTS-STARTPTS[a{i}]"
+        )
+
+    video_concat = "".join([f"[v{i}]" for i in range(len(segments))])
+    audio_concat = "".join([f"[a{i}]" for i in range(len(segments))])
+
+    filters.append(
+        f"{video_concat}{audio_concat}concat=n={len(segments)}:v=1:a=1[outv][outa]"
+    )
+
+    return ";".join(filters)
+
+
+# =====================================================
+# CORE ENGINE
+# =====================================================
+
+def smart_jumpcut(video_path):
+    """
+    Main jump cut engine
+    """
+
+    print("[JUMPCUT] Analyzing video...")
+
+    # Step 1: detect silence
+    log = detect_silence(video_path)
+
+    silences = parse_silence(log)
+
+    print(f"[JUMPCUT] Detected silences: {len(silences)}")
+
+    # Step 2: get duration
+    probe_cmd = [
+        "ffprobe",
+        "-v", "error",
+        "-show_entries",
+        "format=duration",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        video_path
+    ]
+
+    duration = float(subprocess.check_output(probe_cmd).decode().strip())
+
+    # Step 3: build filter
+    filter_complex = build_filter(silences, duration)
+
+    if not filter_complex:
+        print("[JUMPCUT] No silences found, returning original")
+        return video_path
+
+    # Step 4: render output
+    output_path = OUTPUT_FILE
+
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", video_path,
+        "-filter_complex", filter_complex,
+        "-map", "[outv]",
+        "-map", "[outa]",
+        "-c:v", "libx264",
+        "-preset", "ultrafast",
+        "-c:a", "aac",
+        output_path
+    ]
+
+    print("[JUMPCUT] Rendering optimized video...")
+
+    subprocess.run(cmd, check=True)
+
+    print("[JUMPCUT] Done:", output_path)
+
+    return output_path
+
+
+# =====================================================
+# SIMPLE FAST MODE (FALLBACK)
+# =====================================================
+
+def fast_jumpcut(video_path):
+    """
+    Lightweight fallback:
+    removes only large pauses quickly
+    """
+
+    output = "fast_jumpcut.mp4"
+
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", video_path,
+        "-af", "silenceremove=start_periods=1:start_threshold=-30dB:stop_periods=-1",
+        "-c:v", "libx264",
+        "-preset", "ultrafast",
+        "-c:a", "aac",
+        output
+    ]
+
+    subprocess.run(cmd, check=True)
+
+    return output
+
+
+# =====================================================
+# PUBLIC API
+# =====================================================
+
+def smart_jumpcut_engine(video_path, mode="smart"):
+    """
+    Entry point used by main.py
+    """
+
+    if mode == "fast":
+        return fast_jumpcut(video_path)
+
+    return smart_jumpcut(video_path)
\ No newline at end of file
diff --git a/services/whisper/utils/logger.py b/services/whisper/utils/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..74b7676e5b3c7a193a52cd987c617815d267460b
--- /dev/null
+++ b/services/whisper/utils/logger.py
@@ -0,0 +1,8 @@
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s"
+)
+
+logger = logging.getLogger("fast_whisper_api")
\ No newline at end of file
diff --git a/services/whisper/utils/music.py b/services/whisper/utils/music.py
new file mode 100644
index 0000000000000000000000000000000000000000..f209e89f4dd8708e07b6303e468421e15b254b3d
--- /dev/null
+++ b/services/whisper/utils/music.py
@@ -0,0 +1,22 @@
+from moviepy.editor import AudioFileClip, CompositeAudioClip, VideoFileClip
+
+
+def add_music(video_path, music_path):
+
+    video = VideoFileClip(video_path)
+
+    music = (
+        AudioFileClip(music_path)
+        .volumex(0.15)
+        .set_duration(video.duration)
+    )
+
+    final_audio = CompositeAudioClip([video.audio, music])
+
+    video = video.set_audio(final_audio)
+
+    output = video_path.replace(".mp4", "_music.mp4")
+
+    video.write_videofile(output, codec="libx264")
+
+    return output
\ No newline at end of file
diff --git a/services/whisper/utils/pacing.py b/services/whisper/utils/pacing.py
new file mode 100644
index 0000000000000000000000000000000000000000..057160b2dedc1467144477cdfe76a60b6ffe37ad
--- /dev/null
+++ b/services/whisper/utils/pacing.py
@@ -0,0 +1,241 @@
+"""
+pacing.py
+---------------------------------------
+Retention & Pacing Optimization Engine (V8)
+
+Purpose:
+- Adjust video pacing for maximum retention
+- Compress slow segments
+- Emphasize high-value moments
+- Create TikTok / Reels optimized flow
+
+Works in CPU-only environments (FFmpeg-based).
+"""
+
+import subprocess
+import os
+
+
+# =====================================================
+# CONFIG
+# =====================================================
+
+OUTPUT_FILE = "pacing_optimized.mp4"
+
+SLOW_THRESHOLD = 1.25   # speed multiplier for slow segments
+FAST_THRESHOLD = 1.75   # speed multiplier for filler segments
+
+
+# =====================================================
+# BASIC SEGMENT ESTIMATION (NO ML DEPENDENCY)
+# =====================================================
+
+def estimate_segment_value(text):
+    """
+    Heuristic scoring system:
+    determines importance of spoken segment.
+    """
+
+    text = text.lower()
+
+    high_value_keywords = [
+        "you", "secret", "important", "stop",
+        "crazy", "insane", "listen", "this",
+        "money", "success", "life", "truth"
+    ]
+
+    filler_keywords = [
+        "um", "uh", "like", "you know", "so",
+        "actually", "basically"
+    ]
+
+    score = 1.0
+
+    # boost high value words
+    for w in high_value_keywords:
+        if w in text:
+            score += 0.6
+
+    # penalize filler speech
+    for w in filler_keywords:
+        if w in text:
+            score -= 0.4
+
+    return max(0.5, min(score, 2.0))
+
+
+# =====================================================
+# SPEED MAP GENERATOR
+# =====================================================
+
+def build_speed_map(words):
+    """
+    Converts transcript into pacing instructions
+    """
+
+    segments = []
+    buffer = []
+
+    for w in words:
+        buffer.append(w)
+
+        # group into micro segments
+        if len(buffer) >= 6:
+            segments.append(buffer)
+            buffer = []
+
+    if buffer:
+        segments.append(buffer)
+
+    speed_map = []
+
+    for seg in segments:
+
+        text = " ".join([w["word"] for w in seg])
+        score = estimate_segment_value(text)
+
+        start = seg[0]["start"]
+        end = seg[-1]["end"]
+
+        # decide speed
+        if score > 1.4:
+            speed = 1.0   # keep normal (important content)
+        elif score > 1.0:
+            speed = 1.15  # slight compression
+        else:
+            speed = FAST_THRESHOLD  # aggressive speed-up
+
+        speed_map.append({
+            "start": start,
+            "end": end,
+            "speed": speed
+        })
+
+    return speed_map
+
+
+# =====================================================
+# FFMEG FILTER BUILDER
+# =====================================================
+
+def build_filter(speed_map):
+    """
+    Creates FFmpeg atempo + setpts filter chain
+    """
+
+    filters = []
+
+    for i, seg in enumerate(speed_map):
+
+        start = seg["start"]
+        end = seg["end"]
+        speed = seg["speed"]
+
+        # video speed
+        filters.append(
+            f"[0:v]trim=start={start}:end={end},setpts=PTS/{speed}[v{i}]"
+        )
+
+        # audio speed
+        filters.append(
+            f"[0:a]atrim=start={start}:end={end},asetpts=PTS-STARTPTS,"
+            f"atempo={speed}[a{i}]"
+        )
+
+    v_streams = "".join([f"[v{i}]" for i in range(len(speed_map))])
+    a_streams = "".join([f"[a{i}]" for i in range(len(speed_map))])
+
+    filters.append(
+        f"{v_streams}{a_streams}concat=n={len(speed_map)}:v=1:a=1[outv][outa]"
+    )
+
+    return ";".join(filters)
+
+
+# =====================================================
+# MAIN ENGINE
+# =====================================================
+
+def optimize_pacing(video_path, words=None):
+    """
+    Main entry point for V8 pacing system
+    """
+
+    print("[PACING] Starting optimization...")
+
+    if not words:
+        print("[PACING] No transcript provided — returning original video")
+        return video_path
+
+    # Step 1: build speed map
+    speed_map = build_speed_map(words)
+
+    print(f"[PACING] Segments: {len(speed_map)}")
+
+    # Step 2: build ffmpeg filter
+    filter_complex = build_filter(speed_map)
+
+    output_path = OUTPUT_FILE
+
+    # Step 3: render optimized video
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", video_path,
+        "-filter_complex", filter_complex,
+        "-map", "[outv]",
+        "-map", "[outa]",
+        "-c:v", "libx264",
+        "-preset", "ultrafast",
+        "-c:a", "aac",
+        output_path
+    ]
+
+    subprocess.run(cmd, check=True)
+
+    print("[PACING] Done:", output_path)
+
+    return output_path
+
+
+# =====================================================
+# LIGHTWEIGHT MODE (FAST FALLBACK)
+# =====================================================
+
+def fast_pacing(video_path):
+    """
+    Simple fallback: global speed-up only
+    """
+
+    output = "fast_pacing.mp4"
+
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", video_path,
+        "-filter_complex",
+        "[0:v]setpts=0.92*PTS[v];[0:a]atempo=1.08[a]",
+        "-map", "[v]",
+        "-map", "[a]",
+        "-c:v", "libx264",
+        "-preset", "ultrafast",
+        "-c:a", "aac",
+        output
+    ]
+
+    subprocess.run(cmd, check=True)
+
+    return output
+
+
+# =====================================================
+# PUBLIC API
+# =====================================================
+
+def pacing_engine(video_path, words=None, mode="smart"):
+    """
+    Entry point used by main.py
+    """
+
+    if mode == "fast":
+        return fast_pacing(video_path)
+
+    return optimize_pacing(video_path, words)
\ No newline at end of file
diff --git a/services/whisper/utils/persona.py b/services/whisper/utils/persona.py
new file mode 100644
index 0000000000000000000000000000000000000000..08f49ff9f60214e601591690baa6e7dec3031548
--- /dev/null
+++ b/services/whisper/utils/persona.py
@@ -0,0 +1,9 @@
+def predict_audience(words):
+
+    if len(words) < 50:
+        return "short_attention_gen_z"
+
+    if any(w["text"].lower() in ["money", "success"] for w in words):
+        return "motivated_entrepreneur"
+
+    return "general"
\ No newline at end of file
diff --git a/services/whisper/utils/platform.py b/services/whisper/utils/platform.py
new file mode 100644
index 0000000000000000000000000000000000000000..5817bff53e26afd231e8c8e7c7879bfaf3ef48ce
--- /dev/null
+++ b/services/whisper/utils/platform.py
@@ -0,0 +1,15 @@
+def adapt_platform(script, platform="tiktok"):
+
+    if platform == "tiktok":
+        return {
+            "hook": "🔥 " + script["hook"],
+            "style": "fast_cut"
+        }
+
+    if platform == "reels":
+        return {
+            "hook": script["hook"],
+            "style": "cinematic"
+        }
+
+    return script
\ No newline at end of file
diff --git a/services/whisper/utils/render.py b/services/whisper/utils/render.py
new file mode 100644
index 0000000000000000000000000000000000000000..97673c4c7f73e1787d67a4b096e16dc2efd0bc46
--- /dev/null
+++ b/services/whisper/utils/render.py
@@ -0,0 +1,73 @@
+import os
+import subprocess
+import uuid
+from utils.logger import logger
+
+OUTPUT_DIR = "jobs"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+
+def render_subtitles(
+    video_path: str,
+    srt_text: str,
+    output_path: str | None = None,
+):
+    """
+    Universal subtitle renderer for V7.
+
+    Supports:
+    - API render
+    - UI render
+    - Batch jobs
+    - Worker queue
+    """
+
+    if output_path is None:
+        output_path = os.path.join(
+            OUTPUT_DIR,
+            f"{uuid.uuid4()}_render.mp4"
+        )
+
+    # --------------------------------------------------
+    # Write SRT file
+    # --------------------------------------------------
+
+    srt_path = output_path.replace(".mp4", ".srt")
+
+    with open(srt_path, "w", encoding="utf-8") as f:
+        f.write(srt_text)
+
+    logger.info(f"[RENDER] SRT saved → {srt_path}")
+
+    # --------------------------------------------------
+    # FFmpeg Subtitle Burn
+    # --------------------------------------------------
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i", video_path,
+        "-vf", f"subtitles={srt_path}",
+        "-c:a", "copy",
+        output_path,
+    ]
+
+    logger.info("[RENDER] Running ffmpeg render")
+
+    process = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    if process.returncode != 0:
+        logger.error(process.stderr)
+        raise Exception("FFmpeg render failed")
+
+    if not os.path.exists(output_path):
+        raise Exception("Rendered file missing")
+
+    logger.info(f"[RENDER] Output → {output_path}")
+
+    return output_path
\ No newline at end of file
diff --git a/services/whisper/utils/retention.py b/services/whisper/utils/retention.py
new file mode 100644
index 0000000000000000000000000000000000000000..0e05d16a4a5130ab400fc351103bdca8c10054ae
--- /dev/null
+++ b/services/whisper/utils/retention.py
@@ -0,0 +1,22 @@
+def predict_retention(words):
+
+    if not words:
+        return 0
+
+    duration = words[-1]["end"] - words[0]["start"]
+
+    score = 100
+
+    # too long → drop-off risk
+    if duration > 30:
+        score -= 30
+
+    # too short → no engagement
+    if duration < 6:
+        score -= 20
+
+    # weak opening signal
+    if "..." in words[0]["text"]:
+        score -= 10
+
+    return max(0, score)
\ No newline at end of file
diff --git a/services/whisper/utils/silence.py b/services/whisper/utils/silence.py
new file mode 100644
index 0000000000000000000000000000000000000000..f13c4c36bcf82c1f3484f0671f466976769b1d20
--- /dev/null
+++ b/services/whisper/utils/silence.py
@@ -0,0 +1,17 @@
+import subprocess
+
+
+def remove_silence(input_video, output_video):
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i", input_video,
+        "-af",
+        "silenceremove=start_periods=1:start_threshold=-40dB",
+        output_video
+    ]
+
+    subprocess.run(cmd, check=True)
+
+    return output_video
\ No newline at end of file
diff --git a/services/whisper/utils/srt.py b/services/whisper/utils/srt.py
new file mode 100644
index 0000000000000000000000000000000000000000..44bc931df4c4bb09dccef9fd7b713dccc55ff007
--- /dev/null
+++ b/services/whisper/utils/srt.py
@@ -0,0 +1,188 @@
+from typing import List, Dict, Any, Union
+
+
+# =====================================================
+# PUBLIC API (USED BY MAIN.PY)
+# =====================================================
+
+def generate_srt(data: List[Dict[str, Any]]) -> str:
+    """
+    Universal SRT generator for:
+    - Whisper word output (V1–V7)
+    - Highlight segments (start/end grouped words)
+    - Mixed/partial structures
+
+    Expected input formats:
+    1. Word-level:
+        {"text": "...", "start": float, "end": float}
+
+    2. Segment-level:
+        [{"start": float, "end": float, "text": "..."}]
+
+    Returns:
+        SRT formatted string
+    """
+
+    if not data:
+        return ""
+
+    normalized = _normalize_input(data)
+    return _build_srt(normalized)
+
+
+# =====================================================
+# NORMALIZATION LAYER (CRITICAL FOR V1–V7 COMPATIBILITY)
+# =====================================================
+
+def _normalize_input(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Converts any supported structure into unified subtitle blocks
+    """
+
+    normalized = []
+
+    # CASE 1: Already segment-based
+    if isinstance(data[0], dict) and "start" in data[0] and "end" in data[0] and "text" in data[0]:
+        for item in data:
+            normalized.append({
+                "start": float(item.get("start", 0)),
+                "end": float(item.get("end", 0)),
+                "text": str(item.get("text", "")).strip()
+            })
+        return normalized
+
+    # CASE 2: Whisper word-level output
+    buffer = []
+    current_start = None
+
+    for w in data:
+
+        if not isinstance(w, dict):
+            continue
+
+        text = str(w.get("text", "")).strip()
+        start = w.get("start", None)
+        end = w.get("end", None)
+
+        if start is None or end is None:
+            continue
+
+        if current_start is None:
+            current_start = start
+
+        buffer.append(text)
+
+        # Chunking strategy: group every ~8–12 words
+        if len(buffer) >= 10:
+
+            normalized.append({
+                "start": current_start,
+                "end": end,
+                "text": " ".join(buffer)
+            })
+
+            buffer = []
+            current_start = None
+
+    # flush remaining buffer
+    if buffer:
+        normalized.append({
+            "start": current_start or 0,
+            "end": data[-1].get("end", 0),
+            "text": " ".join(buffer)
+        })
+
+    return normalized
+
+
+# =====================================================
+# SRT BUILDER
+# =====================================================
+
+def _build_srt(items: List[Dict[str, Any]]) -> str:
+    """
+    Converts normalized subtitle blocks → SRT format
+    """
+
+    output = []
+    index = 1
+
+    for item in items:
+
+        start = _format_time(item["start"])
+        end = _format_time(item["end"])
+        text = _clean_text(item["text"])
+
+        if not text:
+            continue
+
+        output.append(f"{index}")
+        output.append(f"{start} --> {end}")
+        output.append(f"{text}")
+        output.append("")  # blank line separator
+
+        index += 1
+
+    return "\n".join(output).strip()
+
+
+# =====================================================
+# TIME FORMATTER
+# =====================================================
+
+def _format_time(seconds: Union[int, float]) -> str:
+    """
+    Converts seconds → SRT timestamp format
+    HH:MM:SS,mmm
+    """
+
+    try:
+        seconds = float(seconds)
+    except:
+        seconds = 0.0
+
+    hrs = int(seconds // 3600)
+    mins = int((seconds % 3600) // 60)
+    secs = int(seconds % 60)
+    ms = int((seconds - int(seconds)) * 1000)
+
+    return f"{hrs:02}:{mins:02}:{secs:02},{ms:03}"
+
+
+# =====================================================
+# TEXT CLEANER (IMPORTANT FOR VIDEO RENDERING STABILITY)
+# =====================================================
+
+def _clean_text(text: str) -> str:
+    """
+    Sanitizes subtitle text for rendering engines
+    """
+
+    if not text:
+        return ""
+
+    text = text.replace("\n", " ")
+    text = text.replace("\r", " ")
+
+    # remove excessive spacing
+    text = " ".join(text.split())
+
+    return text.strip()
+
+
+# =====================================================
+# OPTIONAL DEBUG HELPER (SAFE IN PRODUCTION)
+# =====================================================
+
+def debug_srt(data: List[Dict[str, Any]]) -> dict:
+    """
+    Returns structured preview for debugging pipelines
+    """
+
+    normalized = _normalize_input(data)
+
+    return {
+        "blocks": len(normalized),
+        "sample": normalized[:3],
+        "duration": normalized[-1]["end"] if normalized else 0
+    }
\ No newline at end of file
diff --git a/services/whisper/utils/storage.py b/services/whisper/utils/storage.py
new file mode 100644
index 0000000000000000000000000000000000000000..289ca0522e6bd19fd2f30840d868253030ed6a62
--- /dev/null
+++ b/services/whisper/utils/storage.py
@@ -0,0 +1,18 @@
+import os
+import time
+
+MAX_AGE = 60 * 60  # 1 hour
+
+
+def cleanup(folder="jobs"):
+
+    now = time.time()
+
+    for f in os.listdir(folder):
+
+        path = os.path.join(folder, f)
+
+        if os.path.isfile(path):
+
+            if now - os.path.getmtime(path) > MAX_AGE:
+                os.remove(path)
\ No newline at end of file
diff --git a/services/whisper/utils/strategist.py b/services/whisper/utils/strategist.py
new file mode 100644
index 0000000000000000000000000000000000000000..38b107f11fe951ccf77c4be1f213f931931ca891
--- /dev/null
+++ b/services/whisper/utils/strategist.py
@@ -0,0 +1,29 @@
+def rewrite_hook(text):
+    """
+    Simulates GPT-style hook optimization
+    """
+
+    if not text:
+        return "You won’t believe this..."
+
+    return f"Wait—{text.strip().capitalize()}"
+
+
+def strategy_score(words):
+    """
+    Combines virality + structure + hook strength
+    """
+
+    base_score = len(words)
+
+    text = " ".join([w["text"] for w in words]).lower()
+
+    # Hook boost
+    if any(k in text for k in ["you", "stop", "wait", "imagine"]):
+        base_score += 30
+
+    # Emotional intensity
+    if "!" in text:
+        base_score += 15
+
+    return min(100, base_score)
\ No newline at end of file
diff --git a/services/whisper/utils/subtitle_engine.py b/services/whisper/utils/subtitle_engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..20c0cbc7214423856d77f7cd7bccbe7b7cec12ae
--- /dev/null
+++ b/services/whisper/utils/subtitle_engine.py
@@ -0,0 +1,24 @@
+from moviepy.editor import TextClip
+from .config import FONT_PATH
+
+
+def word_clip(word):
+
+    duration = word["end"] - word["start"]
+
+    clip = TextClip(
+        word["text"],
+        font=FONT_PATH,
+        fontsize=80,
+        color="white",
+        stroke_color="black",
+        stroke_width=4,
+        method="caption",
+        size=(900, None),
+    )
+
+    return (
+        clip.set_start(word["start"])
+        .set_duration(duration)
+        .set_position(("center", "center"))
+    )
\ No newline at end of file
diff --git a/services/whisper/utils/transcription.py b/services/whisper/utils/transcription.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcba5a598e453b27d8bdd2f73a7337054a072497
--- /dev/null
+++ b/services/whisper/utils/transcription.py
@@ -0,0 +1,44 @@
+from faster_whisper import WhisperModel
+from .config import MODEL_SIZE, TEMP_DIR
+from .ffmpeg import extract_audio
+import os
+
+model = None
+
+os.makedirs(TEMP_DIR, exist_ok=True)
+
+
+def get_model():
+    global model
+
+    if model is None:
+        model = WhisperModel(
+            MODEL_SIZE,
+            device="cpu",
+            compute_type="int8"
+        )
+
+    return model
+
+
+def transcribe_video(video):
+
+    audio = f"{TEMP_DIR}/audio.wav"
+    extract_audio(video, audio)
+
+    segments, _ = get_model().transcribe(
+        audio,
+        word_timestamps=True
+    )
+
+    words = []
+
+    for seg in segments:
+        for w in seg.words:
+            words.append({
+                "start": w.start,
+                "end": w.end,
+                "text": w.word.strip()
+            })
+
+    return words
diff --git a/services/whisper/utils/validators.py b/services/whisper/utils/validators.py
new file mode 100644
index 0000000000000000000000000000000000000000..e1c5b8039b01af8bc8a3203e93a70953cb1a535a
--- /dev/null
+++ b/services/whisper/utils/validators.py
@@ -0,0 +1,14 @@
+import os
+
+ALLOWED = [".mp4", ".mov", ".mkv"]
+
+
+def validate_video(path):
+
+    if not os.path.exists(path):
+        raise Exception("File not found")
+
+    ext = os.path.splitext(path)[1].lower()
+
+    if ext not in ALLOWED:
+        raise Exception("Unsupported video format")
\ No newline at end of file
diff --git a/services/whisper/utils/variations.py b/services/whisper/utils/variations.py
new file mode 100644
index 0000000000000000000000000000000000000000..06d03abc6845a969006f3db1191ea403eea3dbc5
--- /dev/null
+++ b/services/whisper/utils/variations.py
@@ -0,0 +1,238 @@
+"""
+variations.py
+---------------------------------------
+Hook & Script Variation Engine (V8)
+
+Purpose:
+- Generate multiple viral hooks
+- Create alternative script directions
+- Support A/B testing of edits
+- Enable Multi-Version Render pipeline
+
+Works fully CPU-only.
+No external API required.
+"""
+
+import random
+import hashlib
+
+
+# =====================================================
+# HOOK TEMPLATES (VIRAL PATTERNS)
+# =====================================================
+
+HOOK_PATTERNS = [
+    "You are not going to believe this...",
+    "This is what nobody tells you about {}",
+    "Stop scrolling if you want to understand {}",
+    "The truth about {} will shock you",
+    "Most people get {} wrong",
+    "If you understand this, your {} changes forever",
+    "I wish I knew this before about {}",
+    "This is how you actually win at {}",
+    "Everyone is lying about {}",
+    "Watch this before it's too late..."
+]
+
+
+# =====================================================
+# TEXT CLEANER
+# =====================================================
+
+def extract_keywords(words):
+    """
+    Extract simple keyword candidates from transcript
+    """
+
+    freq = {}
+
+    for w in words:
+        word = w["word"].lower().strip()
+        if len(word) < 3:
+            continue
+        freq[word] = freq.get(word, 0) + 1
+
+    sorted_words = sorted(freq.items(), key=lambda x: x[1], reverse=True)
+
+    return [w[0] for w in sorted_words[:5]]
+
+
+# =====================================================
+# HOOK GENERATION
+# =====================================================
+
+def generate_hooks(words, count=5):
+    """
+    Generate multiple viral hooks from transcript
+    """
+
+    keywords = extract_keywords(words)
+
+    hooks = []
+
+    for i in range(count):
+
+        template = random.choice(HOOK_PATTERNS)
+
+        keyword = random.choice(keywords) if keywords else "this"
+
+        try:
+            hook = template.format(keyword)
+        except:
+            hook = template
+
+        hooks.append(hook)
+
+    return hooks
+
+
+# =====================================================
+# SCRIPT VARIATION ENGINE
+# =====================================================
+
+def generate_script_variations(words):
+    """
+    Creates alternative narrative directions
+    """
+
+    base_text = " ".join([w["word"] for w in words])
+
+    variations = []
+
+    variations.append({
+        "style": "direct",
+        "script": base_text
+    })
+
+    variations.append({
+        "style": "emotional",
+        "script": "Imagine this... " + base_text
+    })
+
+    variations.append({
+        "style": "urgent",
+        "script": "You need to hear this: " + base_text
+    })
+
+    variations.append({
+        "style": "story",
+        "script": "Let me tell you something important. " + base_text
+    })
+
+    return variations
+
+
+# =====================================================
+# CAPTION VARIATION ENGINE
+# =====================================================
+
+def generate_caption_variations(captions):
+    """
+    Creates multiple caption styles for rendering
+    """
+
+    styles = []
+
+    for c in captions:
+
+        styles.append({
+            "style": "bold_center",
+            "text": c["text"].upper()
+        })
+
+        styles.append({
+            "style": "minimal",
+            "text": c["text"]
+        })
+
+        styles.append({
+            "style": "emphasis_words",
+            "text": highlight_keywords(c["text"])
+        })
+
+    return styles
+
+
+# =====================================================
+# KEYWORD HIGHLIGHTER
+# =====================================================
+
+def highlight_keywords(text):
+    """
+    Emphasizes strong words in captions
+    """
+
+    keywords = ["you", "this", "stop", "now", "secret", "important"]
+
+    words = text.split()
+
+    output = []
+
+    for w in words:
+        if w.lower() in keywords:
+            output.append(w.upper())
+        else:
+            output.append(w)
+
+    return " ".join(output)
+
+
+# =====================================================
+# MULTI VERSION RENDER ENGINE
+# =====================================================
+
+def generate_render_variations(video_path, hooks=None):
+    """
+    Creates multiple render variants metadata
+    (actual rendering happens in render.py)
+    """
+
+    if not hooks:
+        hooks = ["Hook 1", "Hook 2", "Hook 3"]
+
+    outputs = []
+
+    for i, hook in enumerate(hooks):
+
+        outputs.append({
+            "version": i + 1,
+            "hook": hook,
+            "output_file": f"render_variant_{i+1}.mp4"
+        })
+
+    return outputs
+
+
+# =====================================================
+# DETERMINISTIC VIRAL HASH
+# =====================================================
+
+def viral_signature(text):
+    """
+    Creates deterministic ID for A/B testing consistency
+    """
+
+    return hashlib.md5(text.encode()).hexdigest()[:10]
+
+
+# =====================================================
+# PUBLIC API
+# =====================================================
+
+def generate_hooks_only(words):
+    return generate_hooks(words)
+
+
+def generate_full_variations(words):
+    """
+    Full pipeline for V8 Multi-Version system
+    """
+
+    hooks = generate_hooks(words)
+
+    scripts = generate_script_variations(words)
+
+    return {
+        "hooks": hooks,
+        "scripts": scripts
+    }
\ No newline at end of file
diff --git a/services/whisper/utils/vertical.py b/services/whisper/utils/vertical.py
new file mode 100644
index 0000000000000000000000000000000000000000..87374164e2c3e96991a5728e40f8d8d1b83d6a42
--- /dev/null
+++ b/services/whisper/utils/vertical.py
@@ -0,0 +1,25 @@
+from moviepy.editor import VideoFileClip
+
+
+def vertical_crop(video_path):
+
+    clip = VideoFileClip(video_path)
+
+    w, h = clip.size
+    target_ratio = 9 / 16
+
+    new_width = int(h * target_ratio)
+
+    x_center = w / 2
+
+    cropped = clip.crop(
+        x_center=x_center,
+        width=new_width,
+        height=h
+    )
+
+    output = video_path.replace(".mp4", "_vertical.mp4")
+
+    cropped.write_videofile(output, codec="libx264")
+
+    return output
\ No newline at end of file
diff --git a/services/whisper/utils/viral_scorer.py b/services/whisper/utils/viral_scorer.py
new file mode 100644
index 0000000000000000000000000000000000000000..c39ce00a1cd16301765bab739a3ff697a289eca1
--- /dev/null
+++ b/services/whisper/utils/viral_scorer.py
@@ -0,0 +1,46 @@
+def score_clip(words):
+    """
+    Returns virality score (0–100)
+    based on speech + structure signals
+    """
+
+    if not words:
+        return 0
+
+    text = " ".join([w["text"] for w in words]).lower()
+
+    score = 0
+
+    # -----------------------------
+    # HOOK SIGNAL (first words)
+    # -----------------------------
+    hook_words = ["you", "imagine", "stop", "listen", "this", "never", "why"]
+
+    if any(h in text[:50] for h in hook_words):
+        score += 25
+
+    # -----------------------------
+    # EMOTION SIGNAL
+    # -----------------------------
+    exclamations = sum(1 for w in words if "!" in w["text"])
+    score += min(exclamations * 5, 20)
+
+    # -----------------------------
+    # LENGTH OPTIMIZATION
+    # -----------------------------
+    duration = words[-1]["end"] - words[0]["start"]
+
+    if 6 <= duration <= 25:
+        score += 25
+    elif duration < 6:
+        score -= 10
+    else:
+        score -= 5
+
+    # -----------------------------
+    # WORD DENSITY
+    # -----------------------------
+    score += min(len(words) / 2, 20)
+
+    # Clamp
+    return max(0, min(100, score))
\ No newline at end of file
diff --git a/services/whisper/utils/zoom_tracker.py b/services/whisper/utils/zoom_tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..08525418ed13d724362e6fa49db1508ecc6627c8
--- /dev/null
+++ b/services/whisper/utils/zoom_tracker.py
@@ -0,0 +1,11 @@
+def zoom_intensity(score):
+    """
+    Maps viral score to zoom level
+    """
+
+    if score > 80:
+        return 1.2  # aggressive zoom
+    elif score > 50:
+        return 1.1
+    else:
+        return 1.0
\ No newline at end of file