Deploy DoppelGen compiled C-extension binary distribution to Hugging Face Space

.gitattributes

@@ -0,0 +1,7 @@
+*.so filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text
+doppelgen/data/jobs/test/input/** filter=lfs diff=lfs merge=lfs -text

.github/scripts/compile_binaries.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""
+DoppelGen Hardened Native Compiler Script
+Compiles core Python packages into Cython native C-extensions (.so)
+with fallback to bytecode (.pyc) for dynamic modules, and strips raw .py source files.
+"""
+
+import os
+import sys
+import shutil
+import py_compile
+import subprocess
+from setuptools import setup, Extension
+from Cython.Build import cythonize
+
+def main():
+    print("=" * 60)
+    print("🚀 Starting DoppelGen Cython Native Binary Compilation (.so)...")
+    print("=" * 60)
+
+    base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+    os.chdir(base_dir)
+
+    target_packages = ["doppelgen", "actora", "voxa", "sonora", "scenea", "captiona", "ocula"]
+
+    compiled_so_count = 0
+    compiled_pyc_count = 0
+    deleted_count = 0
+
+    for pkg in target_packages:
+        pkg_dir = os.path.join(base_dir, pkg)
+        if not os.path.exists(pkg_dir):
+            continue
+
+        os.chdir(pkg_dir)
+
+        py_files_in_pkg = []
+        for root, dirs, files in os.walk(pkg_dir):
+            if "third_party" in root or "hf_cache" in root or ".cache" in root or "checkpoints" in root:
+                continue
+            if os.path.basename(root) in ["runners", "test", "tests", "docker", "build", "__pycache__"]:
+                continue
+            for file in files:
+                if file.endswith(".py") and not file.startswith("."):
+                    if file in ["__init__.py", "preload_models.py", "app.py", "setup.py"]:
+                        continue
+                    full_path = os.path.join(root, file)
+                    rel_to_pkg = os.path.relpath(full_path, pkg_dir)
+                    module_name = rel_to_pkg[:-3].replace(os.sep, ".")
+                    py_files_in_pkg.append((full_path, rel_to_pkg, module_name))
+
+        print(f"📦 Compiling {len(py_files_in_pkg)} modules in {pkg}...")
+
+        for full_path, rel_to_pkg, module_name in py_files_in_pkg:
+            ext = Extension(
+                module_name,
+                sources=[rel_to_pkg],
+                extra_compile_args=["-O3", "-fPIC", "-Wno-unused-variable"],
+            )
+
+            success = False
+            try:
+                setup(
+                    ext_modules=cythonize(
+                        [ext],
+                        compiler_directives={
+                            'language_level': "3",
+                            'always_allow_keywords': True,
+                            'embedsignature': False,
+                            'annotation_typing': False,
+                        },
+                        quiet=True,
+                    ),
+                    script_args=["build_ext", "--inplace"],
+                )
+                dir_name = os.path.dirname(rel_to_pkg)
+                base_name = os.path.basename(rel_to_pkg)[:-3]
+                search_dir = os.path.join(pkg_dir, dir_name) if dir_name else pkg_dir
+                matching_so = [f for f in os.listdir(search_dir) if f.startswith(base_name) and f.endswith(".so")]
+                if matching_so:
+                    success = True
+                    compiled_so_count += 1
+            except Exception as e:
+                print(f"⚠️ Cython compilation fallback to .pyc for {rel_to_pkg}: {e}")
+
+            if not success:
+                pyc_path = full_path + "c"
+                try:
+                    py_compile.compile(full_path, cfile=pyc_path, doraise=True, optimize=2)
+                    compiled_pyc_count += 1
+                except Exception as pe:
+                    print(f"❌ Bytecode compilation failed for {rel_to_pkg}: {pe}")
+
+        for full_path, rel_to_pkg, module_name in py_files_in_pkg:
+            dir_name = os.path.dirname(rel_to_pkg)
+            base_name = os.path.basename(rel_to_pkg)[:-3]
+            search_dir = os.path.join(pkg_dir, dir_name) if dir_name else pkg_dir
+            matching_so = [f for f in os.listdir(search_dir) if f.startswith(base_name) and f.endswith(".so")] if os.path.exists(search_dir) else []
+            if matching_so:
+                if os.path.exists(full_path):
+                    os.remove(full_path)
+                    deleted_count += 1
+            else:
+                print(f"ℹ️ Preserved {rel_to_pkg} as .py source (no .so generated)", flush=True)
+
+    os.chdir(base_dir)
+
+    # Strip debugging symbols from generated .so files
+    for root, dirs, files in os.walk(base_dir):
+        if ".git" in root or "venv" in root or ".venv" in root:
+            continue
+        for file in files:
+            if file.endswith(".so"):
+                so_path = os.path.join(root, file)
+                try:
+                    subprocess.run(["strip", "--strip-debug", so_path], check=False)
+                except Exception:
+                    pass
+
+    print(f"🔒 Compiled {compiled_so_count} Cython .so C-extensions (stripped symbols).")
+    print(f"⚡ Compiled {compiled_pyc_count} modules to optimized .pyc bytecode.")
+
+    # Clean up temporary Cython .c, .cpp, and build directories
+    for root, dirs, files in os.walk(base_dir):
+        for file in files:
+            if file.endswith(".c") or file.endswith(".cpp"):
+                if "third_party" not in root and not file.startswith("c_"):
+                    c_path = os.path.join(root, file)
+                    try:
+                        os.remove(c_path)
+                    except OSError:
+                        pass
+
+        build_dir = os.path.join(root, "build")
+        if os.path.exists(build_dir):
+            shutil.rmtree(build_dir, ignore_errors=True)
+
+    print(f"🧹 Removed {deleted_count} original source .py files (replaced with compiled binaries).")
+    print("=" * 60)
+    print("✨ DoppelGen Hardened Binary Build Complete!")
+    print("=" * 60)
+
+if __name__ == "__main__":
+    main()

.github/workflows/deploy_hf.yml

@@ -0,0 +1,77 @@
+name: Sync to Hugging Face Spaces
+
+on:
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+
+# Single-flight deploys: each run force-pushes an orphan branch with fresh
+# LFS objects; interleaved runs corrupt HF repo state (exit-128 build failures).
+concurrency:
+  group: deploy-hf
+  cancel-in-progress: true
+
+jobs:
+  sync-to-hf:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python 3.12 & GCC Build Toolchain
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Compiler Dependencies & Git LFS
+        run: |
+          sudo apt-get update && sudo apt-get install -y git-lfs
+          git lfs install
+          python -m pip install --upgrade pip
+          pip install Cython setuptools wheel
+
+      - name: Compile Python Core Packages to Native .so C-Binaries
+        run: |
+          python .github/scripts/compile_binaries.py
+
+      - name: Push Compiled Binaries to Hugging Face Space via Git LFS
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          
+          # Create a clean orphan branch without historical git blobs
+          git checkout --orphan hf-deploy
+          
+          # Setup Git LFS for binary tracking
+          git lfs install
+          git lfs track "*.so"
+          git lfs track "*.mp4"
+          git lfs track "*.webm"
+          # Default avatar/background fixtures must ride LFS — HF rejects plain binary blobs
+          git lfs track "doppelgen/data/jobs/test/input/**"
+          git add .gitattributes
+          
+          # Add all files including compiled .so binaries
+          git add -A
+          
+          # Remove heavy binary files to comply with HF storage policy
+          # (Preserve ocula_visuals — tracked via Git LFS, and the small default
+          #  avatar/background fixtures Actora needs at runtime. DreamTalk's
+          #  style-clip/pose .mat assets are downloaded by preload_models.py
+          #  at boot, so they are intentionally stripped here.)
+          find . -type f \( \
+            -name "*.png" -o -name "*.jpg" -o -name "*.jpeg" -o \
+            -name "*.wav" -o -name "*.mp3" -o -name "*.mp4" -o -name "*.m4a" -o -name "*.webm" -o -name "*.gif" -o \
+            -name "*.onnx" -o -name "*.pth" -o -name "*.pt" -o -name "*.tar" -o -name "*.npy" -o -name "*.mat" -o \
+            -name "*.pkl" -o -name "*.zip" -o -name "*.data" -o -name "*.bin" \
+          \) ! -path "*/ocula_visuals/*" ! -path "*doppelgen/data/jobs/test/input/*" -exec git rm -f --cached {} + || true
+          
+          git commit -m "Deploy DoppelGen compiled C-extension binary distribution to Hugging Face Space"
+          
+          git remote add hf https://Hazeezadebayo:$HF_TOKEN@huggingface.co/spaces/Hazeezadebayo/doppelgen
+          git push --force hf hf-deploy:main
+

.gitignore

@@ -0,0 +1,31 @@
+# Environments
+.env
+
+# Model weights (top-level only, NOT the Python model code sub-packages)
+/*/models/
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+
+# Hugging Face cache
+.hf_cache/
+**/.hf_cache/
+
+# OS
+.DS_Store
+
+# Third-party model checkpoints (large weight files)
+**/checkpoints/
+
+# Generated outputs
+**/test/output/
+jobs/
+*.db
+
+# Runtime logs / reports
+**/output_log.md
+**/project_report.md
+walkthrough.md

README.md

@@ -0,0 +1,151 @@
+---
+title: DoppelGen
+emoji: 🎬
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: 6.24.0
+python_version: "3.12"
+app_file: app.py
+pinned: false
+---
+# Talkinghead Orchestrator
+
+**Flora** serves as the centralized master director and orchestration service for the **Creatorium** suite (Voxa, Sonora, Scenea, Actora, Captiona, Ocula, Tempora). Powered by **Google ADK (`google-adk`)** and **Gemini 3.5 Flash**, it unifies deep learning models, computer vision tools, audio processing, NLP sub-services, and social publishing into a single containerized cloud runtime.
+
+## Google Cloud Infrastructure & ADK Multi-Agent Architecture
+
+```mermaid
+graph TD
+    A[User / Single-Page Studio UI] -->|HTTPS POST| B[Google Cloud Run - Flora API Container]
+    B -->|Master Orchestrator| C[Flora ADK Director Agent - Gemini 3.5 Flash]
+  
+    C -->|Persists Job State| D[(Google Cloud Firestore DB)]
+    C -->|Stores Media Assets| E[(Google Cloud Storage GCS Bucket)]
+    C -->|Streams Reasoning Spans| F[OpenTelemetry Telemetry Dashboard]
+  
+    C -->|1. Voxa TTS| G[Speech Waveform & Transcript]
+    C -->|2. Sonora Music| H[Ambient Soundtracks]
+    C -->|3. Scenea B-roll| I[Context B-rolls]
+    C -->|4. Actora LipSync| J[LipSync 1080p Video]
+    C -->|5. Captiona Subtitles| K[Dynamic Styled Subtitles]
+    C -->|6. Tempora Publisher| L[YouTube / Twitter / Playwright CDP Social Media]
+```
+
+## Quick Start — Deploy to Google Cloud Run
+
+Flora is pre-configured for 1-click cloud deployment on **Google Cloud Run** using **Firestore** and **Google Cloud Storage (GCS)**:
+
+```bash
+# Set GCP Project Environment Variables
+export GCP_PROJECT_ID="your-gcp-project-id"
+export GEMINI_API_KEY="your-gemini-api-key"
+
+# Deploy to Google Cloud Run via master runner
+./run_flora.sh deploy
+```
+
+## High-Level Architectural Flow
+
+```mermaid
+graph TD
+    A[Frontend UI] -->|POST Form Data / File paths| B[Flora API]
+    B -->|1. Voxa Runner| C[TTS Waveform & Transcript]
+    B -->|2. Sonora Runner| D[Ambient Soundtracks]
+    B -->|3. Scenea Runner| E[Context B-rolls]
+    B -->|4. Actora Runner| F[LipSync Talking Head]
+    B -->|5. Captiona Runner| G[Composed Subtitled Video]
+    G -->|Success Payload| A
+    B -.->|GET Status Poll| A
+```
+
+## The Creatorium Pipeline
+
+Flora features a unified Web UI (`flora/flora/web`) that serves as a single-page pipeline, sequentially triggering the independent nodes below:
+
+1. **Sonora**
+
+   - **Input:** `[speech.txt] + [optional speech.wav]`
+   - **Process:** Performs semantic similarity analysis on the speech text (and speech.wav if provided) to identify the ideal background context, then anlyze pulled high-quality, royalty-free audio tracks from Mixkit, Freesound, and OpenGameArt for alignment.
+   - **Output:** Background ambient audio track `background.wav`.
+2. **Voxa**
+
+   - **Input:** `[speech.txt] + [sample_audio.wav]`
+   - **Process:** Utilizes advanced ASR and TTS models to clone the provided voice and narrate the speech text.
+   - **Output:** `timestamped_transcript.txt` and `speech.wav` (the cloned narration).
+3. **Scenea**
+
+   - **Input:** `[timestamped_transcript.txt]`
+   - **Process:** Analyzes the transcript to determine which segments require visual enhancement. It fetches relevant, concise B-roll videos (from Pexels or custom sets) tailored perfectly to those specific speech segments (e.g., generating 2 B-rolls if the user specifies a limit of 2).
+   - **Output:** B-roll video assets tightly bound to their transcript timestamps.
+4. **Actora**
+
+   - **Input:** `[background.jpg] + [me.jpg] + [driving_video.mp4] + [speech.wav]`
+   - **Process:** Fuses the assets together, applying human-like mannerisms extracted from the driving video to the static image of "me". The lip-syncing is perfectly matched to `speech.wav`.
+   - **Output:** `talkinghead.mp4` (A complete, high-fidelity talking head video).
+5. **Captiona**
+
+   - **Input:** `[talkinghead.mp4] + [timestamped_transcript.txt]`
+   - **Process:** Overlays dynamic, styled text captions onto the video, perfectly synchronized with the speech and configured to the user's stylistic preferences.
+   - **Output:** `talkinghead_captioned.mp4` (The final, ready-to-publish video).
+
+# Pipeline Parallelization Plan
+
+We will optimize the execution speed of the Flora orchestration pipeline by running independent tasks concurrently.
+
+## Parallel Execution Architecture
+
+Currently, the pipeline runs sequentially:
+
+```
+Voxa (TTS) -> Sonora (Ambient Audio) -> Scenea (B-rolls) -> Actora (Talking Head) -> Captiona (Subtitles)
+```
+
+However, after Voxa runs and produces the `speech.wav` and `speech_transcript.txt` files, the subsequent stages have no data dependencies on each other:
+
+* **Sonora** only depends on `speech_transcript.txt`.
+* **Scenea** only depends on `speech_transcript.txt`.
+* **Actora** only depends on `speech.wav`.
+
+Thus, we can execute Sonora, Scenea, and Actora concurrently using Python's `concurrent.futures.ThreadPoolExecutor`.
+
+```mermaid
+graph TD
+    A[Voxa TTS] --> B[Sonora Ambient Audio]
+    A --> C[Scenea B-rolls]
+    A --> D[Actora Talking Head]
+    B --> E[Captiona Subtitles]
+    C --> E
+    D --> E
+```
+
+## Impact on Execution Time
+
+The total execution time will drop from:
+`Time(Voxa) + Time(Sonora) + Time(Scenea) + Time(Actora) + Time(Captiona)`
+to:
+`Time(Voxa) + max(Time(Sonora), Time(Scenea), Time(Actora)) + Time(Captiona)`
+
+With the host's 32-core CPU capacity, running these three processes simultaneously will not bottleneck local resources, leading to a substantial performance improvement.
+
+## Review
+
+> [!IMPORTANT]
+> Because subprocesses are run concurrently, stdout and stderr logs will write to the container console in an interleaved manner. However, each sub-process will still run as an isolated execution thread and write to its own independent logs if needed. We will update `PIPELINE_STATUS` to show active progress for all running components (e.g. "Generating Video & Fetching Assets...").
+
+## Similar apps:
+
+Flora expects a strict Input/Output contract to seamlessly pass data between the nodes:
+
+- **Talking head gen:**
+- `https://www.veed.io/tools/text-to-speech-avatar/talking-head-video`
+- `https://www.synthesia.io/tools/talking-head-video-maker`
+- `https://toki.ai/ai-talking-avatar`
+- `/https://captions.ai/solutions/talking-head-videos`
+- **Video understanding:**
+- `/https://huggingface.co/openai/clip-vit-base-patch32/tree/main`
+- `/https://huggingface.co/google/siglip-base-patch16-224/tree/main`
+- `/https://huggingface.co/microsoft/xclip-base-patch32/tree/main`
+- `/https://huggingface.co/apple/MobileCLIP2-S3/tree/main`
+
+By treating `flora` as the orchestrator, the entire Creatorium ecosystem operates as a cohesive, highly-optimized production engine.

actora/.dockerignore

@@ -0,0 +1,8 @@
+models/
+.git/
+test/output/
+**/__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.env

actora/.gitignore

@@ -0,0 +1,12 @@
+.env
+__pycache__/
+*.pyc
+.hf_cache/
+# Ignore generated output files but keep the output folder structure
+b_roll_rag/data/output/*
+!b_roll_rag/data/output/.gitkeep
+
+# Ignore local AI agent logs and living documents
+output_log.md
+project_report.md
+git_workflow.md

actora/README.md

@@ -0,0 +1,253 @@
+# ⚡ Edge Talking Head Pipeline (Raspberry Pi Optimized)
+
+An ultra-lightweight, hardware-optimized content creation pipeline designed to run on low-resource edge devices (e.g., Raspberry Pi 4/5). It takes a portrait image, removes the background, composites it onto a beautiful room background, and uses recorded audio to generate a lip-synced talking head video.
+
+---
+
+## Architectural Flow
+```mermaid
+graph TD
+    A[Image / Audio / Driving Video] --> B[EdgePipeline: api.py]
+    B --> C{Talking Engine Model?}
+    C -->|FasterLivePortrait / FOMM / TPSMM| D[Adaptive Keyframe Sampling: skip_pct=0.80]
+    C -->|DreamTalk / Wav2Lip| E[Subsampled Frame Generation: render_every_n=2]
+    D --> F[Linear Interpolation & Composite]
+    E --> F
+    F --> G[Talking Head Output Video]
+```
+
+---
+
+## 📂 Project Directory Structure
+
+```
+actora/
+├── docker/
+│   ├── Dockerfile
+│   ├── docker-compose.yml
+│   ├── entrypoint.sh
+│   └── requirements.txt
+├── models/                     # Auto-created; stores cached ONNX and PyTorch weights
+├── actora_core/           # Core library package
+│   ├── __init__.py
+│   ├── api.py                  # Pipeline orchestration API
+│   ├── base.py                 # Abstract base class definitions
+│   ├── dreamtalk.py            # DreamTalk Audio-driven Expressive Face Generator
+│   ├── dreamtalk_src/          # DreamTalk cloned source repository (auto-cloned)
+│   ├── generator.py            # Lip-sync generation interfaces (Wav2Lip)
+│   ├── matting.py              # Background removal interfaces (MODNet / MediaPipe)
+│   ├── models.py               # Swappable model registry factory
+│   ├── postprocess.py          # Post-processing super-resolution engine (GRL-GAN)
+│   └── utils.py                # Mathematical operations and video encoding helpers
+├── test/
+│   ├── input/                  # Place inputs here (me_1.png, background_1.png, audio_1.wav)
+│   ├── output/                 # Generated outputs will be stored here
+│   ├── test.py                 # Integration testing suite
+│   ├── test_cartoonify.py      # AnimeGAN cartoonification verification test script
+│   └── verify_integrity.py     # Verification utility checking dimensions and frame stats
+├── preload_models.py           # Pre-caches models (checkpoints, Wav2Vec2, ONNX models)
+├── run_actora.sh          # Pipeline container manager wrapper script
+├── project_report.md           # Continuous technical and architectural progress report
+└── README.md                   # This documentation file
+```
+
+---
+
+## 🏗️ Architectural Flow
+
+```
+                                [ INPUT TYPE ]
+                                      │
+           ┌──────────────────────────┴──────────────────────────┐
+           ▼                                                     ▼
+     [ IMAGE INPUT ]                                       [ VIDEO INPUT ]
+           │                                                     │
+ 1. Extract alpha matte once                              1. Read video frames
+ 2. Composite foreground on background once               2. Perform soft-blending 
+ 3. Detect & crop face region (Haar Cascade)                  frame-by-frame (EMA smoothed)
+ 4. Run Expressive Head Generation:                       3. Save composited video
+      a. Generate expressive facial animation sequence
+         using DreamTalk (Diffusion 3DMM + PIRenderer)
+      b. Run optional Wav2Lip ONNX for secondary sync
+      c. Paste crop sequence back onto canvas
+ 5. Mux audio and video stream (FFmpeg)
+```
+
+---
+
+## Technical & Algorithmic Breakdown
+
+1. **Background Matting (Segmentation)**:
+
+   - Uses optimized ONNX runtimes.
+   - Available models: `modnet` (~25 MB, high accuracy) and `mediapipe` (<3 MB, ultra-lightweight and optimized for mobile CPU execution).
+   - Operation: Extract a single-channel alpha matte $A \in [0, 1]^{H \times W}$, then perform alpha compositing:
+     $$
+     I_{out} = I_{fg} \odot A + I_{bg} \odot (1 - A)
+     $$
+2. **Talking Head Generation (Lip Sync & Face Warping)**:
+
+   - **DreamTalk (Diffusion + PIRenderer)**: The default high-fidelity audio-driven talking face generator. It uses a diffusion-based denoising network (DiffusionNet) to generate 3DMM facial motion coefficients from Wav2Vec2 audio embeddings, and a neural PIRenderer to render the generated coefficients onto a static face crop. This produces highly expressive facial animation (gaze shifts, natural expression) from a single photo and an audio file.
+   - **Wav2Lip ONNX + BiomechanicalJoint**: An alternative audio-driven fallback generator. It uses a lightweight Wav2Lip ONNX (~145 MB) model to sync lip movements. To prevent the "lifeless/frozen head" effect typical of Wav2Lip, a physics spring system (`BiomechanicalJoint`) extracts prosodic audio impulses and applies procedural vertical head bobbing to simulate natural human mannerisms.
+   - **FOMM (First-Order Motion Model)**: A video-driven generator. It uses two Qualcomm-exported ONNX models (`detector.onnx` to extract 10 facial keypoints + Jacobians, and `generator.onnx` to warp the source face). Motion is transferred from a driving video using relative keypoint normalization, preserving the source person's identity while applying the driver's head pose, eye blinks, and expressions.
+
+---
+
+## Architectural Flow
+
+```
+[ INPUT CHECK ]
+   │
+   ├──► IF IMAGE + DRIVING VIDEO (video-driven, FOMM):
+   │     1. Extract alpha matte once (MODNet/MediaPipe ONNX)
+   │     2. Composite foreground once onto static background (OpenCV) -> Phase 1 static composite
+   │     3. Save static composite to first_phase_overlay.png
+   │     4. Detect and crop face region from the Phase 1 static composite
+   │     5. Run FOMM: detect source keypoints once; for each driving frame,
+   │        detect driving keypoints, normalize via relative motion transfer,
+   │        run generator to produce animated 256x256 face crop.
+   │     6. Paste generated face crop sequence back onto Phase 1 static composite
+   │     7. Mux final video and input audio (FFmpeg)
+   │
+   ├──► IF IMAGE (audio-driven, DreamTalk):
+   │     1. Extract alpha matte once (MODNet/MediaPipe ONNX)
+   │     2. Composite foreground once onto static background (OpenCV) -> Phase 1 static composite
+   │     3. Save static composite to first_phase_overlay.png
+   │     4. Detect and crop face region from the Phase 1 static composite
+   │     5. Run Selected Generator on the face crop with audio:
+   │         - DreamTalk: Extract Wav2Vec2 audio embeddings, denoise motion coefficients
+   │           via DiffusionNet, render face crops via PIRenderer.
+   │         - Wav2Lip: Run Wav2Lip directly on face crop.
+   │     6. Paste generated/synchronized face crop sequence directly back onto the Phase 1 static composite
+   │     7. Mux final video and input audio (FFmpeg)
+   │
+   └──► IF VIDEO:
+         1. Frame-by-frame background matting (MediaPipe ONNX)
+         2. Soft-blend video frames onto chosen background
+```
+
+---
+
+## 🛠️ Execution & Installation Guide
+
+### 1. Prerequisites
+
+Ensure your edge device has `docker`, `docker-compose`, `python3`, and `pip` installed.
+
+### 2. Model Weight Caching
+
+Run the helper script on the host system to pre-download the optimized model weights, clone the DreamTalk repo, and cache Wav2Vec2 transformers weights:
+
+```bash
+pip install huggingface_hub transformers
+python3 preload_models.py
+```
+
+### 3. Container Lifecycle Commands
+
+Control execution using the centralized wrapper script:
+
+```bash
+# Build the Docker image
+./run_actora.sh build
+
+# Launch the container in the background
+./run_actora.sh up
+
+# Run test execution (processes inputs in test/input/)
+./run_actora.sh test
+
+# Run cartoonification test (processes me_6.jpg through ArcaneGAN and DCT-Net filters)
+docker exec -t actora_edge python3 /app/test/test_cartoonify.py
+
+# Stop container and clean up output folders
+./run_actora.sh down
+./run_actora.sh clean
+```
+
+---
+
+## ⚙️ Configuration & Customization API
+
+The `EdgePipeline` class exposes customization options to control aspect ratio, grading effects, and natural head/body movements:
+
+### 1. Aspect Ratio Canvas Formatting
+
+Define the target output shape of the composition using the `aspect_ratio` parameter:
+
+* `horizontal` (Default: `1024x576`, 16:9 widescreen format)
+* `vertical` (`576x1024`, 9:16 portrait format for social reels/shorts)
+* `square` (`768x768`, 1:1 format)
+
+The system center-crops the background without stretching and auto-scales the foreground subject to occupy ~85% of the frame height, anchoring them at the bottom-center.
+
+### 2. Swappable Generator Engines
+
+Choose the animation quality and execution speed using the `generator_type` parameter in `test/test.py`:
+
+* `dreamtalk` (Default): Expressive, diffusion-based 3DMM talking face generator with natural facial mannerisms.
+* `wav2lip`: Standard audio-driven lip sync with static head posture.
+
+### 3. Cinematic Blending & Effects
+
+Harmonize the composited layers using the `blend_effect` parameter:
+
+* `none`: standard compositing.
+* `cinematic_warm`: warm sunlight grading with enhanced contrast.
+* `b_and_w`: classic high-contrast silver-halide film simulation.
+* `bokeh_blur`: Gaussian blur applied to the background *prior* to compositing, separating the subject with depth-of-field.
+* `arcanegan`: stylized Arcane-like cartoon look using the JIT-compiled ArcaneGANv0.4 model.
+* `dctnet_artstyle`: Artstyle cartoon look using DCT-Net.
+* `dctnet_3d`: 3D cartoon style using DCT-Net.
+* `dctnet_anime`: Anime cartoon style using DCT-Net.
+* **Ambient Color Matcher**: Automatically computes the average color of the background and casts 6% illumination onto the subject for lighting integration.
+
+### 4. Post-Processing & Super-Resolution
+
+Improve facial clarity and eliminate neural morphing/blur artifacts around the talking head's boundary using the `--postprocess` parameter:
+
+* `none` (Default): Bypasses post-processing.
+* `grl_gan`: Applies GRL-GAN ONNX super-resolution upscaling (4x) and downscaling back to 256x256 using Lanczos4, resulting in highly detailed facial features.
+
+---
+
+## 💡 Edge Optimizations & Real-Time Plan
+
+To run this pipeline in real-time on a Raspberry Pi:
+
+1. **MediaPipe Segmentation**: Use the `mediapipe` model key (<3 MB) for matting to reduce compute overhead compared to MODNet (~25 MB).
+2. **Single-Pass Matting**: For static image inputs, perform background matting and compositing *once* instead of frame-by-frame.
+3. **Face Region Localization**: Only run the generator model on a cropped $256 \times 256$ face boundary, then paste it back. This avoids distorting the high-resolution background and significantly reduces processing time.
+4. **Memory / CPU Optimizations**: Load models on CPU with PyTorch CPU-optimized wheels. Monkeypatch `.cuda()` to return CPU tensors to avoid runtime failures without bloated CUDA dependencies.
+
+[github.com/yoyo-nb/Thin-Plate-Spline-Motion-Model](https://github.com/yoyo-nb/Thin-Plate-Spline-Motion-Model)
+
+what our system does:
+
+lipsync: takes a image of me, takes a background image that i desire. puts my segmented cutout on top of the desired background neetly and ensures perfect blending to make a new image. outputs this as this is the first success. then, takes my input audio and process the image to lipsync to the audio and then outputs a video. we have the option to apply visual effects/filters as well as postprocessing for higher quality.
+
+videosync: similar to the above, same image technique and success yardstick. except these models take a video as a driving entity to animate the mannerism so that it is more convincing and then offers a chance to also lypsinc but not required sometimes since the driving video themselves might have contained mouth animations. thereby giving the illusion of speed. this system is therefore matched to our audio and as such outputted. like in the previous options for effects/filters and postprocessing into 4k exists.
+
+lastly, we do:
+
+scenesync: this is essentially us, instead of begining with an image like in the first 2 cases, we have a video of us and perhaps a audio as well, but we simply wanna change the background of our video and blend it appropriately so that it harmonizes well with the new background and audio. then we choose this option, it too has the effect/filters choise as well as post processing options.
+
+here are the desireables:
+
+1. our system is built for edge devices, and as such must priotise cpu only systems. that is, real time operations guarantees and speed comparable to big tech on tiny hardwares.
+2. we do not want a system bloat. hence our models are largely under 300mb individually and as such we do our best to avoid redundancies. perfect object oriented senior level programming is expected. that also means something as simple as investigating which model "/home/azeez/ws/dev_env/py_code/projects/actora/models" no longer referenced or used within our pipeline and safely deleting it.
+
+next, we wanted two things:
+
+1. to tidy up our system and seperate what was thirdparty "/home/azeez/ws/dev_env/py_code/projects/actora/third_party" which would be pulled from the internet on building of our open source talking head project "/home/azeez/ws/dev_env/py_code/projects/actora" and the core files "/home/azeez/ws/dev_env/py_code/projects/actora/actora_core" for orchestration and pipeline.
+2. Optimize our running/algorithmic speed so that processes no longer take an uncomfortable amount of time before getting completed.
+
+in the process of trying to do the above, you have broken things and i require them fixed:
+
+1. the faster_liveportrait no longer works. it no longer cuts out the person, puts them on a new background and animates them. in fact, in its case, we get a video of the desired background but no talking head. just a video of the desired background and the voice overlay. which is sad as we had successfully made this work.
+2. tpsmm and fomm no longer high quality either and rather than a square around the face being animated since its required the detect face to match the driving video face stuff, it appears that we using a face video to animate an entire body hence it morphs in weird shapes and very irregular dimensions are experienced by the human. this was not the case before either.
+
+please i need you to investigate our goals against what we have currently and ensure there is an alignment.
+i need you to only make an implementation plan only after you have completely understood the codebase and the stakes.
+i need you to evaluate the codebase for redundancies and inefficiencies and write a plan to address them and make the code usable to humans.
+fix all errors.

actora/actora/api/__init__.py

@@ -0,0 +1 @@
+from .api import EdgePipeline

actora/actora/api/api.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b86ab223099863e8abc86a088334c99d42fe000444b4f24fd887a8e3142d422
+size 409528

actora/actora/api/main.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1429a26fc2fd3697a372480954e919a895ff01b239502a097c044414a38e0f57
+size 146648

actora/actora/core/__init__.py

@@ -0,0 +1 @@
+from .postprocess import PostProcessEngine

actora/actora/core/arcanegan.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ec6ee10b89ab087180e54099fb7419fb243bde4cb77e0ea9d2f5124e162f70c
+size 153536

actora/actora/core/dctnet.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec7a2b05d11720bd551c780c33fac85361ef96c5310e38cbf1fa8ce21338466
+size 101560

actora/actora/core/filters.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d5be9c0a526deaae8a0ede8219179bdcbb8e8809f9cda37964dbe3444323ca
+size 116976

actora/actora/core/matting.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f15c88a04dce67866e960f4e1c4d3b5840d15d54f5f388ce52f621c65cbb09
+size 106056

actora/actora/core/neural_accelerator.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfcb044b5bf77bed64a62f9d0496ec6d044887c04451e2758dc37f7b5929e36e
+size 136264

actora/actora/core/postprocess.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fa34853533a79334ff8293dd376752b1c309406dd706559b6caf6350e639536
+size 132096

actora/actora/models/__init__.py

@@ -0,0 +1 @@
+from .models import ModelFactory

actora/actora/models/base.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7243b4def147827eb493d29b3244b7c5206974aa3ddbb880e979c81bb111aa88
+size 68120

actora/actora/models/dreamtalk.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:348549ec8077521cb570decb4f7f9065c0a77a6c3b6af22bdd47a9a028adafd1
+size 318032

actora/actora/models/faster_liveportrait.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5bacee2cc64b93262508610f76761c686cfd79a6be7713b15f1c8b6c611271b
+size 247792

actora/actora/models/fomm.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee61a8ca7c7d1c05fe1ecb3c0567ca26f72b23bc251176bce3bd1b17bc702064
+size 229384

actora/actora/models/generator.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5dba13ff15535b66906f1191640c569c17e5e02757e5a743904f21e53399197
+size 193760

actora/actora/models/models.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b45810a7353e02cd2cf23e8c610342db69a829f7a4c56733beacbbca93f6fd4e
+size 67912

actora/actora/models/tpsmm.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b83d2b6f28a3bd6e6b0a4877b6cef65541abda6f6d2dbfefa03d55440e8edd
+size 283848

actora/actora/test/create_test_inputs.py

@@ -0,0 +1,53 @@
+import cv2
+import numpy as np
+import wave
+import struct
+import os
+
+def create_inputs():
+    input_dir = "/app/test/input"
+    os.makedirs(input_dir, exist_ok=True)
+    
+    # 1. Create me.jpg (foreground: simple drawing of a person/face box)
+    me_img = np.zeros((512, 512, 3), dtype=np.uint8) + 240  # Off-white background
+    # Draw head
+    cv2.circle(me_img, (256, 200), 100, (200, 150, 150), -1)
+    # Draw body
+    cv2.rectangle(me_img, (150, 300), (362, 512), (100, 100, 250), -1)
+    # Draw eyes
+    cv2.circle(me_img, (220, 180), 10, (50, 50, 50), -1)
+    cv2.circle(me_img, (292, 180), 10, (50, 50, 50), -1)
+    # Draw mouth
+    cv2.rectangle(me_img, (220, 240), (292, 260), (50, 50, 200), -1)
+    
+    cv2.imwrite(os.path.join(input_dir, "me.jpg"), me_img)
+    print("Created me.jpg")
+    
+    # 2. Create background.png (background space)
+    bg_img = np.zeros((512, 512, 3), dtype=np.uint8)
+    # Draw some grid lines to simulate a living room background
+    for y in range(0, 512, 64):
+        cv2.line(bg_img, (0, y), (512, y), (120, 120, 120), 2)
+    for x in range(0, 512, 64):
+        cv2.line(bg_img, (x, 0), (x, 512), (120, 120, 120), 2)
+    cv2.imwrite(os.path.join(input_dir, "background.png"), bg_img)
+    print("Created background.png")
+    
+    # 3. Create audio.wav (3 seconds of a 440Hz sine wave)
+    sample_rate = 16000
+    duration = 3.0
+    num_samples = int(sample_rate * duration)
+    
+    audio_file = os.path.join(input_dir, "audio.wav")
+    wav_file = wave.open(audio_file, 'w')
+    wav_file.setparams((1, 2, sample_rate, num_samples, 'NONE', 'not compressed'))
+    
+    for i in range(num_samples):
+        value = int(32767.0 * np.sin(2.0 * np.pi * 440.0 * i / sample_rate))
+        data = struct.pack('<h', value)
+        wav_file.writeframesraw(data)
+    wav_file.close()
+    print("Created audio.wav")
+
+if __name__ == "__main__":
+    create_inputs()

actora/actora/test/test.py

@@ -0,0 +1,35 @@
+import sys
+import os
+import time
+# Ensure project root is in PYTHONPATH
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
+import actora.utils.warnings_patch
+
+from actora.api.main import main
+
+if __name__ == "__main__":
+    run_start = time.time()
+
+    # Test by explicitly passing desired arguments
+    test_args = [
+        "--mode", "LipSync", # 'VideoSync', 'LipSync', 'SceneSync'
+        "--img", "/app/actora/test/input/me_6.jpg",
+        "--vid", "/app/actora/test/input/vid_h1.mp4",
+        "--audio", "/app/actora/test/input/audio_1.wav",   
+        "--bg", "/app/actora/test/input/background_n5.png",
+        "--effect", "arcanegan", 
+        "--limit-frames", "150",
+        "--blend-amount", "0",
+        "--generator", "dreamtalk", #(fomm/tpsmm/faster_liveportrait = VideoSync), (wav2lip/dreamtalk = LipSync)
+        "--arcanegan-size", "480",
+        "--postprocess", "grl_gan", # "none,grl_gan,real_esrgan"
+        "--matting-rate", "100"
+    ]
+    main(test_args)
+
+    print(f"\n  [TIMER] Total wall-clock time: {time.time() - run_start:.1f}s")
+
+# Filters: 'none', 'cinematic_warm', 'b_and_w', 'bokeh_blur', 
+#          'arcanegan', 'dctnet_artstyle', 'dctnet_3d', 'dctnet_anime'
+

actora/actora/utils/utils.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0553c81dd4296bed1f5e87f1f3cc7aae5b481ca08b3c3af79320ad6f77fb96a
+size 573664

actora/actora/utils/warnings_patch.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2890d4b1a5b01c48ba7848874ac6a80ebf128213cded923f9a98ce122ac496c6
+size 63192

actora/docker/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6 git && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY docker/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Set HF_HOME so huggingface_hub uses the persistent models cache directory at runtime
+ENV HF_HOME=/app/models/.cache/huggingface
+ENV MODELSCOPE_CACHE=/app/models/.cache/modelscope
+
+ENTRYPOINT ["/entrypoint.sh"]

actora/docker/docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+    edge_processor:
+        build:
+            context: ../
+            dockerfile: docker/Dockerfile
+        container_name: Actora
+        image: actora:latest
+        volumes:
+          - ../actora:/app/actora
+          - ../models:/app/models
+          - ../preload_models.py:/app/preload_models.py
+          - ../third_party:/app/third_party
+        command: tail -f /dev/null

actora/docker/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "Talking Head Edge Container Initialized."
+exec "$@"

actora/docker/requirements.txt

@@ -0,0 +1,27 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+onnxruntime>=1.18.0
+numpy<2.0
+opencv-python-headless
+librosa
+soundfile
+huggingface_hub
+torch
+torchvision
+torchaudio
+transformers
+scipy
+yacs
+tensorflow-cpu==2.14.0
+modelscope>=1.14.0
+addict
+datasets
+oss2
+yapf
+simplejson
+sortedcontainers
+easydict
+torchgeometry
+omegaconf
+munch
+scikit-image
+ffmpeg-python

actora/preload_models.py

@@ -0,0 +1,496 @@
+"""
+TL;DR: Preloads all model weights (ONNX, PyTorch checkpoints, and Transformers cache)
+so the container starts fully self-contained with no runtime downloads required.
+
+Models managed:
+  - modnet.onnx          — Background matting (Hugging Face)
+  - wav2lip.onnx         — Lip sync engine (Hugging Face)
+  - mediapipe.onnx       — Light background matting (Hugging Face)
+  - models/fomm/         — First-Order Motion Model ONNX (Qualcomm S3)
+  - DreamTalk checkpoints— Denoising network + PIRenderer (Hugging Face)
+  - Wav2Vec2             — Transformers audio feature extractor (Hugging Face)
+  - FasterLivePortrait   — 9 ONNX models for video-driven talking head (Hugging Face)
+  - models/animegan/     — AnimeGANv2 and AnimeGANv3 ONNX style-transfer models
+
+Run this once (or on `./run_actora.sh up`) to populate the `models/` directory.
+All checks are local-first: no network call is made if the file already exists.
+"""
+import os
+import sys
+import shutil
+import subprocess
+import urllib.request
+import zipfile
+import asyncio
+import asyncio.base_events
+
+def _patch_asyncio_del():
+    """Monkeypatch BaseEventLoop.__del__ to suppress noisy 'ValueError: Invalid file descriptor: -1' during GC in child preload processes."""
+    _orig_del = asyncio.base_events.BaseEventLoop.__del__
+    def _patched_del(self):
+        try:
+            _orig_del(self)
+        except Exception as e:
+            if "Invalid file descriptor: -1" not in str(e):
+                raise
+    asyncio.base_events.BaseEventLoop.__del__ = _patched_del
+
+_patch_asyncio_del()
+
+def cleanup_event_loops():
+    try:
+        p = asyncio.get_event_loop_policy()
+        if hasattr(p, "_local") and getattr(p._local, "_loop", None) is not None:
+            lp = p._local._loop
+            if lp and not lp.is_running() and not lp.is_closed():
+                lp.close()
+    except Exception:
+        pass
+
+from huggingface_hub import hf_hub_download
+
+
+project_dir = os.path.dirname(os.path.abspath(__file__))
+third_party_dir = os.environ.get("THIRD_PARTY_DIR", os.path.join(project_dir, "third_party"))
+os.makedirs(third_party_dir, exist_ok=True)
+
+
+# ---------------------------------------------------------------------------
+# Hugging Face ONNX manifests (modnet, wav2lip, mediapipe)
+# ---------------------------------------------------------------------------
+MANIFEST = {
+    "modnet":    {"repo": "Xenova/modnet",                            "file": "onnx/model.onnx", "target": "modnet.onnx"},
+    "wav2lip":   {"repo": "bluefoxcreation/Wav2lip-Onnx",            "file": "wav2lip.onnx",    "target": "wav2lip.onnx"},
+    "mediapipe": {"repo": "onnx-community/mediapipe_selfie_segmentation", "file": "onnx/model.onnx", "target": "mediapipe.onnx"},
+}
+
+# ---------------------------------------------------------------------------
+# FOMM — hosted on Qualcomm's public S3 bucket (not Hugging Face)
+# ---------------------------------------------------------------------------
+FOMM_URL      = "https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/models/fomm/releases/v0.56.0/fomm-onnx-float.zip"
+FOMM_SENTINEL = "fomm/generator.onnx"  # Presence of this file means extraction is complete
+
+# ---------------------------------------------------------------------------
+# Thin-Plate Spline Motion Model (TPSMM)
+# ---------------------------------------------------------------------------
+# NOTE: The original vessl/thin-plate-spline-motion-model HF repo is gone (404).
+# AlekseyKorshuk's HF Space mirrors the exact checkpoint (SHA-256 pinned below,
+# verified bit-for-bit identical to the known-good local copy).
+TPSMM_HF_REPO    = "AlekseyKorshuk/thin-plate-spline-motion-model"
+TPSMM_FILE       = "checkpoints/vox.pth.tar"
+TPSMM_SENTINEL   = "tpsmm/vox.pth.tar"
+TPSMM_SHA256     = "52ad8c848e2a1d91b621de96fea83faf57ce3b8c1c06424e317f4df1d3998204"
+
+# ---------------------------------------------------------------------------
+# ArcaneGAN Model Manifest
+# ---------------------------------------------------------------------------
+ARCANEGAN_HF_REPO = "akhaliq/ArcaneGANv0.4"
+ARCANEGAN_FILE    = "ArcaneGANv0.4.jit"
+
+
+
+
+def create_progress_hook(name="Model"):
+    """Creates a simple download progress reporter for urllib.request."""
+    def _hook(block_num, block_size, total_size):
+        if total_size > 0:
+            downloaded = block_num * block_size
+            pct = min(100.0, downloaded / total_size * 100.0)
+            print(f"\r  Downloading {name}... {pct:.1f}%", end="", flush=True)
+    return _hook
+
+
+def sync_cache():
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    os.makedirs(base_dir, exist_ok=True)
+
+    # ------------------------------------------------------------------
+    # 1. Download base ONNX models from Hugging Face
+    # ------------------------------------------------------------------
+    for key, info in MANIFEST.items():
+        target = os.path.join(base_dir, info["target"])
+        if os.path.exists(target):
+            print(f"  [SKIP] {key} already present.")
+            continue
+        print(f"  [SYNC] Downloading {key}...")
+        path = hf_hub_download(
+            repo_id=info["repo"],
+            filename=info["file"],
+            local_dir=base_dir,
+            local_dir_use_symlinks=False
+        )
+        if path != target and os.path.exists(path):
+            shutil.move(path, target)
+
+    # ------------------------------------------------------------------
+    # 2. Download First-Order Motion Model (FOMM) from Qualcomm S3
+    #    Local-first: skip if sentinel file already exists.
+    # ------------------------------------------------------------------
+    fomm_sentinel = os.path.join(base_dir, FOMM_SENTINEL)
+    fomm_dir      = os.path.join(base_dir, "fomm")
+    if os.path.exists(fomm_sentinel):
+        print("  [SKIP] FOMM models already present.")
+    else:
+        print("  [SYNC] Downloading FOMM ONNX package from Qualcomm S3...")
+        zip_path = os.path.join(base_dir, "_fomm_tmp.zip")
+        try:
+            urllib.request.urlretrieve(FOMM_URL, zip_path, reporthook=create_progress_hook("FOMM"))
+            print()  # newline after progress
+            print("  [EXTRACT] Unpacking FOMM...")
+            os.makedirs(fomm_dir, exist_ok=True)
+            with zipfile.ZipFile(zip_path, "r") as zf:
+                for member in zf.infolist():
+                    # Strip the top-level directory from the zip path
+                    parts = member.filename.split("/", 1)
+                    if len(parts) < 2 or not parts[1]:
+                        continue  # skip the root directory entry itself
+                    dest = os.path.join(fomm_dir, parts[1])
+                    if member.is_dir():
+                        os.makedirs(dest, exist_ok=True)
+                    else:
+                        with zf.open(member) as src, open(dest, "wb") as out:
+                            shutil.copyfileobj(src, out)
+            print(f"  [OK] FOMM models extracted to: {fomm_dir}")
+        finally:
+            if os.path.exists(zip_path):
+                os.remove(zip_path)
+
+    # ------------------------------------------------------------------
+    # 3. Clone DreamTalk repository if not already present
+    # ------------------------------------------------------------------
+    dreamtalk_src = os.path.join(third_party_dir, "dreamtalk_src")
+    if not os.path.exists(dreamtalk_src):
+        print("  [SYNC] Cloning DreamTalk repository...")
+        subprocess.run(["git", "clone", "https://github.com/camenduru/dreamtalk.git", dreamtalk_src], check=True)
+    else:
+        print("  [SKIP] DreamTalk repository already cloned.")
+
+    # Mark vendored import roots as regular packages. Upstream has no __init__.py,
+    # so its 'core'/'configs'/'generators' are PEP 420 namespace packages and lose to
+    # identically-named regular packages (e.g. captiona/core, sonora/core) that other
+    # engines place on sys.path. A regular package at the front of sys.path always wins.
+    for _pkg_dir in ("core", "configs", "generators"):
+        _init = os.path.join(dreamtalk_src, _pkg_dir, "__init__.py")
+        if not os.path.exists(_init):
+            with open(_init, "w") as f:
+                f.write("# vendored package marker: prevents namespace-package collision\n")
+
+    # ------------------------------------------------------------------
+    # 4. Download DreamTalk PyTorch checkpoints
+    # ------------------------------------------------------------------
+    checkpoints_dir = os.path.join(dreamtalk_src, "checkpoints")
+    os.makedirs(checkpoints_dir, exist_ok=True)
+
+    dreamtalk_checkpoints = {
+        "denoising_network.pth": "damo/dreamtalk/checkpoints/denoising_network.pth",
+        "renderer.pt":           "damo/dreamtalk/checkpoints/renderer.pt",
+    }
+    for filename, hf_path in dreamtalk_checkpoints.items():
+        target_path = os.path.join(checkpoints_dir, filename)
+        if os.path.exists(target_path):
+            print(f"  [SKIP] DreamTalk checkpoint {filename} already present.")
+            continue
+        print(f"  [SYNC] Downloading DreamTalk checkpoint: {filename}")
+        downloaded = hf_hub_download(
+            repo_id="impactframes/dreamtalk",
+            filename=hf_path,
+            local_dir=base_dir,
+            local_dir_use_symlinks=False
+        )
+        shutil.move(downloaded, target_path)
+
+    # ------------------------------------------------------------------
+    # 4b. Style-clip & pose .mat assets (required by DreamTalkEngine).
+    #     Not part of the upstream git repo — distributed separately by the
+    #     DreamTalk authors. Fetched from a verified public mirror; only the
+    #     two files dreamtalk.py actually references are needed.
+    #     (Uses the module-level `import urllib.request` — a function-local
+    #     import here would shadow it for the whole scope and crash the
+    #     earlier FOMM download with UnboundLocalError.)
+    # ------------------------------------------------------------------
+    _dt_asset_urls = {
+        "data/style_clip/3DMM/M030_front_neutral_level1_001.mat":
+            "https://storage.googleapis.com/falserverless/model_tests/dream_talk/style_clip/3DMM/M030_front_neutral_level1_001.mat",
+        "data/pose/RichardShelby_front_neutral_level1_001.mat":
+            "https://storage.googleapis.com/falserverless/model_tests/dream_talk/pose/RichardShelby_front_neutral_level1_001.mat",
+    }
+    for _rel, _url in _dt_asset_urls.items():
+        _dest = os.path.join(dreamtalk_src, _rel)
+        if os.path.exists(_dest):
+            print(f"  [SKIP] DreamTalk asset {_rel} already present.")
+            continue
+        try:
+            print(f"  [SYNC] Downloading DreamTalk asset: {_rel}")
+            os.makedirs(os.path.dirname(_dest), exist_ok=True)
+            urllib.request.urlretrieve(_url, _dest)
+            print(f"  [OK] Saved to {_dest}")
+        except Exception as e:
+            print(f"  [WARN] Could not download DreamTalk asset {_rel}: {e}")
+
+    # ------------------------------------------------------------------
+    # 5. Preload Wav2Vec2 audio feature extractor (required by DreamTalkEngine)
+    #    Must land in the exact cache layout dreamtalk.py expects:
+    #    <engine_root>/models/.cache/huggingface
+    # ------------------------------------------------------------------
+    print("  [SYNC] Preloading Wav2Vec2 audio feature extractor...")
+    try:
+        from transformers import Wav2Vec2Model, Wav2Vec2Processor
+        _wt_cache = os.path.join(base_dir, ".cache", "huggingface")
+        Wav2Vec2Processor.from_pretrained(
+            "jonatasgrosman/wav2vec2-large-xlsr-53-english", cache_dir=_wt_cache
+        )
+        Wav2Vec2Model.from_pretrained(
+            "jonatasgrosman/wav2vec2-large-xlsr-53-english", cache_dir=_wt_cache
+        )
+        print("  [OK] Wav2Vec2 cached.")
+    except Exception as e:
+        print(f"  [WARN] Could not preload Wav2Vec2 (runtime will retry online): {e}")
+
+    print("\nPre-caching successfully completed.")
+
+
+# Main execution logic moved to bottom
+def sync_tpsmm():
+    """
+    Downloads the TPSMM checkpoint (vox.pth.tar) into models/tpsmm/
+    and clones the Thin-Plate-Spline-Motion-Model source repo into
+    actora/tpsmm_src/.
+    """
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    tpsmm_dir  = os.path.join(base_dir, "tpsmm")
+    os.makedirs(tpsmm_dir, exist_ok=True)
+
+    sentinel = os.path.join(base_dir, TPSMM_SENTINEL)
+    if os.path.exists(sentinel):
+        print("  [SKIP] TPSMM model checkpoint already present.")
+    else:
+        print(f"  [SYNC] Downloading TPSMM model checkpoint from {TPSMM_HF_REPO} (space)...")
+        try:
+            downloaded = hf_hub_download(
+                repo_id=TPSMM_HF_REPO,
+                filename=TPSMM_FILE,
+                repo_type="space",
+                local_dir=tpsmm_dir,
+            )
+            dest = os.path.join(tpsmm_dir, "vox.pth.tar")
+            if downloaded != dest and os.path.exists(downloaded):
+                shutil.move(downloaded, dest)
+            # Integrity check: reject corrupt/incomplete downloads
+            import hashlib
+            h = hashlib.sha256()
+            with open(dest, "rb") as f:
+                for chunk in iter(lambda: f.read(1 << 20), b""):
+                    h.update(chunk)
+            if h.hexdigest() != TPSMM_SHA256:
+                os.remove(dest)
+                raise ValueError(f"SHA-256 mismatch: expected {TPSMM_SHA256}, got {h.hexdigest()}")
+            print(f"  [OK] TPSMM model checkpoint verified and saved to: {dest}")
+        except Exception as e:
+            print(f"  [WARN] Could not download TPSMM checkpoint ({TPSMM_HF_REPO}): {e}")
+
+    # Clone the Thin-Plate Spline Motion Model source repository
+    tpsmm_src = os.path.join(third_party_dir, "tpsmm_src")
+    if os.path.exists(tpsmm_src):
+        print("  [SKIP] TPSMM source repo already cloned.")
+    else:
+        print("  [SYNC] Cloning TPSMM source repo (shallow)...")
+        subprocess.run(
+            ["git", "clone", "--depth", "1",
+             "https://github.com/yoyo-nb/Thin-Plate-Spline-Motion-Model.git", tpsmm_src],
+            check=True
+        )
+
+    # Same namespace-package vaccine as DreamTalk: tpsmm.py imports bare 'modules.*'
+    _modules_init = os.path.join(tpsmm_src, "modules", "__init__.py")
+    if not os.path.exists(_modules_init):
+        with open(_modules_init, "w") as f:
+            f.write("# vendored package marker: prevents namespace-package collision\n")
+
+    print("  [OK] TPSMM setup complete.")
+
+def sync_arcanegan():
+    """
+    Downloads ArcaneGAN v0.4 JIT weights from Hugging Face.
+    """
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    arcanegan_dir = os.path.join(base_dir, "arcanegan")
+    os.makedirs(arcanegan_dir, exist_ok=True)
+    target = os.path.join(arcanegan_dir, ARCANEGAN_FILE)
+    if os.path.exists(target):
+        print(f"  [SKIP] arcanegan/{ARCANEGAN_FILE} already present.")
+        return
+    print(f"  [SYNC] Downloading ArcaneGAN from {ARCANEGAN_HF_REPO}...")
+    downloaded = hf_hub_download(
+        repo_id=ARCANEGAN_HF_REPO,
+        filename=ARCANEGAN_FILE,
+        local_dir=arcanegan_dir,
+        local_dir_use_symlinks=False,
+    )
+    if downloaded != target and os.path.exists(downloaded):
+        shutil.move(downloaded, target)
+    print(f"  [OK]   Saved to {target}")
+
+def sync_dctnet():
+    """
+    Downloads DCT-Net models (Artstyle, 3D, Anime) from ModelScope.
+    """
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    
+    # Check if models are already downloaded to skip network access
+    models_to_check = [
+        "dctnet/artstyle/damo/cv_unet_person-image-cartoon-artstyle_compound-models",
+        "dctnet/3d/damo/cv_unet_person-image-cartoon-3d_compound-models",
+        "dctnet/anime/damo/cv_unet_person-image-cartoon_compound-models"
+    ]
+    
+    all_present = True
+    for path in models_to_check:
+        full_path = os.path.join(base_dir, path)
+        if not os.path.exists(full_path) or len(os.listdir(full_path)) == 0:
+            all_present = False
+            break
+            
+    if all_present:
+        print("  [SKIP] DCT-Net models already present.")
+        return
+
+    try:
+        from modelscope.hub.snapshot_download import snapshot_download
+        print("\nPreloading DCT-Net Models (ModelScope)...")
+        # Artstyle
+        snapshot_download('damo/cv_unet_person-image-cartoon-artstyle_compound-models', cache_dir=os.path.join(base_dir, 'dctnet', 'artstyle'))
+        # 3D
+        snapshot_download('damo/cv_unet_person-image-cartoon-3d_compound-models', cache_dir=os.path.join(base_dir, 'dctnet', '3d'))
+        # Anime
+        snapshot_download('damo/cv_unet_person-image-cartoon_compound-models', cache_dir=os.path.join(base_dir, 'dctnet', 'anime'))
+        print("  [OK] DCT-Net Models downloaded.")
+    except ImportError as e:
+        print(f"  [WARNING] Could not import modelscope. Skipping DCT-Net download: {e}")
+
+
+def sync_faster_liveportrait():
+    """
+    Downloads the FasterLivePortrait checkpoints and clones the repo.
+    """
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    flip_dir  = os.path.join(base_dir, "faster_liveportrait")
+    os.makedirs(flip_dir, exist_ok=True)
+
+    # Clone the FasterLivePortrait source repository
+    flip_src = os.path.join(third_party_dir, "faster_liveportrait_src")
+    if os.path.exists(flip_src):
+        print("  [SKIP] FasterLivePortrait source repo already cloned.")
+    else:
+        print("  [SYNC] Cloning FasterLivePortrait source repo (shallow)...")
+        subprocess.run(
+            ["git", "clone", "--depth", "1",
+             "https://github.com/warmshao/FasterLivePortrait.git", flip_src],
+            check=True
+        )
+
+    # Download human ONNX weights
+    onnx_files = [
+        "appearance_feature_extractor.onnx",
+        "face_2dpose_106_static.onnx",
+        "landmark.onnx",
+        "motion_extractor.onnx",
+        "retinaface_det_static.onnx",
+        "stitching.onnx",
+        "stitching_eye.onnx",
+        "stitching_lip.onnx",
+        "warping_spade.onnx"
+    ]
+    
+    target_dir = os.path.join(flip_dir, "liveportrait_onnx")
+    os.makedirs(target_dir, exist_ok=True)
+    
+    files_to_download = [f for f in onnx_files if not os.path.exists(os.path.join(target_dir, f))]
+    
+    if not files_to_download:
+        print("  [SKIP] FasterLivePortrait ONNX weights already present.")
+    else:
+        print(f"  [SYNC] Downloading {len(files_to_download)} missing FasterLivePortrait ONNX weights...")
+        for f in files_to_download:
+            print(f"    [SYNC] Downloading {f}...")
+            downloaded = hf_hub_download(
+                repo_id="warmshao/FasterLivePortrait",
+                filename=f"liveportrait_onnx/{f}",
+                local_dir=flip_dir,
+                local_dir_use_symlinks=False,
+            )
+            dest = os.path.join(target_dir, f)
+            if downloaded != dest and os.path.exists(downloaded):
+                shutil.move(downloaded, dest)
+            
+    print("  [OK] FasterLivePortrait setup complete.")
+
+
+
+
+def sync_postprocess():
+    """
+    Downloads post-processing super-resolution models (GRL-GAN and Real-ESRGAN).
+    """
+    base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
+    postprocess_dir = os.path.join(base_dir, "postprocess")
+    os.makedirs(postprocess_dir, exist_ok=True)
+    
+    # 1. GRL-GAN
+    grl_gan_target = os.path.join(postprocess_dir, "grl_gan.onnx")
+    if os.path.exists(grl_gan_target):
+        print("  [SKIP] postprocess/grl_gan.onnx already present.")
+    else:
+        print("  [SYNC] Downloading 4x_APISR_GRL_GAN_generator-onnx...")
+        downloaded = hf_hub_download(
+            repo_id="Xenova/4x_APISR_GRL_GAN_generator-onnx",
+            filename="onnx/model.onnx",
+            local_dir=postprocess_dir,
+            local_dir_use_symlinks=False,
+        )
+        if downloaded != grl_gan_target and os.path.exists(downloaded):
+            shutil.move(downloaded, grl_gan_target)
+        onnx_subdir = os.path.join(postprocess_dir, "onnx")
+        if os.path.exists(onnx_subdir):
+            shutil.rmtree(onnx_subdir)
+        print(f"  [OK]   Saved to {grl_gan_target}")
+
+    # 2. Real-ESRGAN
+    real_esrgan_url = "https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/models/real_esrgan_x4plus/releases/v0.57.0/real_esrgan_x4plus-onnx-float.zip"
+    real_esrgan_dir = os.path.join(postprocess_dir, "real_esrgan")
+    real_esrgan_sentinel = os.path.join(real_esrgan_dir, "real_esrgan_x4plus.onnx")
+    
+    if os.path.exists(real_esrgan_sentinel):
+        print("  [SKIP] postprocess/real_esrgan already present.")
+    else:
+        print("  [SYNC] Downloading Real-ESRGAN ONNX package from Qualcomm S3...")
+        zip_path = os.path.join(postprocess_dir, "_realesrgan_tmp.zip")
+        try:
+            urllib.request.urlretrieve(real_esrgan_url, zip_path, reporthook=create_progress_hook("Real-ESRGAN"))
+            print()
+            print("  [EXTRACT] Unpacking Real-ESRGAN...")
+            os.makedirs(real_esrgan_dir, exist_ok=True)
+            with zipfile.ZipFile(zip_path, "r") as zf:
+                for member in zf.infolist():
+                    parts = member.filename.split("/", 1)
+                    if len(parts) < 2 or not parts[1]:
+                        continue
+                    dest = os.path.join(real_esrgan_dir, parts[1])
+                    if member.is_dir():
+                        os.makedirs(dest, exist_ok=True)
+                    else:
+                        with zf.open(member) as src, open(dest, "wb") as out:
+                            shutil.copyfileobj(src, out)
+            print(f"  [OK] Real-ESRGAN extracted to: {real_esrgan_dir}")
+        finally:
+            if os.path.exists(zip_path):
+                os.remove(zip_path)
+
+
+if __name__ == "__main__":
+    sync_cache()
+    sync_tpsmm()
+    sync_faster_liveportrait()
+    sync_arcanegan()
+    sync_dctnet()
+    sync_postprocess()
+

actora/run_actora.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# TL;DR: Docker lifecycle management script for the Talking Head edge pipeline.
+
+#!/bin/bash
+case "$1" in
+    build)
+        docker compose -f docker/docker-compose.yml build
+        ;;
+    up)
+        docker compose -f docker/docker-compose.yml up -d
+        docker exec -t Actora python /app/preload_models.py
+        ;;
+    down)
+        docker compose -f docker/docker-compose.yml down
+        ;;
+    clean)
+        docker system prune -f
+        docker exec -t Actora chown -R $(id -u):$(id -g) /app 2>/dev/null || true
+        rm -rf actora/actora/test/output/*
+        ;;
+    test)
+        docker exec -it Actora python /app/actora/test/test.py
+        docker exec -t Actora chown -R $(id -u):$(id -g) /app 2>/dev/null || true
+        ;;
+    *)
+        echo "Usage: $0 {build|up|down|clean|test}"
+        exit 1
+esac

actora/third_party/dreamtalk_src/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Alibaba TongYi Vision Intelligence Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

actora/third_party/dreamtalk_src/README.md

@@ -0,0 +1,99 @@
+<h2 align="center">DreamTalk: When Expressive Talking Head Generation <br> Meets Diffusion Probabilistic Models</h2>
+<p align="center">
+  <a href='https://dreamtalk-project.github.io/'><img src='https://img.shields.io/badge/Project-Page-Green'></a> <a href='https://arxiv.org/abs/2312.09767'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a> <a href='https://youtu.be/VF4vlE6ZqWQ'><img src='https://badges.aleen42.com/src/youtube.svg'></a>
+</p>
+
+![teaser](media/teaser.gif "teaser")
+
+DreamTalk is a diffusion-based audio-driven expressive talking head generation framework that can produce high-quality talking head videos across diverse speaking styles. DreamTalk exhibits robust performance with a diverse array of inputs, including songs, speech in multiple languages, noisy audio, and out-of-domain portraits.
+
+## News
+- __[2023.12]__ Release inference code and pretrained checkpoint.
+
+## Installation
+
+```
+conda create -n dreamtalk python=3.7.0
+conda activate dreamtalk
+pip install -r requirements.txt
+conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=11.1 -c pytorch -c conda-forge
+conda update ffmpeg
+
+pip install urllib3==1.26.6
+pip install transformers==4.28.1
+pip install dlib
+```
+
+## Download Checkpoints
+Download the checkpoint of the denoising network and the renderer:
+* [HuggingFace](https://huggingface.co/damo-vilab/dreamtalk)
+* [ModelScope](https://modelscope.cn/models/damo/dreamtalk/files) (in `checkpoints` folder)
+
+
+Put the downloaded checkpoints into `checkpoints` folder.
+
+
+## Inference
+Run the script:
+
+```
+python inference_for_demo_video.py \
+--wav_path data/audio/acknowledgement_english.m4a \
+--style_clip_path data/style_clip/3DMM/M030_front_neutral_level1_001.mat \
+--pose_path data/pose/RichardShelby_front_neutral_level1_001.mat \
+--image_path data/src_img/uncropped/male_face.png \
+--cfg_scale 1.0 \
+--max_gen_len 30 \
+--output_name acknowledgement_english@M030_front_neutral_level1_001@male_face
+```
+
+`wav_path` specifies the input audio. The input audio file extensions such as wav, mp3, m4a, and mp4 (video with sound) should all be compatible.
+
+`style_clip_path` specifies the reference speaking style and `pose_path` specifies head pose. They are 3DMM paramenter sequences extracted from reference videos. You can follow [PIRenderer](https://github.com/RenYurui/PIRender) to extract 3DMM parameters from your own videos. Note that the video frame rate should be 25 FPS. Besides, videos used for head pose reference should be first cropped to $256\times256$ using scripts in [FOMM video preprocessing](https://github.com/AliaksandrSiarohin/video-preprocessing).
+
+`image_path` specifies the input portrait. Its resolution should be larger than $256\times256$. Frontal portraits, with the face directly facing forward and not tilted to one side, usually achieve satisfactory results. The input portrait will be cropped to $256\times256$. If your portrait is already cropped to $256\times256$ and you want to disable cropping, use option `--disable_img_crop` like this:
+
+```
+python inference_for_demo_video.py \
+--wav_path data/audio/acknowledgement_chinese.m4a \
+--style_clip_path data/style_clip/3DMM/M030_front_surprised_level3_001.mat \
+--pose_path data/pose/RichardShelby_front_neutral_level1_001.mat \
+--image_path data/src_img/cropped/zp1.png \
+--disable_img_crop \
+--cfg_scale 1.0 \
+--max_gen_len 30 \
+--output_name acknowledgement_chinese@M030_front_surprised_level3_001@zp1
+```
+
+`cfg_scale` controls the scale of classifer-free guidance. It can adjust the intensity of speaking styles.
+
+`max_gen_len` is the maximum video generation duration, measured in seconds. If the input audio exceeds this length, it will be truncated.
+
+The generated video will be named `$(output_name).mp4` and put in the output_video folder. Intermediate results, including the cropped portrait, will be in the `tmp/$(output_name)` folder.
+
+Sample inputs are presented in `data` folder. Due to copyright issues, we are unable to include the songs we have used in this folder.
+
+
+## Acknowledgements
+
+We extend our heartfelt thanks for the invaluable contributions made by preceding works to the development of DreamTalk. This includes, but is not limited to:
+[PIRenderer](https://github.com/RenYurui/PIRender)
+,[AVCT](https://github.com/FuxiVirtualHuman/AAAI22-one-shot-talking-face)
+,[StyleTalk](https://github.com/FuxiVirtualHuman/styletalk)
+,[Deep3DFaceRecon_pytorch](https://github.com/sicxu/Deep3DFaceRecon_pytorch)
+,[Wav2vec2.0](https://huggingface.co/jonatasgrosman/wav2vec2-large-xlsr-53-english)
+,[diffusion-point-cloud](https://github.com/luost26/diffusion-point-cloud)
+,[FOMM video preprocessing](https://github.com/AliaksandrSiarohin/video-preprocessing). We are dedicated to advancing upon these foundational works with the utmost respect for their original contributions.
+
+## Citation
+If you find this codebase useful for your research, please use the following entry.
+```BibTeX
+@article{ma2023dreamtalk,
+  title={DreamTalk: When Expressive Talking Head Generation Meets Diffusion Probabilistic Models},
+  author={Ma, Yifeng and Zhang, Shiwei and Wang, Jiayu and Wang, Xiang and Zhang, Yingya and Deng, Zhidong},
+  journal={arXiv preprint arXiv:2312.09767},
+  year={2023}
+}
+```
+
+

actora/third_party/dreamtalk_src/configs/default.py

@@ -0,0 +1,91 @@
+from yacs.config import CfgNode as CN
+
+
+_C = CN()
+_C.TAG = "style_id_emotion"
+_C.DECODER_TYPE = "DisentangleDecoder"
+_C.CONTENT_ENCODER_TYPE = "ContentW2VEncoder"
+_C.STYLE_ENCODER_TYPE = "StyleEncoder"
+
+_C.DIFFNET_TYPE = "DiffusionNet"
+
+_C.WIN_SIZE = 5
+_C.D_MODEL = 256
+
+_C.DATASET = CN()
+_C.DATASET.FACE3D_DIM = 64
+_C.DATASET.NUM_FRAMES = 64
+_C.DATASET.STYLE_MAX_LEN = 256
+
+_C.TRAIN = CN()
+_C.TRAIN.FACE3D_LATENT = CN()
+_C.TRAIN.FACE3D_LATENT.TYPE = "face3d"
+
+_C.DIFFUSION = CN()
+_C.DIFFUSION.PREDICT_WHAT = "x0"  # noise | x0
+_C.DIFFUSION.SCHEDULE = CN()
+_C.DIFFUSION.SCHEDULE.NUM_STEPS = 1000
+_C.DIFFUSION.SCHEDULE.BETA_1 = 1e-4
+_C.DIFFUSION.SCHEDULE.BETA_T = 0.02
+_C.DIFFUSION.SCHEDULE.MODE = "linear"
+
+_C.CONTENT_ENCODER = CN()
+_C.CONTENT_ENCODER.d_model = _C.D_MODEL
+_C.CONTENT_ENCODER.nhead = 8
+_C.CONTENT_ENCODER.num_encoder_layers = 3
+_C.CONTENT_ENCODER.dim_feedforward = 4 * _C.D_MODEL
+_C.CONTENT_ENCODER.dropout = 0.1
+_C.CONTENT_ENCODER.activation = "relu"
+_C.CONTENT_ENCODER.normalize_before = False
+_C.CONTENT_ENCODER.pos_embed_len = 2 * _C.WIN_SIZE + 1
+
+_C.STYLE_ENCODER = CN()
+_C.STYLE_ENCODER.d_model = _C.D_MODEL
+_C.STYLE_ENCODER.nhead = 8
+_C.STYLE_ENCODER.num_encoder_layers = 3
+_C.STYLE_ENCODER.dim_feedforward = 4 * _C.D_MODEL
+_C.STYLE_ENCODER.dropout = 0.1
+_C.STYLE_ENCODER.activation = "relu"
+_C.STYLE_ENCODER.normalize_before = False
+_C.STYLE_ENCODER.pos_embed_len = _C.DATASET.STYLE_MAX_LEN
+_C.STYLE_ENCODER.aggregate_method = (
+    "self_attention_pooling"  # average | self_attention_pooling
+)
+# _C.STYLE_ENCODER.input_dim = _C.DATASET.FACE3D_DIM
+
+_C.DECODER = CN()
+_C.DECODER.d_model = _C.D_MODEL
+_C.DECODER.nhead = 8
+_C.DECODER.num_decoder_layers = 3
+_C.DECODER.dim_feedforward = 4 * _C.D_MODEL
+_C.DECODER.dropout = 0.1
+_C.DECODER.activation = "relu"
+_C.DECODER.normalize_before = False
+_C.DECODER.return_intermediate_dec = False
+_C.DECODER.pos_embed_len = 2 * _C.WIN_SIZE + 1
+_C.DECODER.network_type = "TransformerDecoder"
+_C.DECODER.dynamic_K = None
+_C.DECODER.dynamic_ratio = None
+# _C.DECODER.output_dim = _C.DATASET.FACE3D_DIM
+# LSFM basis:
+# _C.DECODER.upper_face3d_indices = tuple(list(range(19)) + list(range(46, 51)))
+# _C.DECODER.lower_face3d_indices = tuple(range(19, 46))
+# BFM basis:
+# fmt: off
+_C.DECODER.upper_face3d_indices = [6, 8, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63] 
+# fmt: on
+_C.DECODER.lower_face3d_indices = [0, 1, 2, 3, 4, 5, 7, 9, 10, 11, 12, 13, 14]
+
+_C.CF_GUIDANCE = CN()
+_C.CF_GUIDANCE.TRAINING = True
+_C.CF_GUIDANCE.INFERENCE = True
+_C.CF_GUIDANCE.NULL_PROB = 0.1
+_C.CF_GUIDANCE.SCALE = 1.0
+
+_C.INFERENCE = CN()
+_C.INFERENCE.CHECKPOINT = "checkpoints/denoising_network.pth"
+
+
+def get_cfg_defaults():
+    """Get a yacs CfgNode object with default values for my_project."""
+    return _C.clone()

actora/third_party/dreamtalk_src/core/networks/__init__.py

@@ -0,0 +1,14 @@
+from core.networks.generator import (
+    StyleEncoder,
+    Decoder,
+    ContentW2VEncoder,
+)
+from core.networks.disentangle_decoder import DisentangleDecoder
+
+
+def get_network(name: str):
+    obj = globals().get(name)
+    if obj is None:
+        raise KeyError("Unknown Network: %s" % name)
+    else:
+        return obj

actora/third_party/dreamtalk_src/core/networks/diffusion_net.py

@@ -0,0 +1,340 @@
+import math
+import torch
+import torch.nn.functional as F
+from torch.nn import Module
+from core.networks.diffusion_util import VarianceSchedule
+import numpy as np
+
+
+def face3d_raw_to_norm(face3d_raw, exp_min, exp_max):
+    """
+
+    Args:
+        face3d_raw (_type_): (B, L, C_face3d)
+        exp_min (_type_): (C_face3d)
+        exp_max (_type_): (C_face3d)
+
+    Returns:
+        _type_: (B, L, C_face3d) in [-1, 1]
+    """
+    exp_min_expand = exp_min[None, None, :]
+    exp_max_expand = exp_max[None, None, :]
+    face3d_norm_01 = (face3d_raw - exp_min_expand) / (exp_max_expand - exp_min_expand)
+    face3d_norm = face3d_norm_01 * 2 - 1
+    return face3d_norm
+
+
+def face3d_norm_to_raw(face3d_norm, exp_min, exp_max):
+    """
+
+    Args:
+        face3d_norm (_type_): (B, L, C_face3d)
+        exp_min (_type_): (C_face3d)
+        exp_max (_type_): (C_face3d)
+
+    Returns:
+        _type_: (B, L, C_face3d)
+    """
+    exp_min_expand = exp_min[None, None, :]
+    exp_max_expand = exp_max[None, None, :]
+    face3d_norm_01 = (face3d_norm + 1) / 2
+    face3d_raw = face3d_norm_01 * (exp_max_expand - exp_min_expand) + exp_min_expand
+    return face3d_raw
+
+
+class DiffusionNet(Module):
+    def __init__(self, cfg, net, var_sched: VarianceSchedule):
+        super().__init__()
+        self.cfg = cfg
+        self.net = net
+        self.var_sched = var_sched
+        self.face3d_latent_type = self.cfg.TRAIN.FACE3D_LATENT.TYPE
+        self.predict_what = self.cfg.DIFFUSION.PREDICT_WHAT
+
+        if self.cfg.CF_GUIDANCE.TRAINING:
+            null_style_clip = torch.zeros(
+                self.cfg.DATASET.STYLE_MAX_LEN, self.cfg.DATASET.FACE3D_DIM
+            )
+            self.register_buffer("null_style_clip", null_style_clip)
+
+            null_pad_mask = torch.tensor([False] * self.cfg.DATASET.STYLE_MAX_LEN)
+            self.register_buffer("null_pad_mask", null_pad_mask)
+
+    def _face3d_to_latent(self, face3d):
+        latent = None
+        if self.face3d_latent_type == "face3d":
+            latent = face3d
+        elif self.face3d_latent_type == "normalized_face3d":
+            latent = face3d_raw_to_norm(
+                face3d, exp_min=self.exp_min, exp_max=self.exp_max
+            )
+        else:
+            raise ValueError(f"Invalid face3d latent type: {self.face3d_latent_type}")
+        return latent
+
+    def _latent_to_face3d(self, latent):
+        face3d = None
+        if self.face3d_latent_type == "face3d":
+            face3d = latent
+        elif self.face3d_latent_type == "normalized_face3d":
+            latent = torch.clamp(latent, min=-1, max=1)
+            face3d = face3d_norm_to_raw(
+                latent, exp_min=self.exp_min, exp_max=self.exp_max
+            )
+        else:
+            raise ValueError(f"Invalid face3d latent type: {self.face3d_latent_type}")
+        return face3d
+
+    def ddim_sample(
+        self,
+        audio,
+        style_clip,
+        style_pad_mask,
+        output_dim,
+        flexibility=0.0,
+        ret_traj=False,
+        use_cf_guidance=False,
+        cfg_scale=2.0,
+        ddim_num_step=50,
+        ready_style_code=None,
+    ):
+        """
+
+        Args:
+            audio (_type_): (B, L, W) or (B, L, W, C)
+            style_clip (_type_): (B, L_clipmax, C_face3d)
+            style_pad_mask : (B, L_clipmax)
+            pose_dim (_type_): int
+            flexibility (float, optional): _description_. Defaults to 0.0.
+            ret_traj (bool, optional): _description_. Defaults to False.
+
+
+        Returns:
+            _type_: (B, L, C_face)
+        """
+        if self.predict_what != "x0":
+            raise NotImplementedError(self.predict_what)
+
+        if ready_style_code is not None and use_cf_guidance:
+            raise NotImplementedError("not implement cfg for ready style code")
+
+        c = self.var_sched.num_steps // ddim_num_step
+        time_steps = torch.tensor(
+            np.asarray(list(range(0, self.var_sched.num_steps, c))) + 1
+        )
+        assert len(time_steps) == ddim_num_step
+        prev_time_steps = torch.cat((torch.tensor([0]), time_steps[:-1]))
+
+        batch_size, output_len = audio.shape[:2]
+        # batch_size = context.size(0)
+        context = {
+            "audio": audio,
+            "style_clip": style_clip,
+            "style_pad_mask": style_pad_mask,
+            "ready_style_code": ready_style_code,
+        }
+        if use_cf_guidance:
+            uncond_style_clip = self.null_style_clip.unsqueeze(0).repeat(
+                batch_size, 1, 1
+            )
+            uncond_pad_mask = self.null_pad_mask.unsqueeze(0).repeat(batch_size, 1)
+
+            context_double = {
+                "audio": torch.cat([audio] * 2, dim=0),
+                "style_clip": torch.cat([style_clip, uncond_style_clip], dim=0),
+                "style_pad_mask": torch.cat([style_pad_mask, uncond_pad_mask], dim=0),
+                "ready_style_code": None
+                if ready_style_code is None
+                else torch.cat(
+                    [
+                        ready_style_code,
+                        self.net.style_encoder(uncond_style_clip, uncond_pad_mask),
+                    ],
+                    dim=0,
+                ),
+            }
+
+        x_t = torch.randn([batch_size, output_len, output_dim]).to(audio.device)
+
+        for idx in list(range(ddim_num_step))[::-1]:
+            t = time_steps[idx]
+            t_prev = prev_time_steps[idx]
+            ddim_alpha = self.var_sched.alpha_bars[t]
+            ddim_alpha_prev = self.var_sched.alpha_bars[t_prev]
+
+            t_tensor = torch.tensor([t] * batch_size).to(audio.device).float()
+            if use_cf_guidance:
+                x_t_double = torch.cat([x_t] * 2, dim=0)
+                t_tensor_double = torch.cat([t_tensor] * 2, dim=0)
+                cond_output, uncond_output = self.net(
+                    x_t_double, t=t_tensor_double, **context_double
+                ).chunk(2)
+                diff_output = uncond_output + cfg_scale * (cond_output - uncond_output)
+            else:
+                diff_output = self.net(x_t, t=t_tensor, **context)
+
+            pred_x0 = diff_output
+            eps = (x_t - torch.sqrt(ddim_alpha) * pred_x0) / torch.sqrt(1 - ddim_alpha)
+            c1 = torch.sqrt(ddim_alpha_prev)
+            c2 = torch.sqrt(1 - ddim_alpha_prev)
+
+            x_t = c1 * pred_x0 + c2 * eps
+
+        latent_output = x_t
+        face3d_output = self._latent_to_face3d(latent_output)
+        return face3d_output
+
+    def sample(
+        self,
+        audio,
+        style_clip,
+        style_pad_mask,
+        output_dim,
+        flexibility=0.0,
+        ret_traj=False,
+        use_cf_guidance=False,
+        cfg_scale=2.0,
+        sample_method="ddpm",
+        ddim_num_step=50,
+        ready_style_code=None,
+    ):
+        # sample_method = kwargs["sample_method"]
+        if sample_method == "ddpm":
+            if ready_style_code is not None:
+                raise NotImplementedError("ready style code in ddpm")
+            return self.ddpm_sample(
+                audio,
+                style_clip,
+                style_pad_mask,
+                output_dim,
+                flexibility=flexibility,
+                ret_traj=ret_traj,
+                use_cf_guidance=use_cf_guidance,
+                cfg_scale=cfg_scale,
+            )
+        elif sample_method == "ddim":
+            return self.ddim_sample(
+                audio,
+                style_clip,
+                style_pad_mask,
+                output_dim,
+                flexibility=flexibility,
+                ret_traj=ret_traj,
+                use_cf_guidance=use_cf_guidance,
+                cfg_scale=cfg_scale,
+                ddim_num_step=ddim_num_step,
+                ready_style_code=ready_style_code,
+            )
+
+    def ddpm_sample(
+        self,
+        audio,
+        style_clip,
+        style_pad_mask,
+        output_dim,
+        flexibility=0.0,
+        ret_traj=False,
+        use_cf_guidance=False,
+        cfg_scale=2.0,
+    ):
+        """
+
+        Args:
+            audio (_type_): (B, L, W) or (B, L, W, C)
+            style_clip (_type_): (B, L_clipmax, C_face3d)
+            style_pad_mask : (B, L_clipmax)
+            pose_dim (_type_): int
+            flexibility (float, optional): _description_. Defaults to 0.0.
+            ret_traj (bool, optional): _description_. Defaults to False.
+
+
+        Returns:
+            _type_: (B, L, C_face)
+        """
+        batch_size, output_len = audio.shape[:2]
+        # batch_size = context.size(0)
+        context = {
+            "audio": audio,
+            "style_clip": style_clip,
+            "style_pad_mask": style_pad_mask,
+        }
+        if use_cf_guidance:
+            uncond_style_clip = self.null_style_clip.unsqueeze(0).repeat(
+                batch_size, 1, 1
+            )
+            uncond_pad_mask = self.null_pad_mask.unsqueeze(0).repeat(batch_size, 1)
+            context_double = {
+                "audio": torch.cat([audio] * 2, dim=0),
+                "style_clip": torch.cat([style_clip, uncond_style_clip], dim=0),
+                "style_pad_mask": torch.cat([style_pad_mask, uncond_pad_mask], dim=0),
+            }
+
+        x_T = torch.randn([batch_size, output_len, output_dim]).to(audio.device)
+        traj = {self.var_sched.num_steps: x_T}
+        for t in range(self.var_sched.num_steps, 0, -1):
+            alpha = self.var_sched.alphas[t]
+            alpha_bar = self.var_sched.alpha_bars[t]
+            alpha_bar_prev = self.var_sched.alpha_bars[t - 1]
+            sigma = self.var_sched.get_sigmas(t, flexibility)
+
+            z = torch.randn_like(x_T) if t > 1 else torch.zeros_like(x_T)
+            x_t = traj[t]
+            t_tensor = torch.tensor([t] * batch_size).to(audio.device).float()
+            if use_cf_guidance:
+                x_t_double = torch.cat([x_t] * 2, dim=0)
+                t_tensor_double = torch.cat([t_tensor] * 2, dim=0)
+                cond_output, uncond_output = self.net(
+                    x_t_double, t=t_tensor_double, **context_double
+                ).chunk(2)
+                diff_output = uncond_output + cfg_scale * (cond_output - uncond_output)
+            else:
+                diff_output = self.net(x_t, t=t_tensor, **context)
+
+            if self.predict_what == "noise":
+                c0 = 1.0 / torch.sqrt(alpha)
+                c1 = (1 - alpha) / torch.sqrt(1 - alpha_bar)
+                x_next = c0 * (x_t - c1 * diff_output) + sigma * z
+            elif self.predict_what == "x0":
+                d0 = torch.sqrt(alpha) * (1 - alpha_bar_prev) / (1 - alpha_bar)
+                d1 = torch.sqrt(alpha_bar_prev) * (1 - alpha) / (1 - alpha_bar)
+                x_next = d0 * x_t + d1 * diff_output + sigma * z
+            traj[t - 1] = x_next.detach()
+            traj[t] = traj[t].cpu()
+            if not ret_traj:
+                del traj[t]
+
+        if ret_traj:
+            raise NotImplementedError
+            return traj
+        else:
+            latent_output = traj[0]
+            face3d_output = self._latent_to_face3d(latent_output)
+            return face3d_output
+
+
+if __name__ == "__main__":
+    from core.networks.diffusion_util import NoisePredictor, VarianceSchedule
+
+    diffnet = DiffusionNet(
+        net=NoisePredictor(),
+        var_sched=VarianceSchedule(
+            num_steps=500, beta_1=1e-4, beta_T=0.02, mode="linear"
+        ),
+    )
+
+    import torch
+
+    gt_face3d = torch.randn(16, 64, 64)
+    audio = torch.randn(16, 64, 11)
+    style_clip = torch.randn(16, 256, 64)
+    style_pad_mask = torch.ones(16, 256)
+
+    context = {
+        "audio": audio,
+        "style_clip": style_clip,
+        "style_pad_mask": style_pad_mask,
+    }
+
+    loss = diffnet.get_loss(gt_face3d, context)
+
+    print("hello")

actora/third_party/dreamtalk_src/core/networks/diffusion_util.py

@@ -0,0 +1,131 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.nn import Module
+from core.networks import get_network
+from core.utils import sinusoidal_embedding
+
+
+class VarianceSchedule(Module):
+    def __init__(self, num_steps, beta_1, beta_T, mode="linear"):
+        super().__init__()
+        assert mode in ("linear",)
+        self.num_steps = num_steps
+        self.beta_1 = beta_1
+        self.beta_T = beta_T
+        self.mode = mode
+
+        if mode == "linear":
+            betas = torch.linspace(beta_1, beta_T, steps=num_steps)
+
+        betas = torch.cat([torch.zeros([1]), betas], dim=0)  # Padding
+
+        alphas = 1 - betas
+        log_alphas = torch.log(alphas)
+        for i in range(1, log_alphas.size(0)):  # 1 to T
+            log_alphas[i] += log_alphas[i - 1]
+        alpha_bars = log_alphas.exp()
+
+        sigmas_flex = torch.sqrt(betas)
+        sigmas_inflex = torch.zeros_like(sigmas_flex)
+        for i in range(1, sigmas_flex.size(0)):
+            sigmas_inflex[i] = ((1 - alpha_bars[i - 1]) / (1 - alpha_bars[i])) * betas[
+                i
+            ]
+        sigmas_inflex = torch.sqrt(sigmas_inflex)
+
+        self.register_buffer("betas", betas)
+        self.register_buffer("alphas", alphas)
+        self.register_buffer("alpha_bars", alpha_bars)
+        self.register_buffer("sigmas_flex", sigmas_flex)
+        self.register_buffer("sigmas_inflex", sigmas_inflex)
+
+    def uniform_sample_t(self, batch_size):
+        ts = np.random.choice(np.arange(1, self.num_steps + 1), batch_size)
+        return ts.tolist()
+
+    def get_sigmas(self, t, flexibility):
+        assert 0 <= flexibility and flexibility <= 1
+        sigmas = self.sigmas_flex[t] * flexibility + self.sigmas_inflex[t] * (
+            1 - flexibility
+        )
+        return sigmas
+
+
+class NoisePredictor(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+
+        content_encoder_class = get_network(cfg.CONTENT_ENCODER_TYPE)
+        self.content_encoder = content_encoder_class(**cfg.CONTENT_ENCODER)
+
+        style_encoder_class = get_network(cfg.STYLE_ENCODER_TYPE)
+        cfg.defrost()
+        cfg.STYLE_ENCODER.input_dim = cfg.DATASET.FACE3D_DIM
+        cfg.freeze()
+        self.style_encoder = style_encoder_class(**cfg.STYLE_ENCODER)
+
+        decoder_class = get_network(cfg.DECODER_TYPE)
+        cfg.defrost()
+        cfg.DECODER.output_dim = cfg.DATASET.FACE3D_DIM
+        cfg.freeze()
+        self.decoder = decoder_class(**cfg.DECODER)
+
+        self.content_xt_to_decoder_input_wo_time = nn.Sequential(
+            nn.Linear(cfg.D_MODEL + cfg.DATASET.FACE3D_DIM, cfg.D_MODEL),
+            nn.ReLU(),
+            nn.Linear(cfg.D_MODEL, cfg.D_MODEL),
+            nn.ReLU(),
+            nn.Linear(cfg.D_MODEL, cfg.D_MODEL),
+        )
+
+        self.time_sinusoidal_dim = cfg.D_MODEL
+        self.time_embed_net = nn.Sequential(
+            nn.Linear(cfg.D_MODEL, cfg.D_MODEL),
+            nn.SiLU(),
+            nn.Linear(cfg.D_MODEL, cfg.D_MODEL),
+        )
+
+    def forward(self, x_t, t, audio, style_clip, style_pad_mask, ready_style_code=None):
+        """_summary_
+
+        Args:
+            x_t (_type_): (B, L, C_face)
+            t (_type_): (B,) dtype:float32
+            audio (_type_): (B, L, W)
+            style_clip (_type_): (B, L_clipmax, C_face3d)
+            style_pad_mask : (B, L_clipmax)
+            ready_style_code: (B, C_model)
+        Returns:
+            e_theta : (B, L, C_face)
+        """
+        W = audio.shape[2]
+        content = self.content_encoder(audio)
+        # (B, L, W, C_model)
+        x_t_expand = x_t.unsqueeze(2).repeat(1, 1, W, 1)
+        # (B, L, C_face) -> (B, L, W, C_face)
+        content_xt_concat = torch.cat((content, x_t_expand), dim=3)
+        # (B, L, W, C_model+C_face)
+        decoder_input_without_time = self.content_xt_to_decoder_input_wo_time(
+            content_xt_concat
+        )
+        # (B, L, W, C_model)
+
+        time_sinusoidal = sinusoidal_embedding(t, self.time_sinusoidal_dim)
+        # (B, C_embed)
+        time_embedding = self.time_embed_net(time_sinusoidal)
+        # (B, C_model)
+        B, C = time_embedding.shape
+        time_embed_expand = time_embedding.view(B, 1, 1, C)
+        decoder_input = decoder_input_without_time + time_embed_expand
+        # (B, L, W, C_model)
+
+        if ready_style_code is not None:
+            style_code = ready_style_code
+        else:
+            style_code = self.style_encoder(style_clip, style_pad_mask)
+        # (B, C_model)
+
+        e_theta = self.decoder(decoder_input, style_code)
+        # (B, L, C_face)
+        return e_theta

actora/third_party/dreamtalk_src/core/networks/disentangle_decoder.py

@@ -0,0 +1,240 @@
+import torch
+from torch import nn
+
+from .transformer import (
+    PositionalEncoding,
+    TransformerDecoderLayer,
+    TransformerDecoder,
+)
+from core.networks.dynamic_fc_decoder import DynamicFCDecoderLayer, DynamicFCDecoder
+from core.utils import _reset_parameters
+
+
+def get_decoder_network(
+    network_type,
+    d_model,
+    nhead,
+    dim_feedforward,
+    dropout,
+    activation,
+    normalize_before,
+    num_decoder_layers,
+    return_intermediate_dec,
+    dynamic_K,
+    dynamic_ratio,
+):
+    decoder = None
+    if network_type == "TransformerDecoder":
+        decoder_layer = TransformerDecoderLayer(
+            d_model, nhead, dim_feedforward, dropout, activation, normalize_before
+        )
+        norm = nn.LayerNorm(d_model)
+        decoder = TransformerDecoder(
+            decoder_layer,
+            num_decoder_layers,
+            norm,
+            return_intermediate_dec,
+        )
+    elif network_type == "DynamicFCDecoder":
+        d_style = d_model
+        decoder_layer = DynamicFCDecoderLayer(
+            d_model,
+            nhead,
+            d_style,
+            dynamic_K,
+            dynamic_ratio,
+            dim_feedforward,
+            dropout,
+            activation,
+            normalize_before,
+        )
+        norm = nn.LayerNorm(d_model)
+        decoder = DynamicFCDecoder(
+            decoder_layer, num_decoder_layers, norm, return_intermediate_dec
+        )
+    elif network_type == "DynamicFCEncoder":
+        d_style = d_model
+        decoder_layer = DynamicFCEncoderLayer(
+            d_model,
+            nhead,
+            d_style,
+            dynamic_K,
+            dynamic_ratio,
+            dim_feedforward,
+            dropout,
+            activation,
+            normalize_before,
+        )
+        norm = nn.LayerNorm(d_model)
+        decoder = DynamicFCEncoder(decoder_layer, num_decoder_layers, norm)
+
+    else:
+        raise ValueError(f"Invalid network_type {network_type}")
+
+    return decoder
+
+
+class DisentangleDecoder(nn.Module):
+    def __init__(
+        self,
+        d_model=512,
+        nhead=8,
+        num_decoder_layers=3,
+        dim_feedforward=2048,
+        dropout=0.1,
+        activation="relu",
+        normalize_before=False,
+        return_intermediate_dec=False,
+        pos_embed_len=80,
+        upper_face3d_indices=tuple(list(range(19)) + list(range(46, 51))),
+        lower_face3d_indices=tuple(range(19, 46)),
+        network_type="None",
+        dynamic_K=None,
+        dynamic_ratio=None,
+        **_,
+    ) -> None:
+        super().__init__()
+
+        self.upper_face3d_indices = upper_face3d_indices
+        self.lower_face3d_indices = lower_face3d_indices
+
+        # upper_decoder_layer = TransformerDecoderLayer(
+        #     d_model, nhead, dim_feedforward, dropout, activation, normalize_before
+        # )
+        # upper_decoder_norm = nn.LayerNorm(d_model)
+        # self.upper_decoder = TransformerDecoder(
+        #     upper_decoder_layer,
+        #     num_decoder_layers,
+        #     upper_decoder_norm,
+        #     return_intermediate=return_intermediate_dec,
+        # )
+        self.upper_decoder = get_decoder_network(
+            network_type,
+            d_model,
+            nhead,
+            dim_feedforward,
+            dropout,
+            activation,
+            normalize_before,
+            num_decoder_layers,
+            return_intermediate_dec,
+            dynamic_K,
+            dynamic_ratio,
+        )
+        _reset_parameters(self.upper_decoder)
+
+        # lower_decoder_layer = TransformerDecoderLayer(
+        #     d_model, nhead, dim_feedforward, dropout, activation, normalize_before
+        # )
+        # lower_decoder_norm = nn.LayerNorm(d_model)
+        # self.lower_decoder = TransformerDecoder(
+        #     lower_decoder_layer,
+        #     num_decoder_layers,
+        #     lower_decoder_norm,
+        #     return_intermediate=return_intermediate_dec,
+        # )
+        self.lower_decoder = get_decoder_network(
+            network_type,
+            d_model,
+            nhead,
+            dim_feedforward,
+            dropout,
+            activation,
+            normalize_before,
+            num_decoder_layers,
+            return_intermediate_dec,
+            dynamic_K,
+            dynamic_ratio,
+        )
+        _reset_parameters(self.lower_decoder)
+
+        self.pos_embed = PositionalEncoding(d_model, pos_embed_len)
+
+        tail_hidden_dim = d_model // 2
+        self.upper_tail_fc = nn.Sequential(
+            nn.Linear(d_model, tail_hidden_dim),
+            nn.ReLU(),
+            nn.Linear(tail_hidden_dim, tail_hidden_dim),
+            nn.ReLU(),
+            nn.Linear(tail_hidden_dim, len(upper_face3d_indices)),
+        )
+        self.lower_tail_fc = nn.Sequential(
+            nn.Linear(d_model, tail_hidden_dim),
+            nn.ReLU(),
+            nn.Linear(tail_hidden_dim, tail_hidden_dim),
+            nn.ReLU(),
+            nn.Linear(tail_hidden_dim, len(lower_face3d_indices)),
+        )
+
+    def forward(self, content, style_code):
+        """
+
+        Args:
+            content (_type_): (B, num_frames, window, C_dmodel)
+            style_code (_type_): (B, C_dmodel)
+
+        Returns:
+            face3d: (B, L_clip, C_3dmm)
+        """
+        B, N, W, C = content.shape
+        style = style_code.reshape(B, 1, 1, C).expand(B, N, W, C)
+        style = style.permute(2, 0, 1, 3).reshape(W, B * N, C)
+        # (W, B*N, C)
+
+        content = content.permute(2, 0, 1, 3).reshape(W, B * N, C)
+        # (W, B*N, C)
+        tgt = torch.zeros_like(style)
+        pos_embed = self.pos_embed(W)
+        pos_embed = pos_embed.permute(1, 0, 2)
+
+        upper_face3d_feat = self.upper_decoder(
+            tgt, content, pos=pos_embed, query_pos=style
+        )[0]
+        # (W, B*N, C)
+        upper_face3d_feat = upper_face3d_feat.permute(1, 0, 2).reshape(B, N, W, C)[
+            :, :, W // 2, :
+        ]
+        # (B, N, C)
+        upper_face3d = self.upper_tail_fc(upper_face3d_feat)
+        # (B, N, C_exp)
+
+        lower_face3d_feat = self.lower_decoder(
+            tgt, content, pos=pos_embed, query_pos=style
+        )[0]
+        lower_face3d_feat = lower_face3d_feat.permute(1, 0, 2).reshape(B, N, W, C)[
+            :, :, W // 2, :
+        ]
+        lower_face3d = self.lower_tail_fc(lower_face3d_feat)
+        C_exp = len(self.upper_face3d_indices) + len(self.lower_face3d_indices)
+        face3d = torch.zeros(B, N, C_exp).to(upper_face3d)
+        face3d[:, :, self.upper_face3d_indices] = upper_face3d
+        face3d[:, :, self.lower_face3d_indices] = lower_face3d
+        return face3d
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.path.append("/home/mayifeng/Research/styleTH")
+
+    from configs.default import get_cfg_defaults
+
+    cfg = get_cfg_defaults()
+    cfg.merge_from_file("configs/styleTH_unpair_lsfm_emotion.yaml")
+    cfg.freeze()
+
+    # content_encoder = ContentEncoder(**cfg.CONTENT_ENCODER)
+
+    # dummy_audio = torch.randint(0, 41, (5, 64, 11))
+    # dummy_content = content_encoder(dummy_audio)
+
+    # style_encoder = StyleEncoder(**cfg.STYLE_ENCODER)
+    # dummy_face3d_seq = torch.randn(5, 64, 64)
+    # dummy_style_code = style_encoder(dummy_face3d_seq)
+
+    decoder = DisentangleDecoder(**cfg.DECODER)
+    dummy_content = torch.randn(5, 64, 11, 256)
+    dummy_style = torch.randn(5, 256)
+    dummy_output = decoder(dummy_content, dummy_style)
+
+    print("hello")

actora/third_party/dreamtalk_src/core/networks/dynamic_conv.py

@@ -0,0 +1,156 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+class Attention(nn.Module):
+    def __init__(self, cond_planes, ratio, K, temperature=30, init_weight=True):
+        super().__init__()
+        # self.avgpool = nn.AdaptiveAvgPool2d(1)
+        self.temprature = temperature
+        assert cond_planes > ratio
+        hidden_planes = cond_planes // ratio
+        self.net = nn.Sequential(
+            nn.Conv2d(cond_planes, hidden_planes, kernel_size=1, bias=False),
+            nn.ReLU(),
+            nn.Conv2d(hidden_planes, K, kernel_size=1, bias=False),
+        )
+
+        if init_weight:
+            self._initialize_weights()
+
+    def update_temprature(self):
+        if self.temprature > 1:
+            self.temprature -= 1
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            if isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+    def forward(self, cond):
+        """
+
+        Args:
+            cond (_type_): (B, C_style)
+
+        Returns:
+            _type_: (B, K)
+        """
+
+        # att = self.avgpool(cond)  # bs,dim,1,1
+        att = cond.view(cond.shape[0], cond.shape[1], 1, 1)
+        att = self.net(att).view(cond.shape[0], -1)  # bs,K
+        return F.softmax(att / self.temprature, -1)
+
+
+class DynamicConv(nn.Module):
+    def __init__(
+        self,
+        in_planes,
+        out_planes,
+        cond_planes,
+        kernel_size,
+        stride,
+        padding=0,
+        dilation=1,
+        groups=1,
+        bias=True,
+        K=4,
+        temperature=30,
+        ratio=4,
+        init_weight=True,
+    ):
+        super().__init__()
+        self.in_planes = in_planes
+        self.out_planes = out_planes
+        self.cond_planes = cond_planes
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.padding = padding
+        self.dilation = dilation
+        self.groups = groups
+        self.bias = bias
+        self.K = K
+        self.init_weight = init_weight
+        self.attention = Attention(
+            cond_planes=cond_planes, ratio=ratio, K=K, temperature=temperature, init_weight=init_weight
+        )
+
+        self.weight = nn.Parameter(
+            torch.randn(K, out_planes, in_planes // groups, kernel_size, kernel_size), requires_grad=True
+        )
+        if bias:
+            self.bias = nn.Parameter(torch.randn(K, out_planes), requires_grad=True)
+        else:
+            self.bias = None
+
+        if self.init_weight:
+            self._initialize_weights()
+
+    def _initialize_weights(self):
+        for i in range(self.K):
+            nn.init.kaiming_uniform_(self.weight[i], a=math.sqrt(5))
+            if self.bias is not None:
+                fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight[i])
+                if fan_in != 0:
+                    bound = 1 / math.sqrt(fan_in)
+                    nn.init.uniform_(self.bias, -bound, bound)
+
+    def forward(self, x, cond):
+        """
+
+        Args:
+            x (_type_): (B, C_in, L, 1)
+            cond (_type_): (B, C_style)
+
+        Returns:
+            _type_: (B, C_out, L, 1)
+        """
+        bs, in_planels, h, w = x.shape
+        softmax_att = self.attention(cond)  # bs,K
+        x = x.view(1, -1, h, w)
+        weight = self.weight.view(self.K, -1)  # K,-1
+        aggregate_weight = torch.mm(softmax_att, weight).view(
+            bs * self.out_planes, self.in_planes // self.groups, self.kernel_size, self.kernel_size
+        )  # bs*out_p,in_p,k,k
+
+        if self.bias is not None:
+            bias = self.bias.view(self.K, -1)  # K,out_p
+            aggregate_bias = torch.mm(softmax_att, bias).view(-1)  # bs*out_p
+            output = F.conv2d(
+                x, # 1, bs*in_p, L, 1
+                weight=aggregate_weight,
+                bias=aggregate_bias,
+                stride=self.stride,
+                padding=self.padding,
+                groups=self.groups * bs,
+                dilation=self.dilation,
+            )
+        else:
+            output = F.conv2d(
+                x,
+                weight=aggregate_weight,
+                bias=None,
+                stride=self.stride,
+                padding=self.padding,
+                groups=self.groups * bs,
+                dilation=self.dilation,
+            )
+
+        output = output.view(bs, self.out_planes, h, w)
+        return output
+
+
+if __name__ == "__main__":
+    input = torch.randn(3, 32, 64, 64)
+    m = DynamicConv(in_planes=32, out_planes=64, kernel_size=3, stride=1, padding=1, bias=True)
+    out = m(input)
+    print(out.shape)

actora/third_party/dreamtalk_src/core/networks/dynamic_fc_decoder.py

@@ -0,0 +1,178 @@
+import torch.nn as nn
+import torch
+
+from core.networks.transformer import _get_activation_fn, _get_clones
+from core.networks.dynamic_linear import DynamicLinear
+
+
+class DynamicFCDecoderLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        d_style,
+        dynamic_K,
+        dynamic_ratio,
+        dim_feedforward=2048,
+        dropout=0.1,
+        activation="relu",
+        normalize_before=False,
+    ):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        # self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.linear1 = DynamicLinear(d_model, dim_feedforward, d_style, K=dynamic_K, ratio=dynamic_ratio)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+        # self.linear2 = DynamicLinear(dim_feedforward, d_model, d_style, K=dynamic_K, ratio=dynamic_ratio)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def with_pos_embed(self, tensor, pos):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self,
+        tgt,
+        memory,
+        style,
+        tgt_mask=None,
+        memory_mask=None,
+        tgt_key_padding_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        # q = k = self.with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(tgt, tgt, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(
+            query=tgt, key=memory, value=memory, attn_mask=memory_mask, key_padding_mask=memory_key_padding_mask
+        )[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        # tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt, style))), style)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt, style))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+        return tgt
+
+    # def forward_pre(
+    #     self,
+    #     tgt,
+    #     memory,
+    #     tgt_mask=None,
+    #     memory_mask=None,
+    #     tgt_key_padding_mask=None,
+    #     memory_key_padding_mask=None,
+    #     pos=None,
+    #     query_pos=None,
+    # ):
+    #     tgt2 = self.norm1(tgt)
+    #     # q = k = self.with_pos_embed(tgt2, query_pos)
+    #     tgt2 = self.self_attn(tgt2, tgt2, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask)[0]
+    #     tgt = tgt + self.dropout1(tgt2)
+    #     tgt2 = self.norm2(tgt)
+    #     tgt2 = self.multihead_attn(
+    #         query=tgt2, key=memory, value=memory, attn_mask=memory_mask, key_padding_mask=memory_key_padding_mask
+    #     )[0]
+    #     tgt = tgt + self.dropout2(tgt2)
+    #     tgt2 = self.norm3(tgt)
+    #     tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+    #     tgt = tgt + self.dropout3(tgt2)
+    #     return tgt
+
+    def forward(
+        self,
+        tgt,
+        memory,
+        style,
+        tgt_mask=None,
+        memory_mask=None,
+        tgt_key_padding_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        if self.normalize_before:
+            raise NotImplementedError
+            # return self.forward_pre(
+            #     tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos
+            # )
+        return self.forward_post(
+            tgt, memory, style, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos
+        )
+
+
+class DynamicFCDecoder(nn.Module):
+    def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(
+        self,
+        tgt,
+        memory,
+        tgt_mask=None,
+        memory_mask=None,
+        tgt_key_padding_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        style = query_pos[0]
+        # (B*N, C)
+        output = tgt + pos + query_pos
+
+        intermediate = []
+
+        for layer in self.layers:
+            output = layer(
+                output,
+                memory,
+                style,
+                tgt_mask=tgt_mask,
+                memory_mask=memory_mask,
+                tgt_key_padding_mask=tgt_key_padding_mask,
+                memory_key_padding_mask=memory_key_padding_mask,
+                pos=pos,
+                query_pos=query_pos,
+            )
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate)
+
+        return output.unsqueeze(0)
+
+
+if __name__ == "__main__":
+    query = torch.randn(11, 1024, 256)
+    content = torch.randn(11, 1024, 256)
+    style = torch.randn(1024, 256)
+    pos = torch.randn(11, 1, 256)
+    m = DynamicFCDecoderLayer(256, 4, 256, 4, 4, 1024)
+
+    out = m(query, content, style, pos=pos)
+    print(out.shape)