Deploy frame extraction matcher

app.py

@@ -0,0 +1,4 @@
+from frame_extraction.app import main
+
+if __name__ == "__main__":
+    main()

catalog/README.md

@@ -0,0 +1,3 @@
+# Catalog Placeholder
+
+Upload your generated `catalog.json` and `references/` images here before deploying the Space. Update the `FRAME_CATALOG` environment variable accordingly.

frame_extraction/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+outputs/
+app_outputs/
+dummy_frames/
+catalog/
+__pycache__/

frame_extraction/README.md

@@ -0,0 +1,54 @@
+# Frame Extraction & Character Matching
+
+This package turns raw video into character reference catalogs and lets you match new frames against those references. It is designed to be deployed quickly (e.g., on Hugging Face Spaces) for interactive character discovery.
+
+## Features
+- Shot-aware frame sampling to keep only useful stills.
+- Face detection, embedding, and clustering (MTCNN + InceptionResnet).
+- Automatic reference selection per character (sharpest, most frontal crop).
+- JSON catalog output and optional reference thumbnails.
+- Matching API/CLI for user-uploaded frames with multi-character support.
+- Gradio app template ready for Hugging Face hosting.
+
+## Install
+```bash
+cd projects/UMO-Qwen-Edit/data_curation_scripts/frame_extraction
+pip install -e .
+```
+
+## CLI Usage
+### Build a catalog from a video
+```bash
+frame-catalog catalog \
+  --video-path data/source.mp4 \
+  --output-dir outputs/catalog \
+  --frame-interval 12 \
+  --min-track-length 5
+```
+
+### Match new frames against the catalog
+```bash
+frame-catalog match \
+  --catalog-path outputs/catalog/catalog.json \
+  --frames-dir uploads/ \
+  --output-path outputs/matches.json
+```
+
+## Deploy on Hugging Face Spaces
+1. Copy this folder to a new Space (Python SDK).  
+2. Install dependencies with `pip install -e .`.  
+3. Upload a pre-built `catalog/catalog.json` plus the `references/` images.  
+4. Set environment variables in the Space:
+   - `FRAME_CATALOG=/home/user/app/catalog/catalog.json`
+   - `FRAME_OUTPUT_DIR=/home/user/app/output`
+5. Set the Space entrypoint to `python -m frame_extraction.app`.
+
+## Outputs
+- `catalog.json`: character reference metadata with embeddings and chosen frames.
+- `references/`: cropped reference images per character.
+- `matches.json`: mapping from user frames to character IDs with similarity scores.
+
+## Roadmap
+- Integrate more robust trackers (DeepSort/ByteTrack).
+- Add active learning loop for manual character corrections.
+- Expose REST endpoints for automated ingestion.

frame_extraction/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "frame-extraction"
+version = "0.1.0"
+description = "Character-centric frame extraction and matching pipeline"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "typer>=0.12",
+    "rich>=13.7",
+    "numpy>=1.24",
+    "pandas>=2.1",
+    "opencv-python>=4.8",
+    "torch>=2.1",
+    "torchvision>=0.16",
+    "facenet-pytorch>=2.5.3",
+    "scikit-learn>=1.4",
+    "Pillow>=10.0",
+    "tqdm>=4.66",
+    "gradio>=4.0",
+    "faiss-cpu>=1.7.4",
+]
+
+[project.scripts]
+frame-catalog = "frame_extraction.cli:app"
+
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"

frame_extraction/src/frame_extraction.egg-info/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: frame-extraction
+Version: 0.1.0
+Summary: Character-centric frame extraction and matching pipeline
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: typer>=0.12
+Requires-Dist: rich>=13.7
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.1
+Requires-Dist: opencv-python>=4.8
+Requires-Dist: torch>=2.1
+Requires-Dist: torchvision>=0.16
+Requires-Dist: facenet-pytorch>=2.5.3
+Requires-Dist: scikit-learn>=1.4
+Requires-Dist: Pillow>=10.0
+Requires-Dist: tqdm>=4.66
+Requires-Dist: gradio>=4.0
+Requires-Dist: faiss-cpu>=1.7.4
+
+# Frame Extraction & Character Matching
+
+This package turns raw video into character reference catalogs and lets you match new frames against those references. It is designed to be deployed quickly (e.g., on Hugging Face Spaces) for interactive character discovery.
+
+## Features
+- Shot-aware frame sampling to keep only useful stills.
+- Face detection, embedding, and clustering (MTCNN + InceptionResnet).
+- Automatic reference selection per character (sharpest, most frontal crop).
+- JSON catalog output and optional reference thumbnails.
+- Matching API/CLI for user-uploaded frames with multi-character support.
+- Gradio app template ready for Hugging Face hosting.
+
+## Install
+```bash
+cd projects/UMO-Qwen-Edit/data_curation_scripts/frame_extraction
+pip install -e .
+```
+
+## CLI Usage
+### Build a catalog from a video
+```bash
+frame-catalog catalog \
+  --video-path data/source.mp4 \
+  --output-dir outputs/catalog \
+  --frame-interval 12 \
+  --min-track-length 5
+```
+
+### Match new frames against the catalog
+```bash
+frame-catalog match \
+  --catalog-path outputs/catalog/catalog.json \
+  --frames-dir uploads/ \
+  --output-path outputs/matches.json
+```
+
+## Deploy on Hugging Face Spaces
+1. Copy this folder to a new Space (Python SDK).  
+2. Install dependencies with `pip install -e .`.  
+3. Upload a pre-built `catalog/catalog.json` plus the `references/` images.  
+4. Set environment variables in the Space:
+   - `FRAME_CATALOG=/home/user/app/catalog/catalog.json`
+   - `FRAME_OUTPUT_DIR=/home/user/app/output`
+5. Set the Space entrypoint to `python -m frame_extraction.app`.
+
+## Outputs
+- `catalog.json`: character reference metadata with embeddings and chosen frames.
+- `references/`: cropped reference images per character.
+- `matches.json`: mapping from user frames to character IDs with similarity scores.
+
+## Roadmap
+- Integrate more robust trackers (DeepSort/ByteTrack).
+- Add active learning loop for manual character corrections.
+- Expose REST endpoints for automated ingestion.

frame_extraction/src/frame_extraction.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+pyproject.toml
+src/frame_extraction/__init__.py
+src/frame_extraction/app.py
+src/frame_extraction/catalog.py
+src/frame_extraction/cli.py
+src/frame_extraction/clustering.py
+src/frame_extraction/config.py
+src/frame_extraction/face.py
+src/frame_extraction/matcher.py
+src/frame_extraction/quality.py
+src/frame_extraction/video.py
+src/frame_extraction.egg-info/PKG-INFO
+src/frame_extraction.egg-info/SOURCES.txt
+src/frame_extraction.egg-info/dependency_links.txt
+src/frame_extraction.egg-info/entry_points.txt
+src/frame_extraction.egg-info/requires.txt
+src/frame_extraction.egg-info/top_level.txt

frame_extraction/src/frame_extraction.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

frame_extraction/src/frame_extraction.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+frame-catalog = frame_extraction.cli:app

frame_extraction/src/frame_extraction.egg-info/requires.txt

@@ -0,0 +1,13 @@
+typer>=0.12
+rich>=13.7
+numpy>=1.24
+pandas>=2.1
+opencv-python>=4.8
+torch>=2.1
+torchvision>=0.16
+facenet-pytorch>=2.5.3
+scikit-learn>=1.4
+Pillow>=10.0
+tqdm>=4.66
+gradio>=4.0
+faiss-cpu>=1.7.4

frame_extraction/src/frame_extraction.egg-info/top_level.txt

@@ -0,0 +1 @@
+frame_extraction

frame_extraction/src/frame_extraction/__init__.py

@@ -0,0 +1,7 @@
+"""Frame extraction and character matching utilities."""
+
+from .config import CatalogConfig, MatchConfig
+from .catalog import build_catalog
+from .matcher import match_frames
+
+__all__ = ["CatalogConfig", "MatchConfig", "build_catalog", "match_frames"]

frame_extraction/src/frame_extraction/app.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import numpy as np
+import torch
+
+from .config import MatchConfig
+from .matcher import match_frames
+
+
+CATALOG_PATH = Path(os.getenv("FRAME_CATALOG", "catalog/catalog.json"))
+OUTPUT_DIR = Path(os.getenv("FRAME_OUTPUT_DIR", "app_outputs"))
+
+
+def load_catalog() -> dict[str, Any] | None:
+    path = Path(CATALOG_PATH)
+    if path.exists():
+        return json.loads(path.read_text(encoding="utf-8"))
+    return None
+
+
+catalog_cache = load_catalog()
+
+
+def predict(images: list[np.ndarray]) -> tuple[list[str], list[str]]:
+    if catalog_cache is None:
+        raise gr.Error("Catalog not found. Upload catalog.json or set FRAME_CATALOG.")
+
+    if not images:
+        raise gr.Error("Please upload at least one frame.")
+
+    frames_dir = OUTPUT_DIR / "inputs"
+    frames_dir.mkdir(parents=True, exist_ok=True)
+    for idx, image in enumerate(images):
+        from PIL import Image
+
+        Image.fromarray(image).save(frames_dir / f"upload_{idx:03d}.png")
+
+    output_path = OUTPUT_DIR / "matches.json"
+    cfg = MatchConfig(
+        catalog_path=Path(CATALOG_PATH),
+        frames_dir=frames_dir,
+        output_path=output_path,
+        top_k=1,
+        similarity_threshold=0.5,
+    )
+    match_frames(cfg)
+    data = json.loads(output_path.read_text(encoding="utf-8"))
+    gallery_items = [(item["reference_crop"], f"{item['character_id']} ({item['similarity']:.2f})") for item in data]
+    return data, gallery_items
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Character Reference Matcher")
+    with gr.Row():
+        image_input = gr.Image(type="numpy", image_mode="RGB", label="Upload frames", multiple=True)
+    submit = gr.Button("Match Characters")
+    matches_json = gr.JSON(label="Matches")
+    gallery = gr.Gallery(label="Reference Thumbnails").style(grid=2)
+
+    submit.click(predict, inputs=image_input, outputs=[matches_json, gallery])
+
+
+def main() -> None:
+    demo.launch()
+
+
+if __name__ == "__main__":
+    main()

frame_extraction/src/frame_extraction/catalog.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+from rich.console import Console
+from rich.progress import track
+
+from .clustering import Clusterer
+from .config import CatalogConfig
+from .face import FaceDetector, FaceEmbedder, crop_faces, cv2_to_rgb
+from .quality import laplacian_sharpness
+from .video import iter_frames
+
+console = Console()
+
+
+@dataclass(slots=True)
+class FaceRecord:
+    character_id: str
+    movie: str
+    frame_index: int
+    frame_path: str
+    bbox: list[float]
+    detection_score: float
+    sharpness: float
+    embedding: list[float]
+    reference_path: str | None = None
+
+
+def build_catalog(cfg: CatalogConfig) -> Path:
+    cfg.ensure()
+    detector = FaceDetector(min_face_size=cfg.mtcnn_min_face_size)
+    embedder = FaceEmbedder(batch_size=cfg.embed_batch_size)
+    clusterer = Clusterer(eps=cfg.cluster_eps, min_samples=cfg.cluster_min_samples)
+
+    frames_dir = cfg.output_dir / "frames"
+    faces_dir = cfg.output_dir / "faces"
+    references_dir = cfg.output_dir / "references"
+
+    face_records: list[FaceRecord] = []
+    embeddings: list[np.ndarray] = []
+
+    video_name = cfg.video_path.stem
+
+    for frame_idx, frame in iter_frames(cfg.video_path, cfg.frame_interval):
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        frame_path = frames_dir / f"frame_{frame_idx:06d}.jpg"
+        cv2.imwrite(str(frame_path), cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR))
+
+        boxes, scores = detector.detect(frame_rgb)
+        if boxes.size == 0:
+            continue
+
+        crops = crop_faces(frame_rgb, boxes)
+        if not crops:
+            continue
+
+        crop_embeddings = embedder.embed(crops)
+        embeddings.append(crop_embeddings)
+        for idx, (crop, emb, box, score) in enumerate(zip(crops, crop_embeddings, boxes, scores, strict=False)):
+            crop_path = faces_dir / f"{frame_idx:06d}_{idx}.jpg"
+            crop.save(crop_path)
+            sharpness = laplacian_sharpness(np.asarray(crop))
+            face_records.append(
+                FaceRecord(
+                    character_id="",  # placeholder until clustering
+                    movie=video_name,
+                    frame_index=frame_idx,
+                    frame_path=str(frame_path),
+                    bbox=box.tolist(),
+                    detection_score=float(score),
+                    sharpness=sharpness,
+                    embedding=emb.tolist(),
+                )
+            )
+
+    if not face_records:
+        raise RuntimeError("No faces detected in video.")
+
+    all_embeddings = np.vstack(embeddings)
+    labels = clusterer.cluster(all_embeddings)
+
+    label_counter: dict[int, int] = {}
+    for label in labels:
+        label_counter[label] = label_counter.get(label, 0) + 1
+
+    for record, label in zip(face_records, labels, strict=False):
+        if label < 0 or label_counter[label] < cfg.min_track_length:
+            record.character_id = f"{video_name}_char_noise"
+        else:
+            record.character_id = f"{video_name}_char_{label:03d}"
+
+    references: dict[str, FaceRecord] = {}
+    for record in face_records:
+        if record.character_id.endswith("char_noise"):
+            continue
+        current = references.get(record.character_id)
+        score = record.sharpness if cfg.reference_metric == "sharpness" else record.detection_score
+        if current is None:
+            references[record.character_id] = record
+        else:
+            current_score = current.sharpness if cfg.reference_metric == "sharpness" else current.detection_score
+            if score > current_score:
+                references[record.character_id] = record
+
+    for record in references.values():
+        source = Path(record.frame_path)
+        frame = cv2.imread(str(source))
+        x1, y1, x2, y2 = map(int, record.bbox)
+        crop = frame[y1:y2, x1:x2]
+        output_path = references_dir / f"{record.character_id}.jpg"
+        cv2.imwrite(str(output_path), crop)
+        record.reference_path = str(output_path)
+
+    catalog = {
+        "video": video_name,
+        "config": {
+            "frame_interval": cfg.frame_interval,
+            "min_track_length": cfg.min_track_length,
+            "reference_metric": cfg.reference_metric,
+        },
+        "characters": [asdict(record) for record in face_records if not record.character_id.endswith("char_noise")],
+        "references": [asdict(record) for record in references.values()],
+    }
+
+    catalog_path = cfg.output_dir / "catalog.json"
+    with catalog_path.open("w", encoding="utf-8") as f:
+        json.dump(catalog, f, indent=2)
+
+    console.log(f"Wrote catalog to {catalog_path}")
+    return catalog_path

frame_extraction/src/frame_extraction/cli.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from .catalog import build_catalog
+from .config import CatalogConfig, MatchConfig
+from .matcher import match_frames
+
+app = typer.Typer(add_completion=False)
+console = Console()
+
+
+@app.command()
+def catalog(
+    video_path: Path = typer.Argument(..., help="Input video file"),
+    output_dir: Path = typer.Option(..., "--output-dir", "-o", help="Directory to store catalog outputs"),
+    frame_interval: int = typer.Option(12, help="Sample every Nth frame"),
+    min_track_length: int = typer.Option(5, help="Minimum detections per cluster"),
+    cluster_eps: float = typer.Option(0.55, help="DBSCAN epsilon for clustering"),
+    cluster_min_samples: int = typer.Option(3, help="Minimum samples for DBSCAN"),
+) -> None:
+    """Build a character reference catalog from a video."""
+    cfg = CatalogConfig(
+        video_path=video_path,
+        output_dir=output_dir,
+        frame_interval=frame_interval,
+        min_track_length=min_track_length,
+        cluster_eps=cluster_eps,
+        cluster_min_samples=cluster_min_samples,
+    )
+    build_catalog(cfg)
+
+
+@app.command()
+def match(
+    catalog_path: Path = typer.Argument(..., help="Catalog JSON path"),
+    frames_dir: Path = typer.Option(..., "--frames-dir", "-f", help="Directory with input frames"),
+    output_path: Path = typer.Option(..., "--output-path", "-o", help="Where to write matches JSON"),
+    top_k: int = typer.Option(1, help="Return up to K matches per face"),
+    similarity_threshold: float = typer.Option(0.5, help="Minimum cosine similarity to accept match"),
+) -> None:
+    """Match new frames against an existing catalog."""
+    cfg = MatchConfig(
+        catalog_path=catalog_path,
+        frames_dir=frames_dir,
+        output_path=output_path,
+        top_k=top_k,
+        similarity_threshold=similarity_threshold,
+    )
+    match_frames(cfg)
+
+
+if __name__ == "__main__":
+    app()

frame_extraction/src/frame_extraction/clustering.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable
+
+import numpy as np
+from sklearn.cluster import DBSCAN
+
+
+@dataclass(slots=True)
+class Clusterer:
+    eps: float = 0.55
+    min_samples: int = 3
+
+    def cluster(self, embeddings: np.ndarray) -> np.ndarray:
+        if len(embeddings) == 0:
+            return np.empty((0,), dtype=int)
+        model = DBSCAN(eps=self.eps, min_samples=self.min_samples, metric="cosine")
+        labels = model.fit_predict(embeddings)
+        return labels

frame_extraction/src/frame_extraction/config.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+
+@dataclass(slots=True)
+class CatalogConfig:
+    video_path: Path
+    output_dir: Path
+    frame_interval: int = 12
+    mtcnn_min_face_size: int = 60
+    min_track_length: int = 5
+    embed_batch_size: int = 16
+    cluster_eps: float = 0.55
+    cluster_min_samples: int = 3
+    reference_metric: Literal["sharpness", "confidence"] = "sharpness"
+
+    def ensure(self) -> None:
+        self.video_path = Path(self.video_path)
+        self.output_dir = Path(self.output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        (self.output_dir / "frames").mkdir(exist_ok=True)
+        (self.output_dir / "faces").mkdir(exist_ok=True)
+        (self.output_dir / "references").mkdir(exist_ok=True)
+
+
+@dataclass(slots=True)
+class MatchConfig:
+    catalog_path: Path
+    frames_dir: Path
+    output_path: Path
+    top_k: int = 1
+    similarity_threshold: float = 0.5
+
+    def ensure(self) -> None:
+        self.catalog_path = Path(self.catalog_path)
+        self.frames_dir = Path(self.frames_dir)
+        self.output_path = Path(self.output_path)
+        self.output_path.parent.mkdir(parents=True, exist_ok=True)

frame_extraction/src/frame_extraction/face.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, List, Tuple
+
+import numpy as np
+import torch
+from facenet_pytorch import InceptionResnetV1, MTCNN
+from PIL import Image
+
+
+@dataclass(slots=True)
+class FaceDetector:
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    min_face_size: int = 60
+
+    def __post_init__(self) -> None:
+        self.model = MTCNN(
+            keep_all=True,
+            device=self.device,
+            min_face_size=self.min_face_size,
+            post_process=False,
+        )
+
+    def detect(self, image: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+        pil = Image.fromarray(cv2_to_rgb(image))
+        boxes, probs = self.model.detect(pil)
+        if boxes is None or probs is None:
+            return np.empty((0, 4), dtype=np.float32), np.empty((0,), dtype=np.float32)
+        return boxes.astype(np.float32), probs.astype(np.float32)
+
+
+@dataclass(slots=True)
+class FaceEmbedder:
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    batch_size: int = 16
+
+    def __post_init__(self) -> None:
+        self.model = InceptionResnetV1(pretrained="vggface2").eval().to(self.device)
+
+    @torch.no_grad()
+    def embed(self, crops: Iterable[Image.Image]) -> np.ndarray:
+        embeddings: List[np.ndarray] = []
+        batch: List[torch.Tensor] = []
+        for crop in crops:
+            tensor = preprocess_crop(crop)
+            batch.append(tensor)
+            if len(batch) == self.batch_size:
+                embeddings.append(self._forward(batch))
+                batch.clear()
+        if batch:
+            embeddings.append(self._forward(batch))
+        if not embeddings:
+            return np.empty((0, 512), dtype=np.float32)
+        return np.vstack(embeddings)
+
+    def _forward(self, batch: List[torch.Tensor]) -> np.ndarray:
+        stacked = torch.stack(batch).to(self.device)
+        out = self.model(stacked)
+        return out.cpu().numpy().astype(np.float32)
+
+
+def preprocess_crop(crop: Image.Image) -> torch.Tensor:
+    crop = crop.resize((160, 160))
+    tensor = torch.from_numpy(np.asarray(crop).astype(np.float32))
+    tensor = tensor.permute(2, 0, 1) / 255.0
+    tensor = (tensor - 0.5) / 0.5
+    return tensor
+
+
+def crop_faces(image: np.ndarray, boxes: np.ndarray, margin: float = 0.3) -> list[Image.Image]:
+    h, w = image.shape[:2]
+    crops: list[Image.Image] = []
+    for x1, y1, x2, y2 in boxes:
+        w_box = x2 - x1
+        h_box = y2 - y1
+        x1m = max(0, int(x1 - margin * w_box))
+        y1m = max(0, int(y1 - margin * h_box))
+        x2m = min(w, int(x2 + margin * w_box))
+        y2m = min(h, int(y2 + margin * h_box))
+        crop = image[y1m:y2m, x1m:x2m]
+        crops.append(Image.fromarray(cv2_to_rgb(crop)))
+    return crops
+
+
+def cv2_to_rgb(image: np.ndarray) -> np.ndarray:
+    return image[:, :, ::-1]

frame_extraction/src/frame_extraction/matcher.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import cv2
+import faiss
+import numpy as np
+from facenet_pytorch import MTCNN
+from rich.console import Console
+
+from .config import MatchConfig
+from .face import FaceEmbedder, crop_faces
+
+console = Console()
+
+
+def load_catalog(catalog_path: Path) -> tuple[np.ndarray, list[dict[str, Any]]]:
+    data = json.loads(catalog_path.read_text(encoding="utf-8"))
+    references = data.get("references", [])
+    embeddings = np.array([ref["embedding"] for ref in references], dtype=np.float32)
+    return embeddings, references
+
+
+def build_index(embeddings: np.ndarray) -> faiss.IndexFlatIP:
+    if embeddings.size == 0:
+        raise RuntimeError("Catalog has no reference embeddings.")
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True) + 1e-8
+    normalized = embeddings / norms
+    index = faiss.IndexFlatIP(embeddings.shape[1])
+    index.add(normalized)
+    return index
+
+
+def match_frames(cfg: MatchConfig) -> Path:
+    cfg.ensure()
+    embeddings, references = load_catalog(cfg.catalog_path)
+    index = build_index(embeddings)
+    embedder = FaceEmbedder()
+    detector = MTCNN(keep_all=True, device=embedder.device)
+
+    matches: list[dict[str, Any]] = []
+    image_paths = sorted(p for p in cfg.frames_dir.glob("*") if p.suffix.lower() in {".png", ".jpg", ".jpeg"})
+
+    for image_path in image_paths:
+        image_bgr = cv2.imread(str(image_path))
+        image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
+        boxes, probs = detector.detect(image_rgb)
+        if boxes is None or probs is None:
+            continue
+        crops = crop_faces(image_rgb, boxes)
+        face_embeddings = embedder.embed(crops)
+        if face_embeddings.size == 0:
+            continue
+        norms = np.linalg.norm(face_embeddings, axis=1, keepdims=True) + 1e-8
+        normalized = face_embeddings / norms
+        scores, idxs = index.search(normalized, cfg.top_k)
+        for face_idx, (box, score_row, idx_row) in enumerate(zip(boxes, scores, idxs, strict=False)):
+            for score, idx in zip(score_row, idx_row, strict=False):
+                if score < cfg.similarity_threshold:
+                    continue
+                reference = references[int(idx)]
+                matches.append(
+                    {
+                        "input_frame": str(image_path),
+                        "face_index": face_idx,
+                        "bbox": box.tolist(),
+                        "similarity": float(score),
+                        "character_id": reference["character_id"],
+                        "reference_frame": reference["frame_path"],
+                        "reference_crop": reference["reference_path"],
+                    }
+                )
+
+    cfg.output_path.write_text(json.dumps(matches, indent=2), encoding="utf-8")
+    console.log(f"Wrote matches to {cfg.output_path}")
+    return cfg.output_path

frame_extraction/src/frame_extraction/quality.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import cv2
+import numpy as np
+
+
+def laplacian_sharpness(image: np.ndarray) -> float:
+    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
+    return float(cv2.Laplacian(gray, cv2.CV_64F).var())
+
+
+def brightness_score(image: np.ndarray) -> float:
+    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
+    return float(gray.mean() / 255.0)

frame_extraction/src/frame_extraction/video.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import math
+from pathlib import Path
+from typing import Iterator
+
+import cv2
+from tqdm import tqdm
+
+
+def iter_frames(video_path: str | Path, frame_interval: int = 12) -> Iterator[tuple[int, "cv2.Mat"]]:
+    cap = cv2.VideoCapture(str(video_path))
+    if not cap.isOpened():
+        raise RuntimeError(f"Could not open video: {video_path}")
+
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
+    interval = max(1, frame_interval)
+
+    with tqdm(total=math.ceil(total_frames / interval), desc="Extracting frames") as pbar:
+        frame_idx = 0
+        sampled = 0
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+            if frame_idx % interval == 0:
+                yield frame_idx, frame
+                sampled += 1
+                pbar.update(1)
+            frame_idx += 1
+    cap.release()

requirements.txt

@@ -0,0 +1,13 @@
+typer>=0.12
+rich>=13.7
+numpy>=1.24
+pandas>=2.1
+opencv-python-headless>=4.8
+torch==2.2.2
+torchvision==0.17.2
+facenet-pytorch>=2.6.0
+scikit-learn>=1.7
+Pillow>=10.0
+tqdm>=4.66
+gradio>=5.0
+faiss-cpu>=1.12.0