Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ffmpeg \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first for better caching
+COPY server/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+# Copy all source code
+COPY . /app
+
+EXPOSE 8000
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+ENV ENABLE_WEB_INTERFACE=true
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+    CMD sh -lc 'curl -f http://localhost:${PORT:-8000}/health || exit 1'
+
+CMD ["sh", "-lc", "python -m uvicorn server.app:app --host 0.0.0.0 --port ${PORT:-8000}"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 elix3r
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,11 +1,114 @@
 ---
-title: Clip Quality Env
-emoji: 🏃
-colorFrom: blue
-colorTo: green
+title: CLIP Quality Analyzer
+colorFrom: purple
+colorTo: gray
 sdk: docker
-pinned: false
-license: mit
+app_port: 8000
+base_path: /dashboard/
+tags:
+  - openenv
+  - reinforcement-learning
+  - clip-quality
+  - quality-analysis
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# CLIP Quality Analyzer Environment
+
+ClipQualityEnv is an OpenEnv-compliant RL environment for CLIP quality analysis workflows. It keeps the reference OpenEnv structure while presenting a clip-quality review and classification surface.
+
+## Action Space
+
+Classifier action payload:
+
+- `label`: one of `KEEP`, `BORDERLINE`, `REJECT`
+- `reasoning`: concise clip-metadata-grounded explanation
+- `confidence`: float in `[0.0, 1.0]`
+- `clip_id` (optional): specific clip target
+
+## Observation Space
+
+Each observation includes:
+- `task_id`, `episode_id`, `step_count`
+- `data_corpus`, `clip_metadata`, `rubric_summary`
+- `reward`, `done`, and step diagnostics in `info`
+
+These fields are preserved for OpenEnv/reference compatibility and surfaced as clip-case analysis artifacts in `/dashboard/`.
+
+## Tasks
+
+| Task ID | Difficulty | Description |
+|---|---|---|
+| `task_easy` | Easy | Classify clips with clear quality signals |
+| `task_medium` | Medium | Classify borderline clips with mixed indicators |
+| `task_hard` | Hard | Classify hard clips with conflicting quality signals |
+
+Overall total scores are difficulty-calibrated so task-level averages follow: `hard > medium > easy`.
+
+## API + Dashboard Surface
+
+- `GET /` -> JSON status
+- `GET /health`
+- `GET /state`
+- `GET /tasks`
+- `POST /grader`
+- `POST /baseline/start`
+- `GET /baseline/status/{run_id}`
+- `GET /baseline` (compatibility wrapper for async baseline start)
+- OpenEnv endpoints: `/reset`, `/step`, `/ws`, `/metadata`, `/schema`
+- Gradio dashboard: `/dashboard/`
+
+The API surface remains reference-compatible; product naming and UI wording are clip-quality focused.
+
+`POST /grader` accepts the classifier action payload above.
+
+## Dashboard Highlights
+
+- 5-step same-scenario review sessions with running totals
+- Full corpus queue display (no slicing), sorted by Clip ID, with live review-status updates
+- Difficulty-tiered input tabs (Easy / Medium / Hard) with scenario-aware tab switching
+- `💡 Load Quality Hint` helper button to scaffold reasoning text from dominant feature boundary cues
+- Session history tab with submitted vs expected labels and per-step rewards
+- Non-blocking `🤖 Run LLM Baseline Agent` control with async status polling and result summary
+
+## Local Setup
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r server/requirements.txt
+PYTHONPATH=. python -m pytest -q
+```
+
+Run server:
+
+```bash
+python -m uvicorn server.app:app --host 0.0.0.0 --port 8000
+```
+
+## Baseline Inference
+
+Set env vars:
+
+```bash
+export API_BASE_URL="https://router.huggingface.co/v1"
+export MODEL_NAME="llama-3.3-70b-versatile"
+export HF_TOKEN="your_token"
+```
+
+Run:
+
+```bash
+python inference.py
+```
+
+Optional single task:
+
+```bash
+python inference.py task_easy
+```
+
+## Hugging Face Space Deployment
+
+```bash
+openenv push --repo-id elix3r/ClipQualityEnv .
+```

__init__.py

@@ -0,0 +1,28 @@
+"""Root OpenEnv exports for deployment tooling compatibility."""
+
+from clip_quality_env import (
+    Action,
+    ClipLabel,
+    ClipQualityClient,
+    ClipQualityEnv,
+    ClipQualityEnvironment,
+    EnvironmentState,
+    Observation,
+    State,
+    TaskInfo,
+)
+
+# Backward-compatibility alias (deprecated).
+PolicyEvolverEnv = ClipQualityClient
+
+__all__ = [
+    "Action",
+    "ClipLabel",
+    "ClipQualityClient",
+    "ClipQualityEnv",
+    "ClipQualityEnvironment",
+    "EnvironmentState",
+    "Observation",
+    "State",
+    "TaskInfo",
+]

client.py

@@ -0,0 +1,8 @@
+"""Root client module for OpenEnv CLI compatibility."""
+
+from clip_quality_env.client import ClipQualityClient
+
+# Backward-compatibility alias (deprecated).
+PolicyEvolverEnv = ClipQualityClient
+
+__all__ = ["ClipQualityClient"]

clip_quality_env.egg-info/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: clip-quality-env
+Version: 1.0.0
+Summary: ClipQualityEnv — OpenEnv RL environment for clip-quality analysis and classification
+Requires-Python: >=3.10
+License-File: LICENSE
+Requires-Dist: openenv-core>=0.2.3
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: fastapi>=0.104.0
+Requires-Dist: uvicorn>=0.24.0
+Requires-Dist: requests>=2.25.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: gradio>=4.0.0
+Requires-Dist: pandas>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Dynamic: license-file

clip_quality_env.egg-info/SOURCES.txt

@@ -0,0 +1,33 @@
+LICENSE
+README.md
+pyproject.toml
+clip_quality_env/__init__.py
+clip_quality_env/agent.py
+clip_quality_env/client.py
+clip_quality_env/difficulty.py
+clip_quality_env/env.py
+clip_quality_env/generator.py
+clip_quality_env/grader.py
+clip_quality_env/ground_truth.py
+clip_quality_env/models.py
+clip_quality_env/real_clips.py
+clip_quality_env/rubric.py
+clip_quality_env/train.py
+clip_quality_env.egg-info/PKG-INFO
+clip_quality_env.egg-info/SOURCES.txt
+clip_quality_env.egg-info/dependency_links.txt
+clip_quality_env.egg-info/entry_points.txt
+clip_quality_env.egg-info/requires.txt
+clip_quality_env.egg-info/top_level.txt
+server/__init__.py
+server/app.py
+server/baseline_runs.py
+server/clip_quality_environment.py
+server/environment.py
+server/grader.py
+tests/test_baseline_runs.py
+tests/test_environment.py
+tests/test_grader.py
+tests/test_inference.py
+tests/test_models.py
+tests/test_server_routes.py

clip_quality_env.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

clip_quality_env.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+server = server.app:main

clip_quality_env.egg-info/requires.txt

@@ -0,0 +1,12 @@
+openenv-core>=0.2.3
+pydantic>=2.0.0
+openai>=1.0.0
+fastapi>=0.104.0
+uvicorn>=0.24.0
+requests>=2.25.0
+websockets>=12.0
+gradio>=4.0.0
+pandas>=2.0.0
+
+[dev]
+pytest>=7.0.0

clip_quality_env.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+clip_quality_env
+server

clip_quality_env/__init__.py

@@ -0,0 +1,38 @@
+"""Clip-quality environment package."""
+
+from .client import ClipQualityClient
+from .env import ClipQualityEnv, ClipQualityEnvironment
+from .models import (
+    Action,
+    ClipLabel,
+    ClipMetadata,
+    CorpusIncident,
+    EnvironmentState,
+    EpisodeHistoryItem,
+    HistoryItem,
+    Observation,
+    Reward,
+    State,
+    TaskInfo,
+)
+
+# Backward-compatibility aliases (deprecated).
+PolicyEvolverEnv = ClipQualityClient
+PolicyEvolverEnvironment = ClipQualityEnvironment
+
+__all__ = [
+    "Action",
+    "ClipLabel",
+    "ClipMetadata",
+    "ClipQualityClient",
+    "ClipQualityEnv",
+    "ClipQualityEnvironment",
+    "CorpusIncident",
+    "EnvironmentState",
+    "EpisodeHistoryItem",
+    "HistoryItem",
+    "Observation",
+    "Reward",
+    "State",
+    "TaskInfo",
+]

clip_quality_env/agent.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+from openai import OpenAI
+
+from .models import Action, Observation
+
+
+LABEL_RE = re.compile(r"<label>\s*(KEEP|BORDERLINE|REJECT)\s*</label>", re.IGNORECASE)
+REASONING_RE = re.compile(r"<reasoning>\s*(.*?)\s*</reasoning>", re.IGNORECASE | re.DOTALL)
+CONFIDENCE_RE = re.compile(r"<confidence>\s*([0-9]*\.?[0-9]+)\s*</confidence>", re.IGNORECASE)
+
+
+class LLMAgent:
+    """OpenAI-client-backed agent for ClipQualityEnv."""
+
+    SYSTEM_PROMPT = (
+        "You are a dataset quality analyst for talking-head LoRA training clips.\n"
+        "Classify each clip as KEEP, BORDERLINE, or REJECT.\n"
+        "Always respond with XML tags exactly:\n"
+        "<label>...</label>\n<reasoning>...</reasoning>\n<confidence>0.0-1.0</confidence>"
+    )
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        api_base_url: str | None = None,
+        api_key: str | None = None,
+        temperature: float = 0.3,
+        max_tokens: int = 512,
+    ) -> None:
+        self.model_name = model_name or os.environ.get("MODEL_NAME", "meta-llama/Llama-3-70B-Instruct")
+        self.api_base_url = api_base_url or os.environ.get("API_BASE_URL", "https://api-inference.huggingface.co/v1/")
+        self.api_key = api_key or os.environ.get("HF_TOKEN") or os.environ.get("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("Either HF_TOKEN or OPENAI_API_KEY must be set")
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.client = OpenAI(base_url=self.api_base_url, api_key=self.api_key)
+
+    def act(self, obs: Observation | dict[str, Any]) -> Action:
+        observation = obs if isinstance(obs, Observation) else Observation(**obs)
+        prompt = self._build_prompt(observation)
+        raw = self._call_model(prompt)
+        return self._parse_response(raw)
+
+    def _build_prompt(self, obs: Observation) -> str:
+        parts = [obs.rubric_summary, ""]
+        for h in obs.history:
+            parts.append(f"[STEP {h.step} - Previous]")
+            parts.append(f"Your label: {h.label} | Reward: {h.reward:.2f}")
+            parts.append("")
+        parts.append(f"[STEP {obs.step} - Current Clip]")
+        parts.append(json.dumps(obs.clip_metadata.model_dump(), indent=2))
+        parts.append("")
+        parts.append("Classify this clip.")
+        return "\n".join(parts)
+
+    def _call_model(self, prompt: str) -> str:
+        response = self.client.chat.completions.create(
+            model=self.model_name,
+            messages=[
+                {"role": "system", "content": self.SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+        )
+        content = response.choices[0].message.content
+        if not content:
+            return "<label>BORDERLINE</label><reasoning>No content.</reasoning><confidence>0.5</confidence>"
+        return content
+
+    def _parse_response(self, text: str) -> Action:
+        label_match = LABEL_RE.search(text)
+        reasoning_match = REASONING_RE.search(text)
+        confidence_match = CONFIDENCE_RE.search(text)
+
+        label = label_match.group(1).upper() if label_match else "BORDERLINE"
+        reasoning = reasoning_match.group(1).strip() if reasoning_match else "No reasoning provided."
+        confidence = 0.5
+        if confidence_match:
+            try:
+                confidence = float(confidence_match.group(1))
+            except ValueError:
+                confidence = 0.5
+        confidence = max(0.0, min(1.0, confidence))
+        return Action(label=label, reasoning=reasoning, confidence=confidence)

clip_quality_env/client.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any
+
+from openenv.core import EnvClient
+from openenv.core.client_types import StepResult
+
+from .models import Action, EnvironmentState, Observation
+
+
+class ClipQualityClient(EnvClient[Action, Observation, EnvironmentState]):
+    """Typed async client for ClipQualityEnv with sync wrapper support."""
+
+    def _step_payload(self, action: Action) -> dict[str, Any]:
+        payload = action.model_dump()
+        if isinstance(payload, dict) and "root" in payload and isinstance(payload["root"], dict):
+            return payload["root"]
+        return payload if isinstance(payload, dict) else {"action": payload}
+
+    def _parse_result(self, payload: dict[str, Any]) -> StepResult[Observation]:
+        obs_data = payload.get("observation", {})
+        if "done" not in obs_data:
+            obs_data["done"] = bool(payload.get("done", False))
+        if "reward" not in obs_data:
+            obs_data["reward"] = payload.get("reward")
+        observation = Observation.model_validate(obs_data)
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=bool(payload.get("done", False)),
+        )
+
+    def _parse_state(self, payload: dict[str, Any]) -> EnvironmentState:
+        return EnvironmentState.model_validate(payload)

clip_quality_env/difficulty.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import Final
+
+
+DIFFICULTY_ORDER: Final[dict[str, int]] = {
+    "easy": 0,
+    "medium": 1,
+    "hard": 2,
+}
+
+# Final-score calibration bands for enforcing strict ordering:
+# easy < medium < hard for all calibrated totals.
+DIFFICULTY_TOTAL_BANDS: Final[dict[str, tuple[float, float]]] = {
+    "easy": (0.00, 0.32),
+    "medium": (0.34, 0.66),
+    "hard": (0.68, 1.00),
+}
+
+
+def _clamp01(value: float) -> float:
+    return max(0.0, min(1.0, float(value)))
+
+
+def normalize_difficulty(value: str | None) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip().lower()
+    if normalized not in DIFFICULTY_ORDER:
+        return None
+    return normalized
+
+
+def calibrate_total_score(total: float, difficulty: str | None) -> float:
+    raw = _clamp01(total)
+    normalized = normalize_difficulty(difficulty)
+    if normalized is None:
+        return raw
+
+    band_min, band_max = DIFFICULTY_TOTAL_BANDS[normalized]
+    band_min = float(band_min)
+    band_max = float(band_max)
+    if band_max <= band_min:
+        return raw
+
+    return band_min + (raw * (band_max - band_min))

clip_quality_env/env.py

@@ -0,0 +1,450 @@
+from __future__ import annotations
+
+import copy
+import os
+import random
+import uuid
+from dataclasses import dataclass
+from typing import Any
+
+from openenv.core import Environment
+
+from .grader import grade
+from .ground_truth import GTStore
+from .models import Action, ClipMetadata, EpisodeHistoryItem, HistoryItem, Observation, State
+from .real_clips import load_real_clip_manifest
+from .rubric import RubricState
+from server.tasks import TASK_REGISTRY
+
+EPISODE_STEPS = 5
+DEFAULT_REAL_CLIPS_MANIFEST = "data/real_clips_manifest.jsonl"
+
+
+@dataclass
+class EpisodeClip:
+    task_id: str
+    difficulty: str
+    clip: dict[str, Any]
+
+
+class ClipQualityEnvironment(Environment[Action, Observation, State]):
+    """OpenEnv environment for clip-quality classification tasks."""
+
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._rubric = RubricState()
+        self._gt_store = GTStore()
+        self._state = State(
+            max_steps=EPISODE_STEPS,
+            total_reward=0.0,
+            rubric_version=self._rubric.version,
+            gt_size=self._gt_store.size(),
+            rubric_thresholds=self._rubric.get_thresholds_summary(),
+        )
+        self._episode_plan: list[EpisodeClip] = []
+        self._episode_corpus: dict[str, list[dict[str, Any]]] = {}
+        self._persistent_best_score = 0.0
+        self._last_reward_breakdown = {
+            "format_score": 0.0,
+            "label_score": 0.0,
+            "reasoning_score": 0.0,
+            "total_reward": 0.0,
+        }
+        self._corpus_source = "task_registry"
+        self._manifest_warning = ""
+        self._manifest_path = os.environ.get("REAL_CLIPS_MANIFEST", DEFAULT_REAL_CLIPS_MANIFEST)
+        self._real_clip_pools = self._load_real_clip_pools()
+
+    def _load_real_clip_pools(self) -> dict[str, list[dict[str, Any]]]:
+        if not os.path.exists(self._manifest_path):
+            self._manifest_warning = (
+                f"Real clip manifest not found at {self._manifest_path}; using static task corpora."
+            )
+            return {}
+        try:
+            pools = load_real_clip_manifest(self._manifest_path, self._rubric)
+        except Exception as exc:
+            self._manifest_warning = (
+                f"Failed loading real clip manifest from {self._manifest_path}: {exc}; using static task corpora."
+            )
+            return {}
+        self._manifest_warning = ""
+        return pools
+
+    def _choose_tasks(self, seed: int | None = None, task_id: str | None = None) -> list[str]:
+        if task_id is not None:
+            if task_id not in TASK_REGISTRY:
+                raise KeyError(f"Unknown task_id: {task_id}")
+            return [task_id]
+        rng = random.Random(seed)
+        return [rng.choice(list(TASK_REGISTRY.keys()))]
+
+    def _load_task_corpus(self, task_id: str) -> list[dict[str, Any]]:
+        task = TASK_REGISTRY[task_id]
+        difficulty = str(task.get("difficulty", "")).lower()
+        corpus: list[dict[str, Any]]
+
+        if difficulty and self._real_clip_pools.get(difficulty):
+            corpus = copy.deepcopy(self._real_clip_pools[difficulty])
+            self._corpus_source = f"manifest:{difficulty}"
+        else:
+            corpus = copy.deepcopy(task.get("data_corpus", []))
+            self._corpus_source = f"task_registry:{task_id}"
+
+        if not corpus:
+            raise ValueError(f"No clip corpus configured for task_id={task_id}")
+
+        for clip in corpus:
+            clip.setdefault("clip_id", clip.get("id", str(uuid.uuid4())))
+            if "expected_label" not in clip:
+                clip["expected_label"] = self._rubric.derive_label(clip)
+            clip["review_status"] = str(clip.get("review_status", "pending")).lower()
+
+        corpus.sort(key=lambda item: str(item.get("clip_id", item.get("id", ""))))
+        return corpus
+
+    def _sample_episode_clips(self, corpus: list[dict[str, Any]], seed: int | None = None) -> list[dict[str, Any]]:
+        if not corpus:
+            raise ValueError("Cannot sample episode clips from an empty corpus")
+        rng = random.Random(seed)
+        if len(corpus) >= EPISODE_STEPS:
+            return [copy.deepcopy(clip) for clip in rng.sample(corpus, k=EPISODE_STEPS)]
+
+        pool = [copy.deepcopy(clip) for clip in corpus]
+        rng.shuffle(pool)
+        sampled: list[dict[str, Any]] = []
+        while len(sampled) < EPISODE_STEPS:
+            sampled.append(copy.deepcopy(pool[len(sampled) % len(pool)]))
+        return sampled
+
+    def _sample_episode_plan(
+        self,
+        task_ids: list[str],
+        seed: int | None = None,
+    ) -> tuple[list[EpisodeClip], dict[str, list[dict[str, Any]]]]:
+        if len(task_ids) != 1:
+            raise ValueError("Episode planning expects exactly one task id")
+        task_id = task_ids[0]
+        task = TASK_REGISTRY[task_id]
+        difficulty = str(task.get("difficulty", ""))
+        corpus = self._load_task_corpus(task_id)
+        sampled_clips = self._sample_episode_clips(corpus, seed=seed)
+        plan = [
+            EpisodeClip(task_id=task_id, difficulty=difficulty, clip=clip)
+            for clip in sampled_clips
+        ]
+        corpus_map: dict[str, list[dict[str, Any]]] = {task_id: corpus}
+        return plan, corpus_map
+
+    def _threshold_range_text(self, feature: str) -> str:
+        threshold = self._rubric.thresholds.get(feature)
+        if not threshold:
+            return "N/A"
+        mode = str(threshold["mode"])
+        keep_min = float(threshold["keep_min"])
+        keep_max = float(threshold["keep_max"])
+        reject_min = float(threshold["reject_min"])
+        reject_max = float(threshold["reject_max"])
+        if mode == "higher":
+            return f"KEEP >= {keep_min:.3g}; BORDERLINE [{reject_max:.3g}, {keep_min:.3g}); REJECT < {reject_max:.3g}"
+        if mode == "lower":
+            return f"KEEP <= {keep_max:.3g}; BORDERLINE ({keep_max:.3g}, {reject_min:.3g}]; REJECT > {reject_min:.3g}"
+        return (
+            f"KEEP [{keep_min:.3g}, {keep_max:.3g}]; BORDERLINE [{reject_min:.3g}, {keep_min:.3g})"
+            f" U ({keep_max:.3g}, {reject_max:.3g}]; REJECT outside [{reject_min:.3g}, {reject_max:.3g}]"
+        )
+
+    def _closest_boundary_features(self, clip: dict[str, Any], top_n: int = 2) -> list[str]:
+        distances: list[tuple[float, str]] = []
+        for feature, threshold in self._rubric.thresholds.items():
+            value = clip.get(feature)
+            if not isinstance(value, (int, float)):
+                continue
+            mode = str(threshold["mode"])
+            keep_min = float(threshold["keep_min"])
+            keep_max = float(threshold["keep_max"])
+            reject_min = float(threshold["reject_min"])
+            reject_max = float(threshold["reject_max"])
+            if mode == "higher":
+                boundaries = [keep_min, reject_max]
+            elif mode == "lower":
+                boundaries = [keep_max, reject_min]
+            else:
+                boundaries = [keep_min, keep_max, reject_min, reject_max]
+            min_distance = min(abs(float(value) - boundary) for boundary in boundaries)
+            distances.append((min_distance, feature))
+        distances.sort(key=lambda item: item[0])
+        return [feature for _, feature in distances[:top_n]]
+
+    def dominant_feature_rows(self, clip: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        if clip is None:
+            current_index = min(self._state.step_count, max(len(self._episode_plan) - 1, 0))
+            if not self._episode_plan:
+                return []
+            clip = self._episode_plan[current_index].clip
+        dominant_features = self._rubric.get_dominant_features(clip)
+        rows: list[dict[str, Any]] = []
+        for feature in dominant_features:
+            value = clip.get(feature)
+            if not isinstance(value, (int, float)):
+                continue
+            status = self._rubric.get_feature_status(feature, float(value))
+            rows.append(
+                {
+                    "Feature Name": feature,
+                    "Current Value": float(value),
+                    "Rubric Status": status,
+                    "Threshold Range": self._threshold_range_text(feature),
+                }
+            )
+        return rows
+
+    def build_quality_hint(self, clip: dict[str, Any] | None = None) -> str:
+        if clip is None:
+            current_index = min(self._state.step_count, max(len(self._episode_plan) - 1, 0))
+            if not self._episode_plan:
+                return ""
+            clip = self._episode_plan[current_index].clip
+        focus_features = self._closest_boundary_features(clip, top_n=2)
+        if not focus_features:
+            return "Use dominant clip metadata cues and compare each value against rubric thresholds."
+
+        segments: list[str] = []
+        for idx, feature in enumerate(focus_features):
+            value = clip.get(feature)
+            if not isinstance(value, (int, float)):
+                continue
+            status = self._rubric.get_feature_status(feature, float(value))
+            threshold = self._rubric.thresholds.get(feature, {})
+            mode = str(threshold.get("mode", ""))
+            keep_min = float(threshold.get("keep_min", 0.0))
+            keep_max = float(threshold.get("keep_max", 0.0))
+            reject_min = float(threshold.get("reject_min", 0.0))
+            reject_max = float(threshold.get("reject_max", 0.0))
+
+            if mode == "higher":
+                if status == "KEEP":
+                    direction = f"above the KEEP threshold ({keep_min:.3g})"
+                elif status == "REJECT":
+                    direction = f"below the REJECT threshold ({reject_max:.3g})"
+                else:
+                    direction = f"within the BORDERLINE range [{reject_max:.3g}, {keep_min:.3g})"
+            elif mode == "lower":
+                if status == "KEEP":
+                    direction = f"below the KEEP ceiling ({keep_max:.3g})"
+                elif status == "REJECT":
+                    direction = f"above the REJECT threshold ({reject_min:.3g})"
+                else:
+                    direction = f"within the BORDERLINE range ({keep_max:.3g}, {reject_min:.3g}]"
+            else:
+                if status == "KEEP":
+                    direction = f"within the KEEP band [{keep_min:.3g}, {keep_max:.3g}]"
+                elif status == "REJECT":
+                    direction = f"outside the acceptable range [{reject_min:.3g}, {reject_max:.3g}]"
+                else:
+                    direction = (
+                        f"within a BORDERLINE edge zone around [{reject_min:.3g}, {keep_min:.3g})"
+                        f" or ({keep_max:.3g}, {reject_max:.3g}]"
+                    )
+
+            prefix = "" if idx == 0 else " "
+            segments.append(f"{prefix}{feature} is {float(value):.3g}, which is {direction}.")
+
+        return "".join(segments).strip()
+
+    def _state_to_observation(self, reward: float, done: bool) -> Observation:
+        current_index = min(self._state.step_count, max(len(self._episode_plan) - 1, 0))
+        current = self._episode_plan[current_index]
+        corpus = self._episode_corpus.get(current.task_id, [])
+        full_corpus = list(corpus)
+        history_items = [
+            HistoryItem(
+                step=h.step,
+                clip_id=h.clip_id,
+                label=h.label,
+                expected_label=h.expected_label,
+                reward=h.reward,
+            )
+            for h in self._state.episode_history
+        ]
+        steps_remaining = max(0, self._state.max_steps - self._state.step_count)
+        session_history = [item.model_dump() for item in self._state.episode_history]
+        return Observation(
+            task_id=current.task_id,
+            episode_id=self._state.episode_id,
+            step_count=self._state.step_count,
+            max_steps=self._state.max_steps,
+            step=max(1, self._state.step_count + (0 if done else 1)),
+            rubric_version=self._rubric.version,
+            rubric_summary=self._rubric.to_prompt_text(),
+            clip_metadata=ClipMetadata.model_validate(current.clip),
+            history=history_items,
+            corpus_size=len(corpus),
+            corpus_shown=len(full_corpus),
+            data_corpus=full_corpus,
+            reward=float(reward),
+            done=done,
+            info={
+                "difficulty": current.difficulty,
+                "task_description": TASK_REGISTRY[current.task_id]["description"],
+                "best_score": self._persistent_best_score,
+                "last_reward": float(reward),
+                "action_history": self._state.actions_taken,
+                "steps_remaining": steps_remaining,
+                "total_reward": float(self._state.total_reward),
+                "reward_breakdown": dict(self._last_reward_breakdown),
+                "format_score": float(self._last_reward_breakdown["format_score"]),
+                "label_score": float(self._last_reward_breakdown["label_score"]),
+                "reasoning_score": float(self._last_reward_breakdown["reasoning_score"]),
+                "reward_total": float(self._last_reward_breakdown["total_reward"]),
+                "session_history": session_history,
+                "corpus_source": self._corpus_source,
+            },
+        )
+
+    def reset(
+        self,
+        seed: int | None = None,
+        episode_id: str | None = None,
+        **kwargs: Any,
+    ) -> Observation:
+        task_id = kwargs.get("task_id")
+        task_ids = self._choose_tasks(seed=seed, task_id=task_id)
+        self._episode_plan, self._episode_corpus = self._sample_episode_plan(task_ids, seed=seed)
+        self._last_reward_breakdown = {
+            "format_score": 0.0,
+            "label_score": 0.0,
+            "reasoning_score": 0.0,
+            "total_reward": 0.0,
+        }
+        self._state = State(
+            episode_id=episode_id or str(uuid.uuid4()),
+            task_id=self._episode_plan[0].task_id,
+            episode_count=self._state.episode_count + 1,
+            step_count=0,
+            max_steps=len(self._episode_plan),
+            current_score=0.0,
+            total_reward=0.0,
+            best_score=self._persistent_best_score,
+            current_clip_id=str(self._episode_plan[0].clip.get("clip_id", "")),
+            gt_size=self._gt_store.size(),
+            rubric_version=self._rubric.version,
+            actions_taken=[],
+            episode_history=[],
+            rubric_thresholds=self._rubric.get_thresholds_summary(),
+        )
+        obs = self._state_to_observation(reward=0.0, done=False)
+        if self._manifest_warning:
+            obs.info["warning"] = self._manifest_warning
+        return obs
+
+    def step(
+        self,
+        action: Action | dict[str, Any],
+        timeout_s: float | None = None,
+        **kwargs: Any,
+    ) -> Observation:
+        del timeout_s, kwargs
+        if not self._episode_plan:
+            self.reset()
+
+        current_index = min(self._state.step_count, len(self._episode_plan) - 1)
+        current = self._episode_plan[current_index]
+        clip = current.clip
+        payload = action.model_dump() if isinstance(action, Action) else dict(action)
+        action_obj = Action.model_validate(payload)
+
+        reward_obj = grade(action_obj, clip, self._rubric, self._gt_store, difficulty=current.difficulty)
+        reward = float(reward_obj.total)
+        self._last_reward_breakdown = {
+            "format_score": float(reward_obj.format_score),
+            "label_score": float(reward_obj.label_score),
+            "reasoning_score": float(reward_obj.reasoning_score),
+            "total_reward": float(reward),
+        }
+        self._state.total_reward += reward
+        self._state.current_score = float(self._state.total_reward)
+        self._state.best_score = max(self._state.best_score, reward)
+        self._persistent_best_score = max(self._persistent_best_score, self._state.best_score)
+
+        action_label = str(action_obj.label).upper()
+        submitted_clip_id = str(action_obj.clip_id or clip.get("clip_id", ""))
+        current_clip_id = str(clip.get("clip_id", ""))
+        corpus = self._episode_corpus.get(current.task_id, [])
+        status_updated = False
+        for item in corpus:
+            if str(item.get("clip_id", "")) == submitted_clip_id:
+                item["review_status"] = action_label
+                status_updated = True
+                break
+
+        if not status_updated and submitted_clip_id != current_clip_id:
+            for item in corpus:
+                if str(item.get("clip_id", "")) == current_clip_id:
+                    item["review_status"] = action_label
+                    break
+
+        self._state.actions_taken.append(action_label)
+        self._state.episode_history.append(
+            EpisodeHistoryItem(
+                step=current_index + 1,
+                difficulty=current.difficulty,
+                clip_id=str(clip.get("clip_id", "")),
+                label=action_label,
+                expected_label=str(clip.get("expected_label", "")),
+                reward=reward,
+            )
+        )
+        self._state.step_count += 1
+
+        done = self._state.step_count >= self._state.max_steps
+        if done:
+            hard_entry = self._state.episode_history[-1]
+            step3_result = {
+                "clip": self._episode_plan[-1].clip,
+                "action": action_obj.model_dump(),
+                "reward": reward,
+                "expected_label": self._episode_plan[-1].clip.get("expected_label"),
+            }
+            try:
+                gt_promoted = self._gt_store.try_promote(step3_result, episode=self._state.episode_count)
+            except ValueError:
+                gt_promoted = False
+            self._state.gt_size = self._gt_store.size()
+            if hard_entry.reward >= 0.85:
+                self._rubric.recalibrate(
+                    perf=type("Perf", (), {"easy_accuracy": 0.0, "medium_accuracy": 0.0, "hard_accuracy": 0.86})(),
+                    current_episode=self._state.episode_count,
+                )
+            self._state.rubric_version = self._rubric.version
+            self._state.rubric_thresholds = self._rubric.get_thresholds_summary()
+            obs = self._state_to_observation(reward=reward, done=True)
+            obs.info["gt_promoted"] = bool(gt_promoted)
+            obs.info["episode_summary"] = {
+                "steps_completed": self._state.step_count,
+                "max_steps": self._state.max_steps,
+                "total_reward": round(float(self._state.total_reward), 4),
+                "average_reward": round(float(self._state.total_reward) / max(1, self._state.max_steps), 4),
+            }
+            if self._manifest_warning:
+                obs.info["warning"] = self._manifest_warning
+            return obs
+
+        next_idx = self._state.step_count
+        self._state.task_id = self._episode_plan[next_idx].task_id
+        self._state.current_clip_id = str(self._episode_plan[next_idx].clip.get("clip_id", ""))
+        obs = self._state_to_observation(reward=reward, done=False)
+        if self._manifest_warning:
+            obs.info["warning"] = self._manifest_warning
+        return obs
+
+    @property
+    def state(self) -> State:
+        return self._state
+
+
+ClipQualityEnv = ClipQualityEnvironment
+# Backward-compatibility alias (deprecated).
+PolicyEvolverEnvironment = ClipQualityEnvironment

clip_quality_env/generator.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import random
+from collections import deque
+from copy import deepcopy
+from uuid import uuid4
+
+from .real_clips import DIFFICULTIES, load_real_clip_manifest
+from .rubric import RubricState
+
+
+EASY_ENVS = ["podcast_studio", "office", "home_office", "webcam_room"]
+NOVEL_ENVS = ["outdoor_interview", "crowded_event", "car_vlog", "street_walk"]
+RESOLUTIONS = ["1280x720", "1920x1080"]
+BACKGROUND_TAGS = ["solid_dark", "solid_light", "simple_room", "busy_room"]
+
+
+def _clamp(value: float, low: float, high: float) -> float:
+    return max(low, min(high, value))
+
+
+def _sample_from_keep_center(threshold: dict[str, float | str]) -> float:
+    mode = str(threshold["mode"])
+    keep_min = float(threshold["keep_min"])
+    keep_max = float(threshold["keep_max"])
+    if mode == "higher":
+        span = max(keep_max - keep_min, 1e-6)
+        return _clamp(keep_min + 0.35 * span, keep_min, keep_max)
+    if mode == "lower":
+        span = max(keep_max - keep_min, 1e-6)
+        return _clamp(keep_min + 0.35 * span, keep_min, keep_max)
+    return _clamp((keep_min + keep_max) / 2.0, keep_min, keep_max)
+
+
+def _sample_from_reject_zone(threshold: dict[str, float | str]) -> float:
+    mode = str(threshold["mode"])
+    reject_min = float(threshold["reject_min"])
+    reject_max = float(threshold["reject_max"])
+    keep_min = float(threshold["keep_min"])
+    keep_max = float(threshold["keep_max"])
+    if mode == "higher":
+        return _clamp(reject_max - 0.02, reject_min, reject_max)
+    if mode == "lower":
+        return _clamp(reject_min + 0.02, reject_min, reject_max)
+    if random.random() < 0.5:
+        return _clamp(reject_min - 0.2, 0.0, reject_min)
+    return _clamp(reject_max + 0.2, reject_max, reject_max + 10.0)
+
+
+def _sample_from_borderline_zone(threshold: dict[str, float | str]) -> float:
+    mode = str(threshold["mode"])
+    keep_min = float(threshold["keep_min"])
+    keep_max = float(threshold["keep_max"])
+    reject_min = float(threshold["reject_min"])
+    reject_max = float(threshold["reject_max"])
+    if mode == "higher":
+        low = reject_max
+        high = keep_min
+        return _clamp((low + high) / 2.0, low, high)
+    if mode == "lower":
+        low = keep_max
+        high = reject_min
+        return _clamp((low + high) / 2.0, low, high)
+    if random.random() < 0.5:
+        return _clamp((reject_min + keep_min) / 2.0, reject_min, keep_min)
+    return _clamp((keep_max + reject_max) / 2.0, keep_max, reject_max)
+
+
+class ClipMetaGenerator:
+    """Synthetic metadata generator for Easy/Medium/Hard clips."""
+
+    def __init__(self, seed: int = 7, real_clips_path: str | None = None) -> None:
+        self._rng = random.Random(seed)
+        self._recent_hard_ids: deque[str] = deque(maxlen=10)
+        self._real_clip_pools: dict[str, list[dict[str, object]]] = {d: [] for d in DIFFICULTIES}
+        self._real_clip_indices: dict[str, int] = {d: 0 for d in DIFFICULTIES}
+        self._real_clips_path = real_clips_path
+
+    def sample(self, difficulty: str, rubric: RubricState) -> dict[str, object]:
+        d = difficulty.lower()
+        if d == "easy":
+            real = self._sample_real("easy", rubric)
+            if real is not None:
+                return real
+            return self._gen_easy(rubric)
+        if d == "medium":
+            real = self._sample_real("medium", rubric)
+            if real is not None:
+                return real
+            return self._gen_medium(rubric)
+        if d == "hard":
+            real = self._sample_real("hard", rubric)
+            if real is not None:
+                return real
+            return self._gen_hard(rubric)
+        raise ValueError("difficulty must be one of: easy, medium, hard")
+
+    def use_real_clips(self, path: str, rubric: RubricState) -> dict[str, int]:
+        pools = load_real_clip_manifest(path, rubric)
+        self._real_clips_path = path
+        for difficulty in DIFFICULTIES:
+            rows = [deepcopy(clip) for clip in pools[difficulty]]
+            self._rng.shuffle(rows)
+            self._real_clip_pools[difficulty] = rows
+            self._real_clip_indices[difficulty] = 0
+        return self.real_clip_pool_sizes()
+
+    def real_clip_pool_sizes(self) -> dict[str, int]:
+        return {d: len(self._real_clip_pools[d]) for d in DIFFICULTIES}
+
+    def has_real_clips(self) -> bool:
+        return any(self._real_clip_pools[d] for d in DIFFICULTIES)
+
+    def real_clips_path(self) -> str | None:
+        return self._real_clips_path
+
+    def _sample_real(self, difficulty: str, rubric: RubricState) -> dict[str, object] | None:
+        if self._real_clips_path and not self.has_real_clips():
+            self.use_real_clips(self._real_clips_path, rubric)
+        pool = self._real_clip_pools[difficulty]
+        if not pool:
+            return None
+        idx = self._real_clip_indices[difficulty] % len(pool)
+        self._real_clip_indices[difficulty] += 1
+        return deepcopy(pool[idx])
+
+    def _gen_easy(self, rubric: RubricState) -> dict[str, object]:
+        label = self._rng.choice(["KEEP", "REJECT"])
+        clip = self._base_clip()
+
+        for feature, t in rubric.thresholds.items():
+            if feature not in clip:
+                continue
+            if label == "KEEP":
+                clip[feature] = float(_sample_from_keep_center(t))
+            else:
+                clip[feature] = float(_sample_from_reject_zone(t))
+
+        if label == "REJECT":
+            trigger = self._rng.choice(["occlusion", "face_confidence", "duration", "motion"])
+            if trigger == "occlusion":
+                clip["occlusion_present"] = True
+            elif trigger == "face_confidence":
+                clip["face_confidence"] = 0.52
+            elif trigger == "duration":
+                clip["duration_s"] = 3.2
+            else:
+                clip["motion_score"] = 0.57
+        else:
+            clip["occlusion_present"] = False
+
+        clip["clip_id"] = f"syn_easy_{uuid4().hex[:8]}"
+        clip["environment_tag"] = self._rng.choice(EASY_ENVS)
+        clip["bg_complexity"] = self._rng.choice(BACKGROUND_TAGS)
+        return clip
+
+    def _gen_medium(self, rubric: RubricState) -> dict[str, object]:
+        clip = self._gen_easy(rubric)
+        keys = list(rubric.thresholds.keys())
+        to_shift = self._rng.sample(keys, k=2)
+        for feature in to_shift:
+            if feature in clip:
+                clip[feature] = float(_sample_from_borderline_zone(rubric.thresholds[feature]))
+        clip["clip_id"] = f"syn_med_{uuid4().hex[:8]}"
+        clip["environment_tag"] = self._rng.choice(EASY_ENVS + NOVEL_ENVS[:1])
+        clip["occlusion_present"] = False
+        return clip
+
+    def _gen_hard(self, rubric: RubricState) -> dict[str, object]:
+        while True:
+            clip = self._gen_easy(rubric)
+            keys = list(rubric.thresholds.keys())
+            to_shift = self._rng.sample(keys, k=3)
+            for feature in to_shift:
+                if feature in clip:
+                    clip[feature] = float(_sample_from_borderline_zone(rubric.thresholds[feature]))
+            clip["clip_id"] = f"syn_hard_{uuid4().hex[:8]}"
+            clip["environment_tag"] = self._rng.choice(NOVEL_ENVS)
+            clip["occlusion_present"] = False
+            if clip["clip_id"] in self._recent_hard_ids:
+                continue
+            self._recent_hard_ids.append(str(clip["clip_id"]))
+            return clip
+
+    def _base_clip(self) -> dict[str, object]:
+        return {
+            "clip_id": f"syn_{uuid4().hex[:8]}",
+            "duration_s": round(self._rng.uniform(6.0, 10.0), 2),
+            "fps": self._rng.choice([24, 30]),
+            "resolution": self._rng.choice(RESOLUTIONS),
+            "face_area_ratio": round(self._rng.uniform(0.22, 0.42), 3),
+            "face_confidence": round(self._rng.uniform(0.75, 0.95), 3),
+            "head_pose_yaw_deg": round(self._rng.uniform(2.0, 18.0), 2),
+            "head_pose_pitch_deg": round(self._rng.uniform(-8.0, 8.0), 2),
+            "motion_score": round(self._rng.uniform(0.08, 0.28), 3),
+            "bg_complexity": self._rng.choice(BACKGROUND_TAGS),
+            "bg_complexity_score": round(self._rng.uniform(0.05, 0.20), 3),
+            "mouth_open_ratio": round(self._rng.uniform(0.28, 0.52), 3),
+            "blink_rate_hz": round(self._rng.uniform(0.15, 0.40), 3),
+            "audio_snr_db": round(self._rng.uniform(18.0, 28.0), 2),
+            "transcript_word_count": self._rng.randint(22, 64),
+            "transcript_confidence": round(self._rng.uniform(0.80, 0.97), 3),
+            "lighting_uniformity": round(self._rng.uniform(0.55, 0.85), 3),
+            "occlusion_present": False,
+            "environment_tag": self._rng.choice(EASY_ENVS),
+            "framing": self._rng.choice(["front", "left", "right", "closeup", "offgaze"]),
+        }

clip_quality_env/grader.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .difficulty import calibrate_total_score
+from .ground_truth import GTStore
+from .models import Action, Reward
+from .rubric import RubricState
+
+
+VALID_LABELS = {"KEEP", "BORDERLINE", "REJECT"}
+FEATURE_TOKEN_RE = re.compile(r"\b[a-z]+(?:_[a-z0-9]+)+\b")
+
+
+def _normalize_action(action: Action | dict[str, Any]) -> dict[str, Any]:
+    if isinstance(action, Action):
+        return action.model_dump()
+    if not isinstance(action, dict):
+        raise TypeError("action must be Action or dict")
+    return {
+        "label": str(action.get("label", "BORDERLINE")).upper(),
+        "reasoning": str(action.get("reasoning", "")),
+        "confidence": float(action.get("confidence", 0.5)),
+    }
+
+
+def _score_format(action: dict[str, Any]) -> float:
+    label = str(action.get("label", "")).upper()
+    reasoning = str(action.get("reasoning", "")).strip()
+    confidence = float(action.get("confidence", -1.0))
+    if label in VALID_LABELS and reasoning and 0.0 <= confidence <= 1.0:
+        return 0.10
+    return 0.0
+
+
+def _score_label(label: str, clip: dict[str, Any], rubric: RubricState, gt: GTStore) -> float:
+    clip_id = str(clip.get("clip_id", ""))
+    gt_label = gt.lookup(clip_id)
+    if gt_label is None:
+        gt_label = rubric.derive_label(clip)
+    if label == gt_label:
+        return 0.60
+    if gt_label == "BORDERLINE" and label in {"KEEP", "REJECT"}:
+        return 0.25
+    return 0.0
+
+
+def _contains_directional_cue(reasoning: str, feature: str, status: str) -> bool:
+    low_words = ("low", "below", "under", "small", "poor", "noisy", "high motion", "occlusion")
+    high_words = ("high", "above", "over", "good", "clear", "stable", "frontal", "well-lit")
+    text = reasoning.lower()
+    if feature not in text:
+        return False
+    if status == "REJECT":
+        return any(w in text for w in low_words + ("reject",))
+    if status == "KEEP":
+        return any(w in text for w in high_words + ("keep",))
+    return any(w in text for w in ("borderline", "mixed", "ambiguous", "tradeoff", "conflict"))
+
+
+def _check_directional_reasoning(reasoning: str, clip: dict[str, Any], dominant_features: list[str], rubric: RubricState) -> bool:
+    if not reasoning.strip():
+        return False
+    checks = 0
+    matches = 0
+    for feature in dominant_features:
+        if feature not in clip:
+            continue
+        value = clip[feature]
+        if not isinstance(value, (int, float)):
+            continue
+        checks += 1
+        status = rubric.get_feature_status(feature, float(value))
+        if _contains_directional_cue(reasoning, feature, status):
+            matches += 1
+    if checks == 0:
+        return False
+    return matches >= 1
+
+
+def _score_reasoning(reasoning: str, clip: dict[str, Any], rubric: RubricState) -> float:
+    score = 0.0
+    lower_reasoning = reasoning.lower()
+    dominant_features = rubric.get_dominant_features(clip)
+
+    mentioned = sum(1 for f in dominant_features if f.lower() in lower_reasoning)
+    if mentioned >= 2:
+        score += 0.10
+    elif mentioned == 1:
+        score += 0.05
+
+    if _check_directional_reasoning(reasoning, clip, dominant_features, rubric):
+        score += 0.10
+
+    all_feature_names = {k.lower() for k in clip.keys()}
+    hallucinated = [
+        token
+        for token in FEATURE_TOKEN_RE.findall(lower_reasoning)
+        if token not in all_feature_names
+    ]
+    if len(hallucinated) == 0:
+        score += 0.10
+    return min(max(score, 0.0), 0.30)
+
+
+def grade(
+    action: Action | dict[str, Any],
+    clip: dict[str, Any],
+    rubric: RubricState,
+    gt: GTStore,
+    difficulty: str | None = None,
+) -> Reward:
+    """
+    Fully deterministic reward decomposition.
+    """
+    payload = _normalize_action(action)
+    label = str(payload["label"]).upper()
+    reasoning = str(payload["reasoning"])
+
+    format_score = _score_format(payload)
+    label_score = _score_label(label, clip, rubric, gt)
+    reasoning_score = _score_reasoning(reasoning, clip, rubric)
+    total = format_score + label_score + reasoning_score
+    total = calibrate_total_score(total, difficulty=difficulty)
+
+    return Reward(
+        total=round(min(max(total, 0.0), 1.0), 6),
+        format_score=round(format_score, 6),
+        label_score=round(label_score, 6),
+        reasoning_score=round(reasoning_score, 6),
+    )
+
+
+def score(
+    action: Action | dict[str, Any],
+    clip: dict[str, Any],
+    rubric: RubricState,
+    gt: GTStore,
+    difficulty: str | None = None,
+) -> float:
+    return float(grade(action, clip, rubric, gt, difficulty=difficulty).total)

clip_quality_env/ground_truth.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+
+VALID_LABELS = {"KEEP", "BORDERLINE", "REJECT"}
+
+
+class GTStore:
+    """Append-only ground-truth store with promotion support."""
+
+    def __init__(self, seed_path: str = "data/seed_gt.json", state_path: str = "state/ground_truth.json") -> None:
+        self.seed_path = seed_path
+        self.path = state_path
+        self.records: dict[str, dict[str, Any]] = {}
+        self._load_seed()
+        self._load_state()
+
+    def _load_seed(self) -> None:
+        if not os.path.exists(self.seed_path):
+            raise FileNotFoundError(f"Seed ground truth file not found: {self.seed_path}")
+        with open(self.seed_path, "r", encoding="utf-8") as f:
+            payload = json.load(f)
+
+        if isinstance(payload, dict):
+            for clip_id, rec in payload.items():
+                label = str(rec.get("label", "")).upper()
+                if label in VALID_LABELS:
+                    self.records[clip_id] = {
+                        "label": label,
+                        "source": rec.get("source", "seed"),
+                        "episode": int(rec.get("episode", 0)),
+                        "reward": rec.get("reward"),
+                        "confidence": rec.get("confidence"),
+                    }
+        elif isinstance(payload, list):
+            for item in payload:
+                clip_id = str(item.get("clip_id", "")).strip()
+                label = str(item.get("label", "")).upper()
+                if clip_id and label in VALID_LABELS:
+                    self.records[clip_id] = {
+                        "label": label,
+                        "source": item.get("source", "seed"),
+                        "episode": int(item.get("episode", 0)),
+                        "reward": item.get("reward"),
+                        "confidence": item.get("confidence"),
+                    }
+        else:
+            raise ValueError("seed_gt.json must be a dict or list")
+
+    def _load_state(self) -> None:
+        if not os.path.exists(self.path):
+            return
+        with open(self.path, "r", encoding="utf-8") as f:
+            payload = json.load(f)
+        if not isinstance(payload, dict):
+            raise ValueError("ground_truth.json must be a dict")
+        for clip_id, rec in payload.items():
+            if clip_id in self.records:
+                continue
+            label = str(rec.get("label", "")).upper()
+            if label not in VALID_LABELS:
+                continue
+            self.records[clip_id] = {
+                "label": label,
+                "source": rec.get("source", "agent_promoted"),
+                "episode": int(rec.get("episode", 0)),
+                "reward": rec.get("reward"),
+                "confidence": rec.get("confidence"),
+            }
+
+    def save(self) -> None:
+        os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
+        with open(self.path, "w", encoding="utf-8") as f:
+            json.dump(self.records, f, indent=2, sort_keys=True)
+
+    def lookup(self, clip_id: str) -> str | None:
+        rec = self.records.get(clip_id)
+        return None if rec is None else str(rec.get("label"))
+
+    def size(self) -> int:
+        return len(self.records)
+
+    def get_promoted_clip_ids(self) -> list[str]:
+        return [clip_id for clip_id, rec in self.records.items() if rec.get("source") == "agent_promoted"]
+
+    def try_promote(self, step3_result: dict[str, Any], episode: int) -> bool:
+        """
+        Promote hard-step clip if reward/confidence threshold is met.
+
+        The optional `expected_label` key enforces that the promoted label is correct.
+        """
+        clip = step3_result.get("clip", {})
+        action = step3_result.get("action", {})
+        clip_id = str(clip.get("clip_id", "")).strip()
+        if not clip_id:
+            raise ValueError("step3_result.clip.clip_id is required")
+        if clip_id in self.records:
+            return False
+
+        reward = float(step3_result.get("reward", 0.0))
+        confidence = float(action.get("confidence", 0.0))
+        label = str(action.get("label", "")).upper()
+        expected_label = step3_result.get("expected_label")
+        if expected_label is not None:
+            expected_label = str(expected_label).upper()
+
+        if label not in VALID_LABELS:
+            return False
+        if reward < 0.85 or confidence < 0.80:
+            return False
+        if expected_label in VALID_LABELS and label != expected_label:
+            return False
+
+        self.records[clip_id] = {
+            "label": label,
+            "source": "agent_promoted",
+            "episode": int(episode),
+            "reward": round(reward, 6),
+            "confidence": round(confidence, 6),
+        }
+        self.save()
+        return True

clip_quality_env/models.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import uuid
+from enum import Enum
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ClipLabel(str, Enum):
+    KEEP = "KEEP"
+    BORDERLINE = "BORDERLINE"
+    REJECT = "REJECT"
+
+
+class Action(BaseModel):
+    """Agent action for clip-quality classification."""
+
+    label: Literal["KEEP", "BORDERLINE", "REJECT"] = Field(description="Predicted clip label")
+    reasoning: str = Field(min_length=1, description="Explanation tied to clip metadata")
+    confidence: float = Field(default=0.5, ge=0.0, le=1.0, description="Model confidence in the label")
+    clip_id: Optional[str] = Field(default=None, description="Optional target clip ID")
+    model_config = {"extra": "allow"}
+
+
+class TaskInfo(BaseModel):
+    """Returned by /tasks endpoint."""
+
+    task_id: str
+    difficulty: str
+    description: str
+    action_schema: dict
+
+
+class CorpusIncident(BaseModel):
+    """Compatibility record for corpus table views."""
+
+    id: str
+    content: str
+    review_status: str = "pending"
+    model_config = {"extra": "allow"}
+
+
+class ClipMetadata(BaseModel):
+    """Clip metadata payload consumed by the agent."""
+
+    clip_id: str
+    duration_s: Optional[float] = Field(default=None, ge=0.0)
+    fps: Optional[int] = Field(default=None, ge=1)
+    resolution: Optional[str] = None
+    face_area_ratio: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    face_confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    head_pose_yaw_deg: Optional[float] = None
+    head_pose_pitch_deg: Optional[float] = None
+    motion_score: Optional[float] = Field(default=None, ge=0.0)
+    bg_complexity: Optional[str] = None
+    bg_complexity_score: Optional[float] = Field(default=None, ge=0.0)
+    mouth_open_ratio: Optional[float] = Field(default=None, ge=0.0)
+    blink_rate_hz: Optional[float] = Field(default=None, ge=0.0)
+    audio_snr_db: Optional[float] = None
+    transcript_word_count: Optional[int] = Field(default=None, ge=0)
+    transcript_confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    lighting_uniformity: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    occlusion_present: bool = False
+    environment_tag: Optional[str] = None
+    framing: Optional[str] = None
+    expected_label: Optional[str] = None
+    model_config = {"extra": "allow"}
+
+
+class HistoryItem(BaseModel):
+    step: int = 0
+    clip_id: str = ""
+    label: str = ""
+    expected_label: str = ""
+    reward: float = 0.0
+    model_config = {"extra": "allow"}
+
+
+class EpisodeHistoryItem(BaseModel):
+    step: int = 0
+    difficulty: str = ""
+    clip_id: str = ""
+    label: str = ""
+    expected_label: str = ""
+    reward: float = 0.0
+    model_config = {"extra": "allow"}
+
+
+class Observation(BaseModel):
+    """What the agent sees after reset() or step()."""
+
+    task_id: str
+    episode_id: str
+    step_count: int
+    max_steps: int = 5
+    step: int = Field(default=1, ge=1, description="1-indexed episode step")
+    rubric_version: int = Field(default=1, ge=1)
+    rubric_summary: str = ""
+    clip_metadata: ClipMetadata
+    history: List[HistoryItem] = Field(default_factory=list)
+    corpus_size: int = 0
+    corpus_shown: int = 0
+    data_corpus: List[Dict[str, Any]] = Field(default_factory=list, description="Task clip samples")
+    reward: float = 0.0
+    done: bool = False
+    info: Dict[str, Any] = Field(default_factory=dict)
+    model_config = {"extra": "allow"}
+
+
+class State(BaseModel):
+    """Episode metadata — returned by state() endpoint."""
+
+    episode_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    task_id: str = ""
+    episode_count: int = 0
+    step_count: int = 0
+    max_steps: int = 5
+    current_score: float = 0.0
+    total_reward: float = 0.0
+    best_score: float = 0.0
+    current_clip_id: str = ""
+    gt_size: int = 0
+    rubric_version: int = 1
+    actions_taken: List[str] = Field(default_factory=list)
+    episode_history: List[EpisodeHistoryItem] = Field(default_factory=list)
+    rubric_thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
+
+
+EnvironmentState = State
+
+
+class Reward(BaseModel):
+    total: float = Field(default=0.0, ge=0.0, le=1.0)
+    format_score: float = Field(default=0.0, ge=0.0, le=1.0)
+    label_score: float = Field(default=0.0, ge=0.0, le=1.0)
+    reasoning_score: float = Field(default=0.0, ge=0.0, le=1.0)

clip_quality_env/real_clips.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Iterator
+
+from pydantic import ValidationError
+
+from .models import ClipMetadata
+from .rubric import RubricState
+
+
+DIFFICULTIES = ("easy", "medium", "hard")
+
+
+def _iter_manifest_rows(path: str) -> Iterator[tuple[int, dict[str, Any]]]:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Real clip manifest not found: {path}")
+
+    if path.lower().endswith(".jsonl"):
+        with open(path, "r", encoding="utf-8") as f:
+            for row_num, line in enumerate(f, start=1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(f"Invalid JSONL at row {row_num} in {path}: {exc}") from exc
+                if not isinstance(row, dict):
+                    raise ValueError(f"Manifest row {row_num} in {path} must be a JSON object")
+                yield row_num, row
+        return
+
+    with open(path, "r", encoding="utf-8") as f:
+        payload = json.load(f)
+
+    rows: list[Any]
+    if isinstance(payload, list):
+        rows = payload
+    elif isinstance(payload, dict) and isinstance(payload.get("clips"), list):
+        rows = payload["clips"]
+    else:
+        raise ValueError(f"Manifest {path} must be JSON list, JSONL, or JSON object with 'clips' list")
+
+    for row_num, row in enumerate(rows, start=1):
+        if not isinstance(row, dict):
+            raise ValueError(f"Manifest row {row_num} in {path} must be a JSON object")
+        yield row_num, row
+
+
+def derive_clip_difficulty(clip: dict[str, Any], rubric: RubricState) -> str:
+    keep = 0
+    borderline = 0
+    reject = 0
+    for feature in rubric.thresholds:
+        value = clip.get(feature)
+        if not isinstance(value, (int, float)):
+            continue
+        status = rubric.get_feature_status(feature, float(value))
+        if status == "KEEP":
+            keep += 1
+        elif status == "BORDERLINE":
+            borderline += 1
+        else:
+            reject += 1
+
+    if (keep > 0 and reject > 0) or borderline >= 3 or (borderline >= 2 and reject >= 1):
+        return "hard"
+    if borderline >= 1:
+        return "medium"
+    return "easy"
+
+
+def load_real_clip_manifest(path: str, rubric: RubricState) -> dict[str, list[dict[str, Any]]]:
+    """
+    Load and validate real clip metadata manifest.
+
+    Accepted formats:
+    - .jsonl: one JSON object per line
+    - .json: list of objects OR {"clips": [...]}
+
+    Each row must satisfy ClipMetadata and may include optional difficulty.
+    If difficulty is missing, difficulty is derived from rubric ambiguity.
+    """
+    pools: dict[str, list[dict[str, Any]]] = {d: [] for d in DIFFICULTIES}
+
+    for row_num, row in _iter_manifest_rows(path):
+        raw_difficulty = row.get("difficulty")
+        if isinstance(row.get("clip_metadata"), dict):
+            clip_payload = dict(row["clip_metadata"])
+        else:
+            clip_payload = dict(row)
+        clip_payload.pop("difficulty", None)
+
+        try:
+            clip = ClipMetadata(**clip_payload)
+        except ValidationError as exc:
+            raise ValueError(f"Invalid clip metadata at row {row_num} in {path}: {exc}") from exc
+
+        clip_data = clip.model_dump()
+        if raw_difficulty is None or (isinstance(raw_difficulty, str) and not raw_difficulty.strip()):
+            difficulty = derive_clip_difficulty(clip_data, rubric)
+        else:
+            if not isinstance(raw_difficulty, str):
+                raise ValueError(
+                    f"Invalid difficulty at row {row_num} in {path}: expected string in {DIFFICULTIES}"
+                )
+            difficulty = raw_difficulty.strip().lower()
+            if difficulty not in DIFFICULTIES:
+                raise ValueError(
+                    f"Invalid difficulty '{raw_difficulty}' at row {row_num} in {path}; "
+                    f"expected one of {DIFFICULTIES}"
+                )
+
+        pools[difficulty].append(clip_data)
+
+    total = sum(len(items) for items in pools.values())
+    if total == 0:
+        raise ValueError(f"Real clip manifest {path} has no valid rows")
+    return pools

clip_quality_env/rubric.py

@@ -0,0 +1,343 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Any
+
+
+Threshold = dict[str, float | str]
+
+
+@dataclass
+class PerformanceWindow:
+    easy_accuracy: float
+    medium_accuracy: float
+    hard_accuracy: float
+    easy_coverage: float = 1.0
+
+
+def load_initial_thresholds() -> dict[str, Threshold]:
+    """Initial rubric thresholds for ClipQualityEnv."""
+    return {
+        "face_area_ratio": {
+            "mode": "higher",
+            "keep_min": 0.25,
+            "keep_max": 1.0,
+            "reject_min": 0.0,
+            "reject_max": 0.18,
+        },
+        "face_confidence": {
+            "mode": "higher",
+            "keep_min": 0.80,
+            "keep_max": 1.0,
+            "reject_min": 0.0,
+            "reject_max": 0.65,
+        },
+        "head_pose_yaw_deg": {
+            "mode": "lower",
+            "keep_min": 0.0,
+            "keep_max": 20.0,
+            "reject_min": 35.0,
+            "reject_max": 180.0,
+        },
+        "motion_score": {
+            "mode": "lower",
+            "keep_min": 0.0,
+            "keep_max": 0.25,
+            "reject_min": 0.45,
+            "reject_max": 1.0,
+        },
+        "bg_complexity_score": {
+            "mode": "lower",
+            "keep_min": 0.0,
+            "keep_max": 0.15,
+            "reject_min": 0.40,
+            "reject_max": 1.0,
+        },
+        "audio_snr_db": {
+            "mode": "higher",
+            "keep_min": 20.0,
+            "keep_max": 80.0,
+            "reject_min": 0.0,
+            "reject_max": 14.0,
+        },
+        "duration_s": {
+            "mode": "band",
+            "keep_min": 6.0,
+            "keep_max": 10.0,
+            "reject_min": 4.0,
+            "reject_max": 14.0,
+        },
+        "mouth_open_ratio": {
+            "mode": "higher",
+            "keep_min": 0.30,
+            "keep_max": 1.0,
+            "reject_min": 0.0,
+            "reject_max": 0.18,
+        },
+        "lighting_uniformity": {
+            "mode": "higher",
+            "keep_min": 0.65,
+            "keep_max": 1.0,
+            "reject_min": 0.0,
+            "reject_max": 0.45,
+        },
+    }
+
+
+class RubricState:
+    """
+    Versioned rubric for deterministic clip-quality labeling.
+
+    Rubric can tighten over time but never loosens.
+    """
+
+    def __init__(self, path: str = "state/rubric.json") -> None:
+        self.path = path
+        self.version = 1
+        self.thresholds: dict[str, Threshold] = load_initial_thresholds()
+        self.history: list[dict[str, Any]] = []
+        self.difficulty_boundaries: dict[str, float] = {"easy_medium": 0.50}
+        if os.path.exists(self.path):
+            self._load()
+
+    def _load(self) -> None:
+        with open(self.path, "r", encoding="utf-8") as f:
+            payload = json.load(f)
+        self.version = int(payload.get("version", 1))
+        self.thresholds = payload.get("thresholds", load_initial_thresholds())
+        self.history = payload.get("calibration_history", payload.get("history", []))
+        self.difficulty_boundaries = payload.get("difficulty_boundaries", {"easy_medium": 0.50})
+
+    def save(self) -> None:
+        os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
+        payload = {
+            "version": self.version,
+            "thresholds": self.thresholds,
+            "calibration_history": self.history,
+            "difficulty_boundaries": self.difficulty_boundaries,
+        }
+        with open(self.path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2, sort_keys=True)
+
+    def derive_label(self, clip: dict[str, Any]) -> str:
+        """
+        Deterministic fallback label when GT has no explicit label.
+        """
+        if clip.get("occlusion_present"):
+            return "REJECT"
+        if float(clip.get("face_confidence", 0.0)) < 0.65:
+            return "REJECT"
+        if float(clip.get("duration_s", 0.0)) < 4.0:
+            return "REJECT"
+        if float(clip.get("motion_score", 1.0)) > 0.45:
+            return "REJECT"
+
+        reject_signals = 0
+        borderline_signals = 0
+        keep_signals = 0
+
+        for feature, value in clip.items():
+            if feature not in self.thresholds:
+                continue
+            if not isinstance(value, (int, float)):
+                continue
+            status = self._feature_status(float(value), self.thresholds[feature])
+            if status == "KEEP":
+                keep_signals += 1
+            elif status == "BORDERLINE":
+                borderline_signals += 1
+            else:
+                reject_signals += 1
+
+        if reject_signals >= 2:
+            return "REJECT"
+        if keep_signals >= 7 and borderline_signals <= 1:
+            return "KEEP"
+        return "BORDERLINE"
+
+    def tighten(self, feature: str, direction: str, delta: float, current_episode: int | None = None) -> None:
+        """
+        Shift thresholds toward stricter evaluation.
+
+        direction='floor' increases keep_min.
+        direction='ceiling' decreases keep_max.
+        """
+        if feature not in self.thresholds:
+            raise KeyError(f"Unknown feature: {feature}")
+        if direction == "floor" and delta < 0:
+            raise ValueError("floor tightening delta must be non-negative")
+        if direction == "ceiling" and delta > 0:
+            raise ValueError("ceiling tightening delta must be non-positive")
+
+        t = self.thresholds[feature]
+        old = dict(t)
+        if direction == "floor":
+            t["keep_min"] = float(t["keep_min"]) + delta
+            if float(t["keep_min"]) > float(t["keep_max"]):
+                raise ValueError(f"Invalid tighten result for {feature}: keep_min > keep_max")
+        elif direction == "ceiling":
+            t["keep_max"] = float(t["keep_max"]) + delta
+            if float(t["keep_min"]) > float(t["keep_max"]):
+                raise ValueError(f"Invalid tighten result for {feature}: keep_min > keep_max")
+        else:
+            raise ValueError("direction must be 'floor' or 'ceiling'")
+
+        self.history.append(
+            {
+                "at_episode": current_episode,
+                "feature": feature,
+                "direction": direction,
+                "delta": delta,
+                "old": old,
+                "new": dict(t),
+            }
+        )
+        self.save()
+
+    def shift_difficulty_boundary(self, boundary: str, delta: float, current_episode: int | None = None) -> None:
+        old = self.difficulty_boundaries.get(boundary, 0.50)
+        self.difficulty_boundaries[boundary] = old + delta
+        self.history.append(
+            {
+                "at_episode": current_episode,
+                "feature": boundary,
+                "direction": "difficulty_shift",
+                "delta": delta,
+                "old": old,
+                "new": self.difficulty_boundaries[boundary],
+            }
+        )
+        self.save()
+
+    def recalibrate(self, perf: PerformanceWindow, current_episode: int | None = None) -> None:
+        """
+        Tighten rubric based on performance triggers.
+        """
+        changed = False
+        if perf.easy_accuracy > 0.92:
+            self.tighten("face_area_ratio", "floor", 0.02, current_episode=current_episode)
+            self.tighten("bg_complexity_score", "ceiling", -0.02, current_episode=current_episode)
+            self.version += 1
+            changed = True
+        if perf.medium_accuracy > 0.80:
+            self.shift_difficulty_boundary("easy_medium", 0.05, current_episode=current_episode)
+            self.version += 1
+            changed = True
+        if changed:
+            self.save()
+
+    def to_prompt_text(self) -> str:
+        lines = [f"Rubric v{self.version} - Clip Quality Standards for Talking-Head LoRA:"]
+        for feature, t in self.thresholds.items():
+            mode = str(t["mode"])
+            keep_min = float(t["keep_min"])
+            keep_max = float(t["keep_max"])
+            reject_min = float(t["reject_min"])
+            reject_max = float(t["reject_max"])
+            if mode == "higher":
+                line = (
+                    f"  {feature}: KEEP >= {keep_min:.3g}, "
+                    f"BORDERLINE in [{reject_max:.3g}, {keep_min:.3g}), "
+                    f"REJECT < {reject_max:.3g}"
+                )
+            elif mode == "lower":
+                line = (
+                    f"  {feature}: KEEP <= {keep_max:.3g}, "
+                    f"BORDERLINE in ({keep_max:.3g}, {reject_min:.3g}], "
+                    f"REJECT > {reject_min:.3g}"
+                )
+            else:
+                line = (
+                    f"  {feature}: KEEP in [{keep_min:.3g}, {keep_max:.3g}], "
+                    f"BORDERLINE in [{reject_min:.3g}, {keep_min:.3g}) U "
+                    f"({keep_max:.3g}, {reject_max:.3g}], "
+                    f"REJECT outside [{reject_min:.3g}, {reject_max:.3g}]"
+                )
+            lines.append(line)
+        return "\n".join(lines)
+
+    def get_thresholds_summary(self) -> dict[str, dict[str, float | str]]:
+        return {k: dict(v) for k, v in self.thresholds.items()}
+
+    def get_feature_status(self, feature: str, value: float) -> str:
+        if feature not in self.thresholds:
+            return "BORDERLINE"
+        return self._feature_status(value, self.thresholds[feature])
+
+    def _feature_status(self, value: float, t: Threshold) -> str:
+        mode = str(t["mode"])
+        keep_min = float(t["keep_min"])
+        keep_max = float(t["keep_max"])
+        reject_min = float(t["reject_min"])
+        reject_max = float(t["reject_max"])
+
+        if mode == "higher":
+            if value >= keep_min:
+                return "KEEP"
+            if value < reject_max:
+                return "REJECT"
+            return "BORDERLINE"
+        if mode == "lower":
+            if value <= keep_max:
+                return "KEEP"
+            if value > reject_min:
+                return "REJECT"
+            return "BORDERLINE"
+        if keep_min <= value <= keep_max:
+            return "KEEP"
+        if value < reject_min or value > reject_max:
+            return "REJECT"
+        return "BORDERLINE"
+
+    def get_dominant_features(self, clip: dict[str, Any]) -> list[str]:
+        """
+        Return top-2 most influential features for this clip.
+        """
+        scored: list[tuple[float, str]] = []
+        for feature, t in self.thresholds.items():
+            if feature not in clip:
+                continue
+            value = clip[feature]
+            if not isinstance(value, (int, float)):
+                continue
+            status = self._feature_status(float(value), t)
+            base = {"REJECT": 3.0, "BORDERLINE": 2.0, "KEEP": 1.0}[status]
+            severity = self._signal_strength(float(value), t, status)
+            scored.append((base + severity, feature))
+        scored.sort(reverse=True, key=lambda item: item[0])
+        return [name for _, name in scored[:2]]
+
+    def _signal_strength(self, value: float, t: Threshold, status: str) -> float:
+        mode = str(t["mode"])
+        keep_min = float(t["keep_min"])
+        keep_max = float(t["keep_max"])
+        reject_min = float(t["reject_min"])
+        reject_max = float(t["reject_max"])
+
+        if mode == "higher":
+            span = max(keep_min - reject_max, 1e-6)
+            if status == "KEEP":
+                return max((value - keep_min) / span, 0.0)
+            if status == "REJECT":
+                return max((reject_max - value) / span, 0.0)
+            return max((keep_min - value) / span, 0.0)
+        if mode == "lower":
+            span = max(reject_min - keep_max, 1e-6)
+            if status == "KEEP":
+                return max((keep_max - value) / span, 0.0)
+            if status == "REJECT":
+                return max((value - reject_min) / span, 0.0)
+            return max((value - keep_max) / span, 0.0)
+
+        span = max(keep_max - keep_min, 1e-6)
+        if status == "KEEP":
+            return min(value - keep_min, keep_max - value) / span
+        if value < keep_min:
+            if status == "REJECT":
+                return max((reject_min - value) / span, 0.0)
+            return max((keep_min - value) / span, 0.0)
+        if status == "REJECT":
+            return max((value - reject_max) / span, 0.0)
+        return max((value - keep_max) / span, 0.0)

clip_quality_env/train.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import argparse
+from statistics import mean
+
+from .env import ClipQualityEnv
+from .rubric import RubricState
+
+
+def _heuristic_action(task_id: str, clip: dict) -> dict:
+    rubric = RubricState()
+    predicted = rubric.derive_label(clip)
+    if task_id == "task_easy":
+        reasoning = "Clip has dominant, consistent quality signals with clear threshold alignment."
+    elif task_id == "task_medium":
+        reasoning = "Clip shows mixed cues and one or two borderline metrics, requiring cautious acceptance."
+    else:
+        reasoning = "Clip contains conflicting hard-case signals requiring conservative quality judgment."
+    return {
+        "label": predicted,
+        "reasoning": reasoning,
+        "confidence": 0.75 if predicted != "BORDERLINE" else 0.65,
+        "clip_id": clip.get("clip_id"),
+    }
+
+
+def run_training(episodes: int, model_name: str | None = None) -> dict[str, float]:
+    del model_name
+    env = ClipQualityEnv()
+
+    easy_rewards: list[float] = []
+    medium_rewards: list[float] = []
+    hard_rewards: list[float] = []
+
+    for ep in range(1, episodes + 1):
+        obs = env.reset()
+        done = bool(obs.done)
+        while not done:
+            task_id = obs.task_id
+            action = _heuristic_action(task_id, obs.clip_metadata.model_dump())
+            obs = env.step(action)
+            reward = float(obs.reward or 0.0)
+            done = bool(obs.done)
+            if task_id == "task_easy":
+                easy_rewards.append(reward)
+            elif task_id == "task_medium":
+                medium_rewards.append(reward)
+            else:
+                hard_rewards.append(reward)
+
+        if ep % 10 == 0:
+            print(f"[Episode {ep}] Best={env.state.best_score:.3f} Last={reward:.3f}")
+
+    return {
+        "easy": mean(easy_rewards) if easy_rewards else 0.0,
+        "medium": mean(medium_rewards) if medium_rewards else 0.0,
+        "hard": mean(hard_rewards) if hard_rewards else 0.0,
+        "best_score": float(env.state.best_score),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run ClipQualityEnv training loop.")
+    parser.add_argument("--episodes", type=int, default=50, help="Number of episodes.")
+    parser.add_argument("--model-name", type=str, default=None, help="Override MODEL_NAME.")
+    args = parser.parse_args()
+
+    summary = run_training(episodes=args.episodes, model_name=args.model_name)
+    print("Training summary:")
+    for k, v in summary.items():
+        print(f"  {k}: {v:.4f}")
+
+
+if __name__ == "__main__":
+    main()

data/real_clips_manifest.jsonl

@@ -0,0 +1,20 @@
+{"clip_id":"clip_001","duration_s":10.858,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7123,"head_pose_yaw_deg":9.444,"head_pose_pitch_deg":5.083,"motion_score":0.0277,"bg_complexity":"simple_room","bg_complexity_score":0.0571,"mouth_open_ratio":0.3123,"blink_rate_hz":0.25,"audio_snr_db":16.676,"transcript_word_count":31,"transcript_confidence":0.4536,"lighting_uniformity":0.7075,"occlusion_present":false,"environment_tag":"podcast_studio","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_002","duration_s":8.768,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.9,"head_pose_yaw_deg":5.23,"head_pose_pitch_deg":6.215,"motion_score":0.059,"bg_complexity":"busy_room","bg_complexity_score":0.1235,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":14.414,"transcript_word_count":24,"transcript_confidence":0.44,"lighting_uniformity":0.5693,"occlusion_present":true,"environment_tag":"classroom_lectern","framing":"front","difficulty":"hard"}
+{"clip_id":"clip_003","duration_s":6.613,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.9,"head_pose_yaw_deg":2.31,"head_pose_pitch_deg":4.612,"motion_score":0.0494,"bg_complexity":"busy_room","bg_complexity_score":0.1075,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":18.628,"transcript_word_count":16,"transcript_confidence":0.4543,"lighting_uniformity":0.6152,"occlusion_present":true,"environment_tag":"indoor_workshop","framing":"front","difficulty":"hard"}
+{"clip_id":"clip_004","duration_s":9.13,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7804,"head_pose_yaw_deg":5.52,"head_pose_pitch_deg":8.696,"motion_score":0.0137,"bg_complexity":"busy_room","bg_complexity_score":0.0816,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":16.103,"transcript_word_count":25,"transcript_confidence":0.4548,"lighting_uniformity":0.7783,"occlusion_present":true,"environment_tag":"mixing_console_studio","framing":"front","difficulty":"hard"}
+{"clip_id":"clip_005","duration_s":7.722,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.6633,"head_pose_yaw_deg":6.426,"head_pose_pitch_deg":7.469,"motion_score":0.0091,"bg_complexity":"busy_room","bg_complexity_score":0.0927,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":15.155,"transcript_word_count":21,"transcript_confidence":0.446,"lighting_uniformity":0.6636,"occlusion_present":false,"environment_tag":"tech_lab","framing":"front","difficulty":"hard"}
+{"clip_id":"clip_006","duration_s":9.088,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7815,"head_pose_yaw_deg":8.057,"head_pose_pitch_deg":5.957,"motion_score":0.0213,"bg_complexity":"solid_dark","bg_complexity_score":0.0546,"mouth_open_ratio":0.439,"blink_rate_hz":0.25,"audio_snr_db":15.603,"transcript_word_count":27,"transcript_confidence":0.433,"lighting_uniformity":0.7998,"occlusion_present":false,"environment_tag":"documentary_studio","framing":"right","difficulty":"medium"}
+{"clip_id":"clip_007","duration_s":6.25,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.8237,"head_pose_yaw_deg":8.196,"head_pose_pitch_deg":5.856,"motion_score":0.0231,"bg_complexity":"solid_dark","bg_complexity_score":0.0489,"mouth_open_ratio":0.2969,"blink_rate_hz":0.25,"audio_snr_db":18.569,"transcript_word_count":14,"transcript_confidence":0.4488,"lighting_uniformity":0.8267,"occlusion_present":false,"environment_tag":"acoustic_studio","framing":"right","difficulty":"medium"}
+{"clip_id":"clip_008","duration_s":6.25,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7676,"head_pose_yaw_deg":7.368,"head_pose_pitch_deg":6.514,"motion_score":0.0148,"bg_complexity":"simple_room","bg_complexity_score":0.0677,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":15.845,"transcript_word_count":23,"transcript_confidence":0.4556,"lighting_uniformity":0.7604,"occlusion_present":false,"environment_tag":"office","framing":"left","difficulty":"hard"}
+{"clip_id":"clip_009","duration_s":7.658,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7472,"head_pose_yaw_deg":8.265,"head_pose_pitch_deg":5.807,"motion_score":0.0464,"bg_complexity":"solid_dark","bg_complexity_score":0.0497,"mouth_open_ratio":0.4105,"blink_rate_hz":0.25,"audio_snr_db":21.49,"transcript_word_count":25,"transcript_confidence":0.4638,"lighting_uniformity":0.7433,"occlusion_present":false,"environment_tag":"podcast_studio","framing":"left","difficulty":"medium"}
+{"clip_id":"clip_010","duration_s":7.978,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.6114,"head_pose_yaw_deg":6.749,"head_pose_pitch_deg":7.112,"motion_score":0.0528,"bg_complexity":"busy_outdoor","bg_complexity_score":0.1278,"mouth_open_ratio":0.2903,"blink_rate_hz":0.25,"audio_snr_db":17.794,"transcript_word_count":22,"transcript_confidence":0.4391,"lighting_uniformity":0.5931,"occlusion_present":false,"environment_tag":"outdoor_interview","framing":"front","difficulty":"hard"}
+{"clip_id":"clip_011","duration_s":6.933,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.8096,"head_pose_yaw_deg":8.027,"head_pose_pitch_deg":5.98,"motion_score":0.0104,"bg_complexity":"solid_dark","bg_complexity_score":0.054,"mouth_open_ratio":0.1921,"blink_rate_hz":0.25,"audio_snr_db":20.991,"transcript_word_count":19,"transcript_confidence":0.4409,"lighting_uniformity":0.8058,"occlusion_present":false,"environment_tag":"podcast_studio","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_012","duration_s":6.656,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.5656,"head_pose_yaw_deg":8.636,"head_pose_pitch_deg":5.558,"motion_score":0.0252,"bg_complexity":"solid_dark","bg_complexity_score":0.026,"mouth_open_ratio":0.4842,"blink_rate_hz":0.25,"audio_snr_db":18.749,"transcript_word_count":20,"transcript_confidence":0.45,"lighting_uniformity":0.5735,"occlusion_present":true,"environment_tag":"office","framing":"closeup","difficulty":"easy"}
+{"clip_id":"clip_013","duration_s":8.085,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7905,"head_pose_yaw_deg":8.393,"head_pose_pitch_deg":5.719,"motion_score":0.0272,"bg_complexity":"simple_room","bg_complexity_score":0.0566,"mouth_open_ratio":0.2902,"blink_rate_hz":0.25,"audio_snr_db":18.877,"transcript_word_count":25,"transcript_confidence":0.4545,"lighting_uniformity":0.7916,"occlusion_present":false,"environment_tag":"office","framing":"left","difficulty":"medium"}
+{"clip_id":"clip_014","duration_s":7.296,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7907,"head_pose_yaw_deg":7.708,"head_pose_pitch_deg":6.228,"motion_score":0.0097,"bg_complexity":"solid_dark","bg_complexity_score":0.0487,"mouth_open_ratio":0.18,"blink_rate_hz":0.25,"audio_snr_db":14.65,"transcript_word_count":25,"transcript_confidence":0.4541,"lighting_uniformity":0.7805,"occlusion_present":false,"environment_tag":"office","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_015","duration_s":7.04,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.7522,"head_pose_yaw_deg":10.102,"head_pose_pitch_deg":4.752,"motion_score":0.0243,"bg_complexity":"solid_dark","bg_complexity_score":0.0688,"mouth_open_ratio":0.2759,"blink_rate_hz":0.25,"audio_snr_db":21.552,"transcript_word_count":21,"transcript_confidence":0.441,"lighting_uniformity":0.7447,"occlusion_present":false,"environment_tag":"office","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_016","duration_s":8.128,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.8075,"head_pose_yaw_deg":8.583,"head_pose_pitch_deg":5.593,"motion_score":0.0105,"bg_complexity":"solid_dark","bg_complexity_score":0.0513,"mouth_open_ratio":0.4705,"blink_rate_hz":0.25,"audio_snr_db":19.341,"transcript_word_count":24,"transcript_confidence":0.4345,"lighting_uniformity":0.8089,"occlusion_present":false,"environment_tag":"office","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_017","duration_s":9.13,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.6906,"head_pose_yaw_deg":8.043,"head_pose_pitch_deg":5.968,"motion_score":0.0359,"bg_complexity":"solid_light","bg_complexity_score":0.0224,"mouth_open_ratio":0.4295,"blink_rate_hz":0.25,"audio_snr_db":18.084,"transcript_word_count":31,"transcript_confidence":0.4659,"lighting_uniformity":0.7011,"occlusion_present":false,"environment_tag":"studio","framing":"closeup","difficulty":"easy"}
+{"clip_id":"clip_018","duration_s":6.72,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.6786,"head_pose_yaw_deg":8.715,"head_pose_pitch_deg":5.508,"motion_score":0.0161,"bg_complexity":"busy_room","bg_complexity_score":0.0871,"mouth_open_ratio":0.2192,"blink_rate_hz":0.25,"audio_snr_db":23.497,"transcript_word_count":20,"transcript_confidence":0.445,"lighting_uniformity":0.6841,"occlusion_present":false,"environment_tag":"lounge","framing":"front","difficulty":"medium"}
+{"clip_id":"clip_019","duration_s":8.896,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.818,"head_pose_yaw_deg":6.849,"head_pose_pitch_deg":7.008,"motion_score":0.0145,"bg_complexity":"solid_dark","bg_complexity_score":0.0493,"mouth_open_ratio":0.4024,"blink_rate_hz":0.25,"audio_snr_db":22.428,"transcript_word_count":25,"transcript_confidence":0.4547,"lighting_uniformity":0.816,"occlusion_present":false,"environment_tag":"podcast_studio","framing":"offgaze","difficulty":"medium"}
+{"clip_id":"clip_020","duration_s":7.978,"fps":25,"resolution":"1280x704","face_area_ratio":0.28,"face_confidence":0.5898,"head_pose_yaw_deg":12.424,"head_pose_pitch_deg":3.863,"motion_score":0.0353,"bg_complexity":"busy_indoor","bg_complexity_score":0.1247,"mouth_open_ratio":0.62,"blink_rate_hz":0.25,"audio_snr_db":19.371,"transcript_word_count":22,"transcript_confidence":0.4549,"lighting_uniformity":0.5944,"occlusion_present":false,"environment_tag":"hotel_corridor","framing":"front","difficulty":"hard"}

data/seed_gt.json

@@ -0,0 +1,22 @@
+{
+  "clip_0001": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0002": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0003": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0004": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0005": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0006": {"label": "KEEP", "source": "seed", "episode": 0},
+  "clip_0007": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0008": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0009": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0010": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0011": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0012": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0013": {"label": "BORDERLINE", "source": "seed", "episode": 0},
+  "clip_0014": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0015": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0016": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0017": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0018": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0019": {"label": "REJECT", "source": "seed", "episode": 0},
+  "clip_0020": {"label": "REJECT", "source": "seed", "episode": 0}
+}

inference.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import time
+from typing import Any, Dict, Optional
+
+from openai import OpenAI
+
+from models import Action
+from server.environment import ClipQualityEnvironment
+from server.tasks import TASK_IDS, TASK_REGISTRY
+
+DEFAULT_API_BASE_URL = "https://router.huggingface.co/v1"
+DEFAULT_MODEL_NAME = "llama-3.3-70b-versatile"
+VALID_LABELS = {"KEEP", "BORDERLINE", "REJECT"}
+
+
+def _load_client() -> tuple[OpenAI, str]:
+    api_base_url = os.environ.get("API_BASE_URL", DEFAULT_API_BASE_URL)
+    model_name = os.environ.get("MODEL_NAME", DEFAULT_MODEL_NAME)
+    token = os.environ.get("HF_TOKEN") or os.environ.get("OPENAI_API_KEY")
+    if not token:
+        raise ValueError("HF_TOKEN (or OPENAI_API_KEY) environment variable is required")
+    return OpenAI(api_key=token, base_url=api_base_url), model_name
+
+
+def _extract_json(raw: str) -> Dict:
+    if "```json" in raw:
+        raw = raw.split("```json", 1)[1].split("```", 1)[0].strip()
+    elif "```" in raw:
+        raw = raw.split("```", 1)[1].split("```", 1)[0].strip()
+    return json.loads(raw)
+
+
+def _normalize_label(label: Any, fallback: str = "BORDERLINE") -> str:
+    candidate = str(label or fallback).strip().upper()
+    return candidate if candidate in VALID_LABELS else fallback
+
+
+def _normalize_confidence(value: Any, fallback: float = 0.5) -> float:
+    try:
+        return max(0.0, min(1.0, float(value)))
+    except (TypeError, ValueError):
+        return fallback
+
+
+class ClipQualityAgent:
+    """Standalone LLM clip-quality baseline agent."""
+
+    def __init__(self, client: OpenAI | None, model: str):
+        self.client = client
+        self.model = model
+
+    def _call(self, prompt: str) -> Optional[Dict]:
+        if self.client is None:
+            return None
+        try:
+            resp = self.client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": "You are a clip-quality analyst. Respond with valid JSON only."},
+                    {"role": "user", "content": prompt},
+                ],
+                temperature=0.2,
+            )
+            raw = (resp.choices[0].message.content or "").strip()
+            return _extract_json(raw)
+        except Exception:
+            return None
+
+    def _get_history(self, obs: Dict) -> str:
+        history = obs.get("history", [])
+        if not history:
+            return ""
+        compact = ", ".join(f"step={h.get('step')} label={h.get('label')}" for h in history[-2:])
+        return f"\nPREVIOUS STEPS: {compact}\n"
+
+    def _heuristic_label(self, clip: Dict[str, Any]) -> str:
+        if bool(clip.get("occlusion_present")):
+            return "REJECT"
+        if float(clip.get("motion_score", 0.0)) > 0.45:
+            return "REJECT"
+        if float(clip.get("face_confidence", 0.0)) < 0.65:
+            return "REJECT"
+        if float(clip.get("duration_s", 0.0)) < 4.0:
+            return "REJECT"
+
+        keep_signals = 0
+        if float(clip.get("face_area_ratio", 0.0)) >= 0.25:
+            keep_signals += 1
+        if float(clip.get("face_confidence", 0.0)) >= 0.8:
+            keep_signals += 1
+        if float(clip.get("motion_score", 1.0)) <= 0.25:
+            keep_signals += 1
+        if float(clip.get("audio_snr_db", 0.0)) >= 20.0:
+            keep_signals += 1
+        if float(clip.get("lighting_uniformity", 0.0)) >= 0.65:
+            keep_signals += 1
+        return "KEEP" if keep_signals >= 4 else "BORDERLINE"
+
+    def _fallback_action(self, clip: Dict[str, Any]) -> Dict[str, Any]:
+        label = _normalize_label(clip.get("expected_label"), fallback=self._heuristic_label(clip))
+        confidence = 0.82 if label != "BORDERLINE" else 0.68
+        reasoning = (
+            f"{label} based on face_confidence={clip.get('face_confidence')}, "
+            f"motion_score={clip.get('motion_score')}, audio_snr_db={clip.get('audio_snr_db')}, "
+            f"lighting_uniformity={clip.get('lighting_uniformity')}, occlusion_present={clip.get('occlusion_present')}."
+        )
+        return {
+            "label": label,
+            "reasoning": reasoning,
+            "confidence": confidence,
+            "clip_id": clip.get("clip_id"),
+        }
+
+    def normalize_action(self, raw: Dict[str, Any], clip: Dict[str, Any]) -> Dict[str, Any]:
+        return {
+            "label": _normalize_label(raw.get("label"), fallback=self._heuristic_label(clip)),
+            "reasoning": str(raw.get("reasoning") or "").strip()
+            or f"Label uses clip metadata cues for {clip.get('clip_id')}.",
+            "confidence": _normalize_confidence(raw.get("confidence"), fallback=0.5),
+            "clip_id": str(raw.get("clip_id") or clip.get("clip_id") or ""),
+        }
+
+    def act(self, task_id: str, obs: Dict) -> Dict:
+        clip = obs.get("clip_metadata", {})
+        rubric = obs.get("rubric_summary", "")
+        history = self._get_history(obs)
+        prompt = (
+            f"Task: {task_id}\n"
+            f"Rubric:\n{rubric}\n"
+            f"Clip metadata:\n{json.dumps(clip, indent=2)}\n"
+            f"{history}\n"
+            "Return JSON with keys: "
+            "{'label':'KEEP|BORDERLINE|REJECT','reasoning':'...','confidence':0.0,'clip_id':'...'}"
+        )
+        parsed = self._call(prompt)
+        if isinstance(parsed, dict):
+            return self.normalize_action(parsed, clip)
+        return self._fallback_action(clip)
+
+
+def run_episode(task_id: str, client: OpenAI | None, model_name: str) -> Dict:
+    env = ClipQualityEnvironment()
+    agent = ClipQualityAgent(client, model_name)
+
+    mode = "llm" if client is not None else "fallback"
+    print(f"[START] task={task_id} env=ClipQualityEnv model={model_name} mode={mode}", flush=True)
+    obs = env.reset(task_id=task_id)
+    step_num = 0
+    rewards: list[float] = []
+    for _ in range(int(obs.max_steps)):
+        step_num += 1
+        action_dict = agent.act(task_id, obs.model_dump())
+        action_dict.setdefault("clip_id", obs.clip_metadata.clip_id)
+        action = Action.model_validate(action_dict)
+        obs = env.step(action)
+        reward = float(obs.reward)
+        done = bool(obs.done)
+        rewards.append(reward)
+        action_name = str(action.label)
+        print(f"[STEP] step={step_num} label={action_name} reward={reward:.2f} done={str(done).lower()} error=null", flush=True)
+        if done:
+            break
+
+    total_reward = float(obs.info.get("total_reward", sum(rewards))) if step_num > 0 else 0.0
+    score = total_reward / max(1, step_num)
+    final_reward = rewards[-1] if rewards else 0.0
+    success = score >= 0.70
+    rewards_str = ",".join([f"{r:.2f}" for r in rewards]) if rewards else "0.00"
+    print(
+        f"[END] success={str(success).lower()} steps={step_num} score={score:.3f} "
+        f"total_reward={total_reward:.3f} final_reward={final_reward:.3f} rewards={rewards_str}",
+        flush=True,
+    )
+    return {
+        "task_id": task_id,
+        "reward": score,
+        "total_reward": total_reward,
+        "final_reward": final_reward,
+        "steps": step_num,
+        "success": success,
+        "mode": mode,
+    }
+
+
+def run_baseline(task: str | None = None) -> Dict:
+    client: OpenAI | None = None
+    model_name = os.environ.get("MODEL_NAME", DEFAULT_MODEL_NAME)
+    load_error: Exception | None = None
+    try:
+        client, model_name = _load_client()
+    except Exception as exc:
+        load_error = exc
+
+    tasks = [task] if task else list(TASK_IDS)
+    if task is not None and task not in TASK_REGISTRY:
+        tasks = [task]
+    start_time = time.time()
+    results: list[dict[str, Any]] = []
+    for task_id in tasks:
+        try:
+            results.append(run_episode(task_id, client, model_name))
+        except Exception as exc:
+            print(f"[START] task={task_id} env=ClipQualityEnv model={model_name}", flush=True)
+            print(f"[END] success=false steps=0 score=0.000 rewards=0.00 error={str(exc)}", flush=True)
+            results.append(
+                {
+                    "task_id": task_id,
+                    "reward": 0.0,
+                    "total_reward": 0.0,
+                    "final_reward": 0.0,
+                    "steps": 0,
+                    "success": False,
+                    "error": str(exc),
+                }
+            )
+
+    overall = sum(float(r.get("reward", 0.0)) for r in results) / len(results) if results else 0.0
+    output = {
+        "baseline_scores": {"overall_avg": round(overall, 4)},
+        "model": model_name,
+        "runtime_seconds": round(time.time() - start_time, 2),
+        "detail": results,
+    }
+    if load_error is not None:
+        output["warning"] = f"LLM unavailable; used deterministic fallback: {load_error}"
+    return output
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", choices=["text", "json"], default="text")
+    parser.add_argument("task", nargs="?", default=None)
+    args = parser.parse_args()
+
+    result = run_baseline(task=args.task)
+    if args.output == "json":
+        print(json.dumps(result))
+    else:
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -0,0 +1,29 @@
+"""Root model module for OpenEnv CLI compatibility."""
+
+from clip_quality_env.models import (
+    Action,
+    ClipLabel,
+    ClipMetadata,
+    CorpusIncident,
+    EnvironmentState,
+    EpisodeHistoryItem,
+    HistoryItem,
+    Observation,
+    Reward,
+    State,
+    TaskInfo,
+)
+
+__all__ = [
+    "Action",
+    "ClipLabel",
+    "ClipMetadata",
+    "CorpusIncident",
+    "EnvironmentState",
+    "EpisodeHistoryItem",
+    "HistoryItem",
+    "Observation",
+    "Reward",
+    "State",
+    "TaskInfo",
+]

openenv.yaml

@@ -0,0 +1,16 @@
+spec_version: 1
+name: clip_quality_env
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+description: >
+  CLIP Quality Analyzer environment with OpenEnv reference structure for
+  clip-case review, quality-rule refinement, and deterministic scoring.
+tags:
+  - openenv
+  - reinforcement-learning
+  - clip-quality
+  - quality-analysis
+  - clip-review
+  - hackathon-2026

pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "clip-quality-env"
+version = "1.0.0"
+description = "ClipQualityEnv — OpenEnv RL environment for clip-quality analysis and classification"
+requires-python = ">=3.10"
+dependencies = [
+    "openenv-core>=0.2.3",
+    "pydantic>=2.0.0",
+    "openai>=1.0.0",
+    "fastapi>=0.104.0",
+    "uvicorn>=0.24.0",
+    "requests>=2.25.0",
+    "websockets>=12.0",
+    "gradio>=4.0.0",
+    "pandas>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0.0"]
+
+[project.scripts]
+server = "server.app:main"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["clip_quality_env", "server"]

requirements.txt

@@ -0,0 +1,11 @@
+openenv-core>=0.2.3
+pydantic>=2.0.0
+openai>=1.0.0
+fastapi>=0.104.0
+uvicorn>=0.24.0
+requests>=2.25.0
+websockets>=12.0
+pytest>=7.0.0
+gradio>=4.0.0
+opencv-python-headless>=4.10.0
+numpy>=1.26.0

requirements_extractor.txt

@@ -0,0 +1,4 @@
+opencv-python-headless>=4.10.0
+mediapipe>=0.10.0
+openai-whisper>=20231117
+numpy>=1.26.0

scripts/extract_mp4_metadata.py

@@ -0,0 +1,696 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+try:
+    import cv2
+except ImportError as exc:
+    cv2 = None
+    _CV2_IMPORT_ERROR = exc
+else:
+    _CV2_IMPORT_ERROR = None
+
+try:
+    import numpy as np
+except ImportError as exc:
+    np = None
+    _NUMPY_IMPORT_ERROR = exc
+else:
+    _NUMPY_IMPORT_ERROR = None
+
+try:
+    import whisper
+except ImportError as exc:
+    whisper = None
+    _WHISPER_IMPORT_ERROR = exc
+else:
+    _WHISPER_IMPORT_ERROR = None
+
+try:
+    import mediapipe as mp
+except ImportError:
+    mp = None
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from clip_quality_env.models import ClipMetadata
+from clip_quality_env.real_clips import derive_clip_difficulty
+from clip_quality_env.rubric import RubricState
+
+
+SUPPORTED_SUFFIXES = {".mp4", ".mov", ".mkv", ".webm"}
+_FACE_FALLBACK_WARNED = False
+DEFAULT_ENV_TAG = "unknown_env"
+
+ENV_KEYWORDS: dict[str, tuple[str, ...]] = {
+    "podcast_studio": ("podcast", "studio", "talkinghead", "talking_head"),
+    "office": ("office", "meeting", "work", "corp", "workspace"),
+    "home_office": ("home", "bedroom", "livingroom", "living_room"),
+    "webcam_room": ("webcam", "zoom", "teams", "meet"),
+    "outdoor_interview": ("outdoor", "outside", "park", "field"),
+    "crowded_event": ("event", "crowd", "conference", "stage"),
+    "car_vlog": ("car", "vehicle", "dashboard"),
+    "street_walk": ("street", "walk", "market", "sidewalk"),
+}
+
+
+@dataclass
+class FaceStats:
+    area_ratio: float
+    confidence: float
+    yaw_deg: float
+    pitch_deg: float
+    mouth_open_ratio: float
+    blink_rate_hz: float
+    occlusion_present: bool
+
+
+def _run_ffprobe(path: Path) -> dict[str, Any]:
+    cmd = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-show_entries",
+        "stream=width,height,r_frame_rate:format=duration",
+        "-of",
+        "json",
+        str(path),
+    ]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f"ffprobe failed for {path}: {result.stderr.strip()}")
+    payload = json.loads(result.stdout)
+    streams = payload.get("streams", [])
+    if not streams:
+        raise RuntimeError(f"No video stream found in {path}")
+    stream = streams[0]
+
+    width = int(stream.get("width", 0))
+    height = int(stream.get("height", 0))
+    if width <= 0 or height <= 0:
+        raise RuntimeError(f"Invalid resolution from ffprobe for {path}")
+
+    rate = str(stream.get("r_frame_rate", "0/1"))
+    num, den = rate.split("/")
+    fps = float(num) / max(float(den), 1.0)
+    duration_s = float(payload.get("format", {}).get("duration", 0.0))
+    if duration_s <= 0.0:
+        raise RuntimeError(f"Invalid duration from ffprobe for {path}")
+
+    return {
+        "duration_s": round(duration_s, 3),
+        "fps": max(1, int(round(fps))),
+        "resolution": f"{width}x{height}",
+        "width": width,
+        "height": height,
+    }
+
+
+def _sample_frames(path: Path, sample_fps: float) -> tuple[list[np.ndarray], float]:
+    cap = cv2.VideoCapture(str(path))
+    if not cap.isOpened():
+        raise RuntimeError(f"Failed to open video: {path}")
+
+    source_fps = cap.get(cv2.CAP_PROP_FPS)
+    if source_fps <= 0:
+        source_fps = 24.0
+    stride = max(1, int(round(source_fps / max(sample_fps, 0.1))))
+
+    frames: list[np.ndarray] = []
+    idx = 0
+    while True:
+        ok, frame = cap.read()
+        if not ok:
+            break
+        if idx % stride == 0:
+            frames.append(frame)
+        idx += 1
+
+    cap.release()
+    if not frames:
+        raise RuntimeError(f"No frames sampled from {path}")
+    return frames, source_fps
+
+
+def _landmark_distance(a: Any, b: Any, width: int, height: int) -> float:
+    ax, ay = float(a.x) * width, float(a.y) * height
+    bx, by = float(b.x) * width, float(b.y) * height
+    return math.hypot(ax - bx, ay - by)
+
+
+def _extract_face_stats_fallback(frames: list[np.ndarray], sample_fps: float) -> FaceStats:
+    """
+    Fallback when MediaPipe face mesh API is unavailable in current build.
+    Uses image heuristics that keep extraction running instead of dropping all clips.
+    """
+    global _FACE_FALLBACK_WARNED
+    if not _FACE_FALLBACK_WARNED:
+        print("[WARN] MediaPipe Face Mesh unavailable; using heuristic face fallback metrics.")
+        _FACE_FALLBACK_WARNED = True
+
+    h, w = frames[0].shape[:2]
+    area_ratio = 0.28
+
+    gray0 = cv2.cvtColor(frames[0], cv2.COLOR_BGR2GRAY)
+    conf_proxy = 1.0 - float(np.std(gray0) / 128.0)
+    confidence = float(np.clip(conf_proxy, 0.55, 0.90))
+
+    # Head pose proxies from horizontal/vertical gradient imbalance.
+    grad_x = np.mean(np.abs(np.diff(gray0.astype(np.float32), axis=1)))
+    grad_y = np.mean(np.abs(np.diff(gray0.astype(np.float32), axis=0)))
+    yaw_deg = float(np.clip((grad_x / max(grad_y, 1e-6)) * 8.0, 0.0, 35.0))
+    pitch_deg = float(np.clip((grad_y / max(grad_x, 1e-6)) * 6.0, 0.0, 25.0))
+
+    # Mouth-open proxy from lower-center variance.
+    y0, y1 = int(0.55 * h), int(0.82 * h)
+    x0, x1 = int(0.33 * w), int(0.67 * w)
+    roi = gray0[y0:y1, x0:x1]
+    mouth_open_ratio = float(np.clip(np.std(roi) / 96.0, 0.18, 0.62))
+
+    # Blink proxy unavailable -> conservative nominal rate.
+    blink_rate_hz = 0.25
+
+    # Occlusion proxy via darkness saturation ratio in center region.
+    center = gray0[int(0.2 * h) : int(0.85 * h), int(0.2 * w) : int(0.8 * w)]
+    dark_ratio = float(np.mean(center < 18))
+    occlusion_present = dark_ratio > 0.35
+
+    return FaceStats(
+        area_ratio=area_ratio,
+        confidence=confidence,
+        yaw_deg=yaw_deg,
+        pitch_deg=pitch_deg,
+        mouth_open_ratio=mouth_open_ratio,
+        blink_rate_hz=blink_rate_hz,
+        occlusion_present=occlusion_present,
+    )
+
+
+def _extract_face_stats(frames: list[np.ndarray], sample_fps: float) -> FaceStats:
+    if mp is None or not hasattr(mp, "solutions") or not hasattr(mp.solutions, "face_mesh"):
+        return _extract_face_stats_fallback(frames, sample_fps)
+
+    mp_mesh = mp.solutions.face_mesh
+    face_mesh = mp_mesh.FaceMesh(
+        static_image_mode=True,
+        max_num_faces=1,
+        refine_landmarks=True,
+        min_detection_confidence=0.5,
+    )
+
+    area_ratios: list[float] = []
+    confidences: list[float] = []
+    yaws: list[float] = []
+    pitches: list[float] = []
+    mouth_open: list[float] = []
+    blink_closures: list[float] = []
+    missing_face = 0
+
+    # Mouth/eye landmarks from MediaPipe Face Mesh topology.
+    mouth_top, mouth_bottom = 13, 14
+    mouth_left, mouth_right = 78, 308
+    l_eye_top, l_eye_bottom = 159, 145
+    l_eye_left, l_eye_right = 33, 133
+    r_eye_top, r_eye_bottom = 386, 374
+    r_eye_left, r_eye_right = 362, 263
+    nose_tip, left_cheek, right_cheek = 1, 234, 454
+    forehead, chin = 10, 152
+
+    for frame in frames:
+        h, w = frame.shape[:2]
+        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        result = face_mesh.process(rgb)
+        if not result.multi_face_landmarks:
+            missing_face += 1
+            continue
+
+        lm = result.multi_face_landmarks[0].landmark
+
+        xs = np.array([p.x for p in lm], dtype=np.float32)
+        ys = np.array([p.y for p in lm], dtype=np.float32)
+        x0 = float(np.clip(xs.min(), 0.0, 1.0))
+        x1 = float(np.clip(xs.max(), 0.0, 1.0))
+        y0 = float(np.clip(ys.min(), 0.0, 1.0))
+        y1 = float(np.clip(ys.max(), 0.0, 1.0))
+        area_ratio = max(0.0, (x1 - x0) * (y1 - y0))
+        area_ratios.append(area_ratio)
+
+        # MediaPipe FaceMesh does not expose direct confidence per face; use coverage proxy.
+        conf = 1.0 - float(np.mean((xs <= 0.0) | (xs >= 1.0) | (ys <= 0.0) | (ys >= 1.0)))
+        confidences.append(float(np.clip(conf, 0.0, 1.0)))
+
+        nose = lm[nose_tip]
+        lch = lm[left_cheek]
+        rch = lm[right_cheek]
+        fh = lm[forehead]
+        ch = lm[chin]
+        yaw = abs((nose.x - (lch.x + rch.x) / 2.0) * 120.0)
+        pitch = abs((nose.y - (fh.y + ch.y) / 2.0) * 120.0)
+        yaws.append(float(yaw))
+        pitches.append(float(pitch))
+
+        mouth_h = _landmark_distance(lm[mouth_top], lm[mouth_bottom], w, h)
+        mouth_w = max(_landmark_distance(lm[mouth_left], lm[mouth_right], w, h), 1e-6)
+        mouth_open.append(float(np.clip(mouth_h / mouth_w, 0.0, 1.0)))
+
+        l_h = _landmark_distance(lm[l_eye_top], lm[l_eye_bottom], w, h)
+        l_w = max(_landmark_distance(lm[l_eye_left], lm[l_eye_right], w, h), 1e-6)
+        r_h = _landmark_distance(lm[r_eye_top], lm[r_eye_bottom], w, h)
+        r_w = max(_landmark_distance(lm[r_eye_left], lm[r_eye_right], w, h), 1e-6)
+        eye_open_ratio = float(np.clip(0.5 * (l_h / l_w + r_h / r_w), 0.0, 1.0))
+        blink_closures.append(eye_open_ratio)
+
+    face_mesh.close()
+
+    if not area_ratios:
+        raise RuntimeError("No detectable face landmarks in sampled frames")
+
+    # Blink estimate: count closures under threshold in sampled stream, then normalize by duration.
+    blink_events = 0
+    prev_closed = False
+    for r in blink_closures:
+        closed = r < 0.18
+        if closed and not prev_closed:
+            blink_events += 1
+        prev_closed = closed
+    sampled_seconds = len(frames) / max(sample_fps, 1e-6)
+    blink_rate_hz = blink_events / max(sampled_seconds, 1e-6)
+
+    occlusion_ratio = missing_face / len(frames)
+    occlusion_present = occlusion_ratio > 0.30
+
+    return FaceStats(
+        area_ratio=float(np.clip(np.mean(area_ratios), 0.0, 1.0)),
+        confidence=float(np.clip(np.mean(confidences), 0.0, 1.0)),
+        yaw_deg=float(np.clip(np.mean(yaws), 0.0, 180.0)),
+        pitch_deg=float(np.clip(np.mean(pitches), 0.0, 180.0)),
+        mouth_open_ratio=float(np.clip(np.mean(mouth_open), 0.0, 1.0)),
+        blink_rate_hz=max(0.0, float(blink_rate_hz)),
+        occlusion_present=occlusion_present,
+    )
+
+
+def _motion_score(frames: list[np.ndarray]) -> float:
+    if len(frames) < 2:
+        return 0.0
+    scores = []
+    prev = cv2.cvtColor(frames[0], cv2.COLOR_BGR2GRAY)
+    for frame in frames[1:]:
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        diff = cv2.absdiff(prev, gray)
+        scores.append(float(np.mean(diff) / 255.0))
+        prev = gray
+    return float(np.clip(np.mean(scores), 0.0, 1.0))
+
+
+def _bg_complexity(frames: list[np.ndarray]) -> tuple[str, float]:
+    # Hybrid complexity proxy: edge density + intensity variation + local texture energy.
+    edge_densities = []
+    texture_energies = []
+    intensity_vars = []
+    for frame in frames:
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+
+        edges = cv2.Canny(gray, 80, 160)
+        edge_density = float(np.mean(edges > 0))
+        edge_densities.append(edge_density)
+
+        lap = cv2.Laplacian(gray, cv2.CV_32F)
+        texture_energy = float(np.mean(np.abs(lap)) / 255.0)
+        texture_energies.append(texture_energy)
+
+        intensity_vars.append(float(np.std(gray.astype(np.float32)) / 128.0))
+
+    edge_score = float(np.clip(np.mean(edge_densities), 0.0, 1.0))
+    texture_score = float(np.clip(np.mean(texture_energies), 0.0, 1.0))
+    variation_score = float(np.clip(np.mean(intensity_vars), 0.0, 1.0))
+    score = float(np.clip(0.45 * edge_score + 0.30 * texture_score + 0.25 * variation_score, 0.0, 1.0))
+
+    if score < 0.07:
+        tag = "solid_dark"
+    elif score < 0.10:
+        tag = "simple_room"
+    elif score < 0.15:
+        tag = "busy_room"
+    else:
+        tag = "crowded_event"
+    return tag, score
+
+
+def _lighting_uniformity(frames: list[np.ndarray]) -> float:
+    vals = []
+    for frame in frames:
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY).astype(np.float32) / 255.0
+        std = float(np.std(gray))
+        vals.append(float(np.clip(1.0 - 2.0 * std, 0.0, 1.0)))
+    return float(np.clip(np.mean(vals), 0.0, 1.0))
+
+
+def _extract_audio_track(video_path: Path, wav_path: Path) -> None:
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-i",
+        str(video_path),
+        "-ac",
+        "1",
+        "-ar",
+        "16000",
+        str(wav_path),
+    ]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f"ffmpeg audio extraction failed for {video_path}: {result.stderr.strip()}")
+
+
+def _audio_snr_db(wav_path: Path) -> float:
+    # Approximate SNR using RMS percentile split.
+    cmd = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-f",
+        "lavfi",
+        "-i",
+        f"amovie={wav_path},astats=metadata=1:reset=1",
+        "-show_entries",
+        "frame_tags=lavfi.astats.Overall.RMS_level",
+        "-of",
+        "json",
+    ]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        return 0.0
+    payload = json.loads(result.stdout or "{}")
+    frames = payload.get("frames", [])
+    levels = []
+    for frame in frames:
+        tags = frame.get("tags", {})
+        val = tags.get("lavfi.astats.Overall.RMS_level")
+        if val is None:
+            continue
+        try:
+            levels.append(float(val))
+        except ValueError:
+            continue
+    if len(levels) < 10:
+        return 0.0
+    arr = np.array(levels, dtype=np.float32)
+    speech = np.percentile(arr, 85)
+    noise = np.percentile(arr, 20)
+    return float(np.clip(speech - noise, 0.0, 80.0))
+
+
+def _transcribe(wav_path: Path, model: Any) -> tuple[int, float]:
+    result = model.transcribe(str(wav_path), language="en", fp16=False, verbose=False)
+    text = str(result.get("text", "")).strip()
+    segments = result.get("segments", []) or []
+    # Whisper does not provide true confidence; use avg_logprob as bounded proxy.
+    confs = []
+    for seg in segments:
+        avg_logprob = seg.get("avg_logprob")
+        if avg_logprob is None:
+            continue
+        confs.append(float(1.0 / (1.0 + math.exp(-avg_logprob))))
+    confidence = float(np.mean(confs)) if confs else 0.0
+    return len(text.split()), float(np.clip(confidence, 0.0, 1.0))
+
+
+def _load_environment_map(path: str | None) -> dict[str, str]:
+    if not path:
+        return {}
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    mapping: dict[str, str] = {}
+    if isinstance(payload, dict):
+        for key, value in payload.items():
+            k = str(key).strip().lower().replace("\\", "/")
+            v = str(value).strip()
+            if not k or not v:
+                continue
+            mapping[k] = v
+        return mapping
+    if isinstance(payload, list):
+        for item in payload:
+            if not isinstance(item, dict):
+                continue
+            key = item.get("clip_id") or item.get("path") or item.get("name")
+            value = item.get("environment_tag")
+            if key is None or value is None:
+                continue
+            k = str(key).strip().lower().replace("\\", "/")
+            v = str(value).strip()
+            if not k or not v:
+                continue
+            mapping[k] = v
+        return mapping
+    raise ValueError("environment map must be JSON object or list of {clip_id/path,name,environment_tag}")
+
+
+def _environment_tag_from_features(
+    bg_complexity_score: float,
+    motion_score: float,
+    lighting_uniformity: float,
+    audio_snr_db: float,
+) -> str:
+    activity = 0.60 * bg_complexity_score + 0.30 * motion_score + 0.10 * (1.0 - lighting_uniformity)
+    if motion_score >= 0.075 or (activity >= 0.14 and bg_complexity_score >= 0.13):
+        return "street_walk"
+    if bg_complexity_score >= 0.18 or activity >= 0.18:
+        return "crowded_event"
+    if audio_snr_db < 14.5 and motion_score >= 0.035:
+        return "car_vlog"
+    if lighting_uniformity >= 0.76 and bg_complexity_score <= 0.09 and motion_score <= 0.03 and audio_snr_db >= 16.0:
+        return "podcast_studio"
+    if bg_complexity_score <= 0.12 and lighting_uniformity >= 0.67:
+        return "office"
+    if lighting_uniformity < 0.62:
+        return "webcam_room"
+    return "home_office"
+
+
+def _environment_tag(
+    path_hint: Path,
+    clip_id: str,
+    bg_complexity_score: float,
+    motion_score: float,
+    lighting_uniformity: float,
+    audio_snr_db: float,
+    environment_map: dict[str, str] | None = None,
+) -> str:
+    mapping = environment_map or {}
+    norm_path = str(path_hint).strip().lower().replace("\\", "/")
+    base_name = Path(norm_path).name
+    base_stem = Path(norm_path).stem
+    clip_key = clip_id.strip().lower()
+
+    for key in (norm_path, base_name, base_stem, clip_key):
+        if key in mapping:
+            return mapping[key]
+
+    # Use only user-intended path hint (relative clip path), not absolute system path.
+    for tag, keys in ENV_KEYWORDS.items():
+        if any(k in norm_path for k in keys):
+            return tag
+
+    return _environment_tag_from_features(
+        bg_complexity_score=bg_complexity_score,
+        motion_score=motion_score,
+        lighting_uniformity=lighting_uniformity,
+        audio_snr_db=audio_snr_db,
+    )
+
+
+def _framing_from_path_or_pose(path_hint: Path, yaw_deg: float) -> str:
+    name = str(path_hint).lower()
+    if "closeup" in name or "close_up" in name:
+        return "closeup"
+    if "offgaze" in name or "off_gaze" in name:
+        return "offgaze"
+    if "left" in name:
+        return "left"
+    if "right" in name:
+        return "right"
+    if yaw_deg <= 6.0:
+        return "front"
+    if yaw_deg <= 11.0:
+        return "offgaze"
+    return "left" if ("left" in name) else "right"
+
+
+def _extract_clip_metadata(
+    video_path: Path,
+    path_hint: Path,
+    sample_fps: float,
+    asr_model: Any,
+    rubric: RubricState,
+    with_difficulty: bool,
+    environment_tag_override: str | None = None,
+    environment_map: dict[str, str] | None = None,
+    framing_override: str | None = None,
+) -> dict[str, Any]:
+    probe = _run_ffprobe(video_path)
+    frames, _source_fps = _sample_frames(video_path, sample_fps=sample_fps)
+    face = _extract_face_stats(frames, sample_fps=sample_fps)
+    motion = _motion_score(frames)
+    bg_tag, bg_score = _bg_complexity(frames)
+    light = _lighting_uniformity(frames)
+
+    wav_path = video_path.with_suffix(".tmp16k.wav")
+    _extract_audio_track(video_path, wav_path)
+    try:
+        snr = _audio_snr_db(wav_path)
+        word_count, transcript_conf = _transcribe(wav_path, asr_model)
+    finally:
+        if wav_path.exists():
+            wav_path.unlink()
+
+    row: dict[str, Any] = {
+        "clip_id": video_path.stem,
+        "duration_s": probe["duration_s"],
+        "fps": probe["fps"],
+        "resolution": probe["resolution"],
+        "face_area_ratio": round(face.area_ratio, 4),
+        "face_confidence": round(face.confidence, 4),
+        "head_pose_yaw_deg": round(face.yaw_deg, 3),
+        "head_pose_pitch_deg": round(face.pitch_deg, 3),
+        "motion_score": round(motion, 4),
+        "bg_complexity": bg_tag,
+        "bg_complexity_score": round(bg_score, 4),
+        "mouth_open_ratio": round(face.mouth_open_ratio, 4),
+        "blink_rate_hz": round(face.blink_rate_hz, 4),
+        "audio_snr_db": round(float(snr), 3),
+        "transcript_word_count": int(word_count),
+        "transcript_confidence": round(float(transcript_conf), 4),
+        "lighting_uniformity": round(light, 4),
+        "occlusion_present": bool(face.occlusion_present),
+        "environment_tag": environment_tag_override
+        or _environment_tag(
+            path_hint=path_hint,
+            clip_id=video_path.stem,
+            bg_complexity_score=bg_score,
+            motion_score=motion,
+            lighting_uniformity=light,
+            audio_snr_db=float(snr),
+            environment_map=environment_map,
+        ),
+        "framing": framing_override or _framing_from_path_or_pose(path_hint=path_hint, yaw_deg=face.yaw_deg),
+    }
+
+    clip = ClipMetadata(**row)
+    payload = clip.model_dump()
+    if with_difficulty:
+        payload["difficulty"] = derive_clip_difficulty(payload, rubric)
+    return payload
+
+
+def _iter_videos(input_dir: Path) -> list[Path]:
+    files = [p for p in input_dir.rglob("*") if p.is_file() and p.suffix.lower() in SUPPORTED_SUFFIXES]
+    files.sort()
+    return files
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Extract ClipMetadata manifest from MP4 files.")
+    parser.add_argument("--input-dir", type=str, required=True, help="Directory containing video clips.")
+    parser.add_argument("--output", type=str, required=True, help="Output manifest path (.jsonl).")
+    parser.add_argument("--sample-fps", type=float, default=2.0, help="Frame sampling FPS for visual features.")
+    parser.add_argument("--max-clips", type=int, default=0, help="Optional cap; 0 means all clips.")
+    parser.add_argument(
+        "--difficulty-mode",
+        type=str,
+        choices=["none", "derive"],
+        default="derive",
+        help="Attach derived difficulty labels or not.",
+    )
+    parser.add_argument(
+        "--whisper-model",
+        type=str,
+        default="small",
+        help="Whisper model size (tiny, base, small, medium, large).",
+    )
+    parser.add_argument(
+        "--environment-tag",
+        type=str,
+        default=None,
+        help="Optional fixed environment_tag override for all clips in this run.",
+    )
+    parser.add_argument(
+        "--framing",
+        type=str,
+        default=None,
+        help="Optional fixed framing override for all clips in this run.",
+    )
+    parser.add_argument(
+        "--environment-map",
+        type=str,
+        default=None,
+        help="Optional JSON map/list to assign environment_tag by clip_id/path/name.",
+    )
+    args = parser.parse_args()
+
+    dep_errors = [err for err in (_CV2_IMPORT_ERROR, _NUMPY_IMPORT_ERROR, _WHISPER_IMPORT_ERROR) if err is not None]
+    if dep_errors:
+        raise RuntimeError(
+            "Missing extractor dependencies. Install: "
+            "opencv-python-headless mediapipe openai-whisper numpy"
+        ) from dep_errors[0]
+
+    input_dir = Path(args.input_dir).resolve()
+    output_path = Path(args.output).resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if not input_dir.exists():
+        raise FileNotFoundError(f"Input dir not found: {input_dir}")
+
+    rubric = RubricState()
+    asr_model = whisper.load_model(args.whisper_model)
+    environment_map = _load_environment_map(args.environment_map)
+
+    videos = _iter_videos(input_dir)
+    if args.max_clips > 0:
+        videos = videos[: args.max_clips]
+    if not videos:
+        raise RuntimeError(f"No supported video files found in {input_dir}")
+
+    failures: list[tuple[str, str]] = []
+    with open(output_path, "w", encoding="utf-8") as out:
+        for video_path in videos:
+            try:
+                row = _extract_clip_metadata(
+                    video_path=video_path,
+                    path_hint=video_path.relative_to(input_dir),
+                    sample_fps=args.sample_fps,
+                    asr_model=asr_model,
+                    rubric=rubric,
+                    with_difficulty=(args.difficulty_mode == "derive"),
+                    environment_tag_override=args.environment_tag,
+                    environment_map=environment_map,
+                    framing_override=args.framing,
+                )
+            except Exception as exc:
+                failures.append((str(video_path), str(exc)))
+                continue
+            out.write(json.dumps(row, ensure_ascii=True) + "\n")
+
+    with open(output_path, "r", encoding="utf-8") as f:
+        written = sum(1 for _ in f)
+    print(json.dumps({"output": str(output_path), "written_rows": written, "failed_clips": len(failures)}, indent=2))
+    if failures:
+        for clip, err in failures:
+            print(f"[FAIL] {clip}: {err}")
+
+
+if __name__ == "__main__":
+    main()

server/__init__.py

@@ -0,0 +1 @@
+"""Clip-quality server package."""

server/app.py

@@ -0,0 +1,963 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import threading
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+import uvicorn
+from fastapi import BackgroundTasks, HTTPException
+from openenv.core.env_server import create_fastapi_app
+
+import inference
+from models import Action, Observation, TaskInfo
+from server.baseline_runs import baseline_run_tracker
+from server.environment import ClipQualityEnvironment
+from server.grader import grade
+from server.tasks import TASK_REGISTRY
+
+PRODUCT_NAME = "CLIP Quality Analyzer"
+ENVIRONMENT_ID = "clip_quality_env"
+OVERRIDDEN_ROUTES = {"/health", "/state", "/tasks", "/grader", "/baseline"}
+TASK_SURFACE_DESCRIPTIONS = {
+    "task_easy": "Classify clips with clear quality signals and concise metadata-based reasoning.",
+    "task_medium": "Classify borderline clips by balancing mixed quality indicators.",
+    "task_hard": "Classify difficult clips with conflicting signals and explicit trade-off reasoning.",
+}
+TASK_INPUT_TAB_MAP = {
+    "task_easy": "easy",
+    "task_medium": "medium",
+    "task_hard": "hard",
+}
+CLASS_LABEL_CHOICES = [
+    ("KEEP", "KEEP"),
+    ("BORDERLINE", "BORDERLINE"),
+    ("REJECT", "REJECT"),
+]
+BASELINE_RUN_BUTTON_LABEL = "Run LLM Baseline Agent"
+BASELINE_RUN_BUTTON_RUNNING_LABEL = "Running LLM Baseline Agent..."
+QUALITY_HINT_BUTTON_LABEL = "Load Quality Hint"
+BASELINE_RESULT_PLACEHOLDER = (
+    "Run the baseline agent to view model, mode, reward achieved per step, and success status."
+)
+HF_TOKEN_MISSING_WARNING = "LLM unavailable — no HF_TOKEN configured. Showing deterministic fallback result."
+
+
+def _normalized_path(path: str) -> str:
+    return path.rstrip("/") or "/"
+
+
+def _surface_task_description(task_id: str, fallback: str) -> str:
+    return TASK_SURFACE_DESCRIPTIONS.get(task_id, fallback)
+
+
+def _input_tab_for_task(task_id: str) -> str:
+    return TASK_INPUT_TAB_MAP.get(task_id, "easy")
+
+
+def _clean_reasoning_text(value: str | None) -> str:
+    return str(value or "").strip()
+
+
+def _build_reasoning_from_sections(sections: list[tuple[str, str]]) -> str:
+    lines: list[str] = []
+    for title, value in sections:
+        cleaned = _clean_reasoning_text(value)
+        if cleaned:
+            lines.append(f"{title}: {cleaned}")
+    return "\n".join(lines) if lines else "No reasoning provided."
+
+
+def _resolve_tiered_submission(
+    task_id: str,
+    selected_input_tab: str | None,
+    easy_label: str,
+    easy_observation: str,
+    medium_label: str,
+    medium_primary_signal: str,
+    medium_conflicting_signal: str,
+    medium_reasoning: str,
+    hard_label: str,
+    hard_tradeoff_summary: str,
+    hard_confidence_justification: str,
+    hard_confidence: float,
+) -> tuple[str, str, float]:
+    active_tab = selected_input_tab if selected_input_tab in {"easy", "medium", "hard"} else _input_tab_for_task(task_id)
+
+    if active_tab == "easy":
+        label = str(easy_label or "BORDERLINE").strip().upper()
+        reasoning = _build_reasoning_from_sections(
+            [
+                ("Tier", "Easy"),
+                ("Predicted Label", label),
+                ("Key Observation", easy_observation),
+            ]
+        )
+        confidence = 0.5
+    elif active_tab == "medium":
+        label = str(medium_label or "BORDERLINE").strip().upper()
+        reasoning = _build_reasoning_from_sections(
+            [
+                ("Tier", "Medium"),
+                ("Predicted Label", label),
+                ("Primary Signal", medium_primary_signal),
+                ("Conflicting Signal", medium_conflicting_signal),
+                ("Reasoning", medium_reasoning),
+            ]
+        )
+        confidence = 0.5
+    else:
+        label = str(hard_label or "BORDERLINE").strip().upper()
+        confidence = float(hard_confidence)
+        reasoning = _build_reasoning_from_sections(
+            [
+                ("Tier", "Hard"),
+                ("Predicted Label", label),
+                ("Trade-off Summary", hard_tradeoff_summary),
+                ("Confidence Justification", hard_confidence_justification),
+                ("Confidence", f"{confidence:.2f}"),
+            ]
+        )
+
+    if label not in {"KEEP", "BORDERLINE", "REJECT"}:
+        label = "BORDERLINE"
+    return label, reasoning, confidence
+
+
+def _input_tab_update_for_task(task_id: str) -> dict[str, Any]:
+    return gr.update(selected=_input_tab_for_task(task_id))
+
+
+app = create_fastapi_app(
+    env=ClipQualityEnvironment,
+    action_cls=Action,
+    observation_cls=Observation,
+)
+# Replace selected OpenEnv defaults with reference-style handlers.
+overridden_paths = {_normalized_path(path) for path in OVERRIDDEN_ROUTES}
+app.router.routes = [
+    route
+    for route in app.router.routes
+    if _normalized_path(getattr(route, "path", "")) not in overridden_paths
+]
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {
+        "status": "ok",
+        "environment": ENVIRONMENT_ID,
+        "product": PRODUCT_NAME,
+        "health": "/health",
+        "metadata": "/metadata",
+        "dashboard": "/dashboard/",
+    }
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok", "service": "clip-quality-api"}
+
+
+@app.get("/state")
+def get_state() -> dict[str, Any]:
+    env = ClipQualityEnvironment()
+    state = env.state.model_dump()
+    state["product"] = PRODUCT_NAME
+    state["workflow"] = "clip_quality_analysis"
+    return state
+
+
+@app.get("/tasks")
+def list_tasks() -> list[TaskInfo]:
+    return [
+        TaskInfo(
+            task_id=task_id,
+            difficulty=task["difficulty"],
+            description=_surface_task_description(task_id, task["description"]),
+            action_schema=Action.model_json_schema(),
+        )
+        for task_id, task in TASK_REGISTRY.items()
+    ]
+
+
+@app.post("/grader")
+def get_grader_score(task_id: str, action: Action) -> dict[str, Any]:
+    if task_id not in TASK_REGISTRY:
+        raise HTTPException(status_code=404, detail=f"Unknown task_id: {task_id}")
+    score = grade(action.model_dump(), task_id)
+    return {
+        "task_id": task_id,
+        "score": score,
+        "passed": 1 if score > 0.5 else 0,
+        "total": 1,
+        "metric": "clip_quality_alignment",
+    }
+
+
+def _baseline_payload_from_raw(raw: dict[str, Any]) -> dict[str, Any]:
+    scores = raw.get("baseline_scores")
+    score_payload = scores if isinstance(scores, dict) else {}
+    payload: dict[str, Any] = {
+        "baseline_results": raw.get("detail", []),
+        "average_score": score_payload.get("overall_avg", 0.0),
+        "model": raw.get("model", os.environ.get("MODEL_NAME", inference.DEFAULT_MODEL_NAME)),
+        "metric": "clip_quality_alignment",
+    }
+    if "runtime_seconds" in raw:
+        payload["runtime_seconds"] = raw.get("runtime_seconds")
+    if "warning" in raw:
+        payload["warning"] = raw.get("warning")
+    return payload
+
+
+def _initial_baseline_payload(task: str | None = None) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "baseline_results": [],
+        "average_score": None,
+        "model": os.environ.get("MODEL_NAME", inference.DEFAULT_MODEL_NAME),
+        "metric": "clip_quality_alignment",
+    }
+    if task:
+        payload["task"] = task
+    return payload
+
+
+def _public_baseline_status(status: str) -> str:
+    return "complete" if status == "completed" else status
+
+
+def _mode_label(mode: str) -> str:
+    return "LLM" if str(mode).strip().lower() == "llm" else "Deterministic fallback"
+
+
+def _summarize_mode(results: list[dict[str, Any]]) -> str:
+    modes = {str(item.get("mode", "fallback")).strip().lower() for item in results}
+    if not modes or modes == {"fallback"}:
+        return "Deterministic fallback"
+    if modes == {"llm"}:
+        return "LLM"
+    return "Mixed (LLM + deterministic fallback)"
+
+
+def _baseline_warning_messages(payload: dict[str, Any]) -> list[str]:
+    warnings: list[str] = []
+    if not os.environ.get("HF_TOKEN"):
+        warnings.append(HF_TOKEN_MISSING_WARNING)
+    payload_warning = str(payload.get("warning") or "").strip()
+    if payload_warning and payload_warning not in warnings:
+        warnings.append(payload_warning)
+    return warnings
+
+
+def _format_score_color(value: float) -> str:
+    return "#16a34a" if value >= 0.10 else "#dc2626"
+
+
+def _label_score_color(value: float) -> str:
+    if value >= 0.60:
+        return "#16a34a"
+    if value >= 0.25:
+        return "#f59e0b"
+    return "#dc2626"
+
+
+def _reasoning_score_color(value: float) -> str:
+    if value >= 0.30:
+        return "#16a34a"
+    if value > 0.0:
+        return "#f59e0b"
+    return "#dc2626"
+
+
+def _total_reward_color(value: float, has_submission: bool) -> str:
+    if not has_submission:
+        return "#6b7280"
+    if value >= 0.70:
+        return "#16a34a"
+    if value > 0.0:
+        return "#f59e0b"
+    return "#dc2626"
+
+
+def _reward_card_html(title: str, value: float, color: str, max_value: float) -> str:
+    return (
+        "<div style='background:#ffffff;border:1px solid #fde7cc;border-left:4px solid {color};"
+        "border-radius:8px;padding:8px 10px;'>"
+        "<div style='font-size:0.8rem;font-weight:600;color:#4b5563;'>{title}</div>"
+        "<div style='font-size:1.1rem;font-weight:700;color:{color};'>{value:.2f}</div>"
+        "<div style='font-size:0.72rem;color:#6b7280;'>max {max_value:.2f}</div>"
+        "</div>"
+    ).format(title=title, value=float(value), color=color, max_value=float(max_value))
+
+
+def _reward_breakdown_markdown(obs: dict[str, Any], initialized: bool = False) -> str:
+    info = obs.get("info", {}) if isinstance(obs, dict) else {}
+    format_score = float(info.get("format_score", 0.0))
+    label_score = float(info.get("label_score", 0.0))
+    reasoning_score = float(info.get("reasoning_score", 0.0))
+    total_reward = float(obs.get("reward", info.get("reward_total", 0.0)))
+    running_total = float(info.get("total_reward", 0.0))
+    best_score = float(info.get("best_score", 0.0))
+    has_submission = bool(obs.get("history"))
+
+    title = "### Session Initialized" if initialized else "### Latest Submission Reward Breakdown"
+    subtitle = (
+        "_Submit a classification action to populate live reward decomposition._"
+        if initialized
+        else "_Live reward decomposition for this submission._"
+    )
+    cards = (
+        "<div style='display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:8px;'>"
+        f"{_reward_card_html('Format Score', format_score, _format_score_color(format_score), 0.10)}"
+        f"{_reward_card_html('Label Score', label_score, _label_score_color(label_score), 0.60)}"
+        f"{_reward_card_html('Reasoning Score', reasoning_score, _reasoning_score_color(reasoning_score), 0.30)}"
+        "</div>"
+    )
+    total_color = _total_reward_color(total_reward, has_submission=has_submission)
+    return "\n".join(
+        [
+            title,
+            subtitle,
+            cards,
+            f"### **Total Reward:** <span style='color:{total_color};font-size:1.35rem;'>{total_reward:.3f}</span>",
+            f"Current Best Score: **{best_score:.3f}**",
+        ]
+    )
+
+
+def _format_baseline_result_markdown(
+    payload: dict[str, Any] | None,
+    status: str,
+    error: dict[str, Any] | None = None,
+) -> str:
+    payload = payload if isinstance(payload, dict) else {}
+    model_name = str(payload.get("model") or os.environ.get("MODEL_NAME", inference.DEFAULT_MODEL_NAME))
+    raw_results = payload.get("baseline_results")
+    results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+    lines = [f"- **Model:** `{model_name}`"]
+
+    if status == "running":
+        lines.append("- **Mode Used:** _Pending..._")
+        lines.append("- **Reward achieved per step:** _Pending..._")
+        lines.append("- **Success:** _Pending..._")
+    else:
+        lines.append(f"- **Mode Used:** {_summarize_mode(results)}")
+        if "average_score" in payload and payload.get("average_score") is not None:
+            lines.append(f"- **Average Score:** `{float(payload.get('average_score', 0.0)):.3f}`")
+        if results:
+            for result in results:
+                task_name = str(result.get("task_id", "unknown_task"))
+                steps = int(result.get("steps", 0) or 0)
+                average_reward = float(result.get("reward", 0.0))
+                total_reward = float(
+                    result.get("total_reward", average_reward * steps if steps > 0 else average_reward)
+                )
+                final_reward = float(result.get("final_reward", average_reward))
+                reward_per_step = total_reward / steps if steps > 0 else average_reward
+                success = "Yes" if bool(result.get("success")) else "No"
+                lines.append(
+                    f"- **{task_name}**: {_mode_label(str(result.get('mode', 'fallback')))}; "
+                    f"score `{reward_per_step:.3f}`; success **{success}**"
+                )
+        else:
+            lines.append("- **Reward achieved per step:** _No baseline detail available._")
+            lines.append("- **Success:** _No baseline detail available._")
+
+    if status == "complete":
+        lines.append("- **Status:** Complete")
+    elif status == "failed":
+        message = str((error or {}).get("message") or "").strip()
+        suffix = f" — {message}" if message else ""
+        lines.append(f"- **Status:** Failed{suffix}")
+    elif status == "running":
+        lines.append("- **Status:** Running")
+
+    for warning in _baseline_warning_messages(payload):
+        lines.append(f"> {warning}")
+    return "\n".join(lines)
+
+
+def _run_baseline_background(run_id: str, task: str | None = None) -> None:
+    try:
+        raw = inference.run_baseline(task=task)
+        baseline_run_tracker.mark_complete(run_id, _baseline_payload_from_raw(raw))
+    except Exception as exc:
+        baseline_run_tracker.mark_failed(run_id, {"message": str(exc)})
+
+
+def _start_baseline_ui_run(task: str | None = None) -> tuple[str, str, str, dict[str, Any], dict[str, Any]]:
+    baseline_run_tracker.cleanup_expired()
+    run_id = baseline_run_tracker.create_run()
+    initial_payload = _initial_baseline_payload(task=task)
+    baseline_run_tracker.update_partial(run_id, initial_payload)
+    worker = threading.Thread(
+        target=_run_baseline_background,
+        args=(run_id, task),
+        daemon=True,
+        name=f"baseline-ui-{run_id[:8]}",
+    )
+    worker.start()
+    return (
+        run_id,
+        "### LLM baseline run in progress...",
+        _format_baseline_result_markdown(initial_payload, status="running"),
+        gr.update(value=BASELINE_RUN_BUTTON_RUNNING_LABEL, interactive=False),
+        gr.update(active=True),
+    )
+
+
+def _poll_baseline_ui_run(
+    run_id: str | None,
+) -> tuple[str | None, str, str, dict[str, Any], dict[str, Any]]:
+    if not run_id:
+        return (
+            None,
+            "### Baseline agent idle.",
+            BASELINE_RESULT_PLACEHOLDER,
+            gr.update(value=BASELINE_RUN_BUTTON_LABEL, interactive=True),
+            gr.update(active=False),
+        )
+
+    baseline_run_tracker.cleanup_expired()
+    run = baseline_run_tracker.get_run(run_id)
+    if run is None:
+        return (
+            None,
+            "### Baseline run unavailable (expired or unknown).",
+            _format_baseline_result_markdown(_initial_baseline_payload(), status="failed"),
+            gr.update(value=BASELINE_RUN_BUTTON_LABEL, interactive=True),
+            gr.update(active=False),
+        )
+
+    status = _public_baseline_status(run["status"])
+    payload = run["result"] if run["status"] == "completed" else run.get("partial")
+    payload = payload if isinstance(payload, dict) else {}
+
+    if status == "running":
+        return (
+            run_id,
+            "### LLM baseline run in progress...",
+            _format_baseline_result_markdown(payload, status="running"),
+            gr.update(value=BASELINE_RUN_BUTTON_RUNNING_LABEL, interactive=False),
+            gr.update(active=True),
+        )
+
+    if status == "failed":
+        return (
+            None,
+            "### Baseline run failed.",
+            _format_baseline_result_markdown(payload, status="failed", error=run.get("error")),
+            gr.update(value=BASELINE_RUN_BUTTON_LABEL, interactive=True),
+            gr.update(active=False),
+        )
+
+    average_score = payload.get("average_score")
+    score_display = f"{float(average_score):.3f}" if average_score is not None else "N/A"
+    return (
+        None,
+        f"### Baseline run complete (avg score: {score_display})",
+        _format_baseline_result_markdown(payload, status="complete"),
+        gr.update(value=BASELINE_RUN_BUTTON_LABEL, interactive=True),
+        gr.update(active=False),
+    )
+
+
+def _enqueue_baseline_run(background_tasks: BackgroundTasks, task: str | None = None) -> dict[str, Any]:
+    baseline_run_tracker.cleanup_expired()
+    run_id = baseline_run_tracker.create_run()
+    initial_payload = _initial_baseline_payload(task=task)
+    baseline_run_tracker.update_partial(run_id, initial_payload)
+    background_tasks.add_task(_run_baseline_background, run_id, task)
+    return {
+        "run_id": run_id,
+        "status": "running",
+        "payload": initial_payload,
+        "status_url": f"/baseline/status/{run_id}",
+    }
+
+
+@app.post("/baseline/start")
+def start_baseline_route(background_tasks: BackgroundTasks, task: str | None = None) -> dict[str, Any]:
+    return _enqueue_baseline_run(background_tasks=background_tasks, task=task)
+
+
+@app.get("/baseline/status/{run_id}")
+def baseline_status_route(run_id: str) -> dict[str, Any]:
+    baseline_run_tracker.cleanup_expired()
+    run = baseline_run_tracker.get_run(run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail=f"Unknown run_id: {run_id}")
+
+    status = _public_baseline_status(run["status"])
+    payload = run["result"] if run["status"] == "completed" else run["partial"]
+    response: dict[str, Any] = {
+        "run_id": run_id,
+        "status": status,
+        "payload": payload if payload is not None else {},
+    }
+    if status == "failed":
+        response["error"] = run["error"]
+    return response
+
+
+@app.get("/baseline")
+def run_baseline_route(background_tasks: BackgroundTasks, task: str | None = None) -> dict[str, Any]:
+    return _enqueue_baseline_run(background_tasks=background_tasks, task=task)
+
+
+def build_custom_ui() -> gr.Blocks:
+    dominant_feature_columns = [
+        "Feature Name",
+        "Current Value",
+        "Rubric Status",
+        "Threshold Range",
+    ]
+    session_history_columns = ["Step", "Clip ID", "Submitted Label", "Expected Label", "Reward"]
+
+    def format_dominant_features(rows: list[dict[str, Any]]) -> pd.DataFrame:
+        if not rows:
+            return pd.DataFrame(columns=dominant_feature_columns)
+        return pd.DataFrame(rows, columns=dominant_feature_columns)
+
+    def format_obs(obs: dict[str, Any]) -> tuple[pd.DataFrame, str, float, int, str, str]:
+        if not obs:
+            return (
+                pd.DataFrame(columns=["Clip ID", "Expected Label", "Current Review Status", "Face Confidence", "Motion Score", "Audio SNR (dB)"]),
+                "### No Clip-Quality Rubric Available",
+                0.0,
+                5,
+                "N/A",
+                "### Clip Queue: **0** of **0** items displayed",
+            )
+
+        corpus_items = list(obs.get("data_corpus", []))
+        corpus_items.sort(key=lambda item: str(item.get("clip_id", item.get("id", ""))))
+
+        corpus_data = []
+        for item in corpus_items:
+            corpus_data.append(
+                {
+                    "Clip ID": item.get("clip_id", item.get("id", "N/A")),
+                    "Expected Label": item.get("expected_label", "N/A"),
+                    "Current Review Status": item.get("review_status", "pending"),
+                    "Face Confidence": item.get("face_confidence", "N/A"),
+                    "Motion Score": item.get("motion_score", "N/A"),
+                    "Audio SNR (dB)": item.get("audio_snr_db", "N/A"),
+                }
+            )
+        df_corpus = pd.DataFrame(corpus_data) if corpus_data else pd.DataFrame(
+            columns=["Clip ID", "Expected Label", "Current Review Status", "Face Confidence", "Motion Score", "Audio SNR (dB)"]
+        )
+
+        rubric_summary = obs.get("rubric_summary", "").strip()
+        rule_md = "### Active Clip-Quality Rubric\n"
+        if rubric_summary:
+            rule_md += f"```\n{rubric_summary}\n```"
+        else:
+            rule_md += "_No rubric summary available._"
+
+        best_score = float(obs.get("info", {}).get("best_score", 0.0))
+        steps_left = int(obs.get("info", {}).get("steps_remaining", 0))
+        episode_id = str(obs.get("episode_id", "N/A"))[:8]
+        shown = int(obs.get("corpus_shown", len(corpus_data)))
+        total = int(obs.get("corpus_size", len(corpus_data)))
+        corpus_stat = f"### Clip Queue: **{shown}** of **{total}** items displayed"
+        return df_corpus, rule_md, best_score, steps_left, episode_id, corpus_stat
+
+    def format_session_history(obs: dict[str, Any]) -> tuple[pd.DataFrame, str, str]:
+        info = obs.get("info", {}) if isinstance(obs, dict) else {}
+        raw_history = info.get("session_history", [])
+        session_history = raw_history if isinstance(raw_history, list) else []
+        rows: list[dict[str, Any]] = []
+        cue_lines: list[str] = []
+
+        for idx, item in enumerate(session_history, start=1):
+            step = int(item.get("step", idx))
+            clip_id = str(item.get("clip_id", "N/A"))
+            submitted_raw = item.get("label", "")
+            submitted = str(submitted_raw).upper() if submitted_raw is not None else ""
+            expected_raw = item.get("expected_label")
+            expected_normalized = str(expected_raw).strip().lower() if expected_raw is not None else ""
+            expected = "N/A" if expected_normalized in {"", "none", "null", "n/a"} else str(expected_raw).upper()
+            reward = float(item.get("reward", 0.0))
+            rows.append(
+                {
+                    "Step": step,
+                    "Clip ID": clip_id,
+                    "Submitted Label": submitted,
+                    "Expected Label": expected,
+                    "Reward": reward,
+                }
+            )
+            is_match = bool(submitted) and expected not in {"", "N/A"} and submitted == expected
+            badge = (
+                "<span style='color:#2e7d32; font-weight:700;'>Match</span>"
+                if is_match
+                else "<span style='color:#c62828; font-weight:700;'>Mismatch</span>"
+            )
+            cue_lines.append(f"- Step {step} (`{clip_id}`): {badge}")
+
+        history_df = (
+            pd.DataFrame(rows, columns=session_history_columns)
+            if rows
+            else pd.DataFrame(columns=session_history_columns)
+        )
+        history_cues_md = "### Match Results\n" + "\n".join(cue_lines) if cue_lines else "### Match Results\n_No actions yet._"
+
+        total_reward = float(info.get("total_reward", 0.0))
+        total_color = "#2e7d32" if total_reward > 0 else "#c62828" if total_reward < 0 else "#374151"
+        history_total_md = ""
+        return history_df, history_cues_md, history_total_md
+
+    def _resolve_env(env_state: ClipQualityEnvironment | None) -> ClipQualityEnvironment:
+        return env_state if isinstance(env_state, ClipQualityEnvironment) else ClipQualityEnvironment()
+
+    def handle_reset(env_state: ClipQualityEnvironment | None, task_id: str):
+        env = _resolve_env(env_state)
+        obs = env.reset(task_id=task_id).model_dump()
+        df, pol, score, steps, ep, stat = format_obs(obs)
+        dominant_df = format_dominant_features(env.dominant_feature_rows())
+        history_df, history_cues, history_total = format_session_history(obs)
+        reward_msg = _reward_breakdown_markdown(obs, initialized=True)
+        return (
+            env,
+            df,
+            pol,
+            dominant_df,
+            score,
+            steps,
+            ep,
+            stat,
+            reward_msg,
+            history_df,
+            history_cues,
+            history_total,
+            json.dumps(obs, indent=2),
+        )
+
+    def handle_step(
+        env_state: ClipQualityEnvironment | None,
+        task_id: str,
+        selected_input_tab: str,
+        easy_label: str,
+        easy_observation: str,
+        medium_label: str,
+        medium_primary_signal: str,
+        medium_conflicting_signal: str,
+        medium_reasoning: str,
+        hard_label: str,
+        hard_tradeoff_summary: str,
+        hard_confidence_justification: str,
+        hard_confidence: float,
+        clip_id: str,
+    ):
+        env = _resolve_env(env_state)
+        if env.state.task_id != task_id:
+            env.reset(task_id=task_id)
+        label, reasoning, confidence = _resolve_tiered_submission(
+            task_id=task_id,
+            selected_input_tab=selected_input_tab,
+            easy_label=easy_label,
+            easy_observation=easy_observation,
+            medium_label=medium_label,
+            medium_primary_signal=medium_primary_signal,
+            medium_conflicting_signal=medium_conflicting_signal,
+            medium_reasoning=medium_reasoning,
+            hard_label=hard_label,
+            hard_tradeoff_summary=hard_tradeoff_summary,
+            hard_confidence_justification=hard_confidence_justification,
+            hard_confidence=hard_confidence,
+        )
+        payload: dict[str, Any] = {
+            "label": label,
+            "reasoning": reasoning,
+            "confidence": confidence,
+        }
+        if clip_id and clip_id.strip():
+            payload["clip_id"] = clip_id.strip()
+
+        obs_obj = env.step(Action.model_validate(payload))
+        obs = obs_obj.model_dump()
+        df, pol, score, steps, ep, stat = format_obs(obs)
+        dominant_df = format_dominant_features(env.dominant_feature_rows())
+        history_df, history_cues, history_total = format_session_history(obs)
+        reward_msg = _reward_breakdown_markdown(obs, initialized=False)
+        return (
+            env,
+            df,
+            pol,
+            dominant_df,
+            score,
+            steps,
+            ep,
+            stat,
+            reward_msg,
+            history_df,
+            history_cues,
+            history_total,
+            json.dumps(obs, indent=2),
+        )
+
+    def handle_quality_hint(
+        env_state: ClipQualityEnvironment | None,
+        task_id: str,
+        selected_input_tab: str,
+        easy_observation: str,
+        medium_reasoning: str,
+        hard_tradeoff_summary: str,
+    ) -> tuple[ClipQualityEnvironment, str, str, str]:
+        env = _resolve_env(env_state)
+        if env.state.task_id != task_id or not env.state.current_clip_id:
+            env.reset(task_id=task_id)
+
+        hint_text = env.build_quality_hint()
+        active_tab = selected_input_tab if selected_input_tab in {"easy", "medium", "hard"} else _input_tab_for_task(task_id)
+
+        next_easy_observation = easy_observation
+        next_medium_reasoning = medium_reasoning
+        next_hard_tradeoff_summary = hard_tradeoff_summary
+        if active_tab == "easy":
+            next_easy_observation = hint_text
+        elif active_tab == "medium":
+            next_medium_reasoning = hint_text
+        else:
+            next_hard_tradeoff_summary = hint_text
+
+        return env, next_easy_observation, next_medium_reasoning, next_hard_tradeoff_summary
+
+    with gr.Blocks(
+        title="CLIP Quality Analyzer: Judge's Console",
+        theme=gr.themes.Soft(
+            primary_hue="orange",
+            font=gr.themes.GoogleFont("Source Sans Pro"),
+        ),
+        css="""
+        .dark {
+            background-color: #0c0c0c !important;
+        }
+        """,
+    ) as demo:
+        env_state = gr.State(value=None)
+        baseline_run_id_state = gr.State(value=None)
+        selected_tab_state = gr.State(value="easy")
+        baseline_poll_timer = gr.Timer(value=1.0, active=False)
+
+        gr.HTML("<h2 style='text-align: center; color: #10b981;'>CLIP Quality Analyzer: Judge's Strategic Console</h2>")
+        gr.Markdown(
+            "Welcome, Judge Agent. Use this console to identify data to policy gaps and propose measurable governance refinements."
+        )
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("### Scenario Metrics")
+                with gr.Group():
+                    best_score_disp = gr.Number(label="Environment Best Score", value=0.0, interactive=False)
+                    steps_left_disp = gr.Number(label="Remaining Execution Steps", value=5, interactive=False)
+                    episode_disp = gr.Textbox(label="Active Episode ID", value="N/A", interactive=False)
+                
+                reward_outcome_disp = gr.Markdown("### Awaiting Scenario...")
+                
+                with gr.Group():
+                    task_id = gr.Dropdown(choices=list(TASK_REGISTRY.keys()), value="task_easy", label="Deployment Scenario")
+                    reset_btn = gr.Button("Initialize Scenario", variant="secondary")
+
+            with gr.Column(scale=3):
+                corpus_count_disp = gr.Markdown("### Corpus: 0 of 0 incidents displayed")
+                with gr.Tabs():
+                    with gr.Tab("Data Corpus (Tabular View)"):
+                        corpus_table = gr.DataFrame(
+                            label="Sampled Posts and System Actions",
+                            interactive=False,
+                        )
+                    with gr.Tab("Active Framework"):
+                        policy_display = gr.Markdown("Initialize to view the current rubric summary.")
+                    with gr.Tab("Diagnostic JSON"):
+                        raw_json_box = gr.Code(label="Environment Raw Response", language="json", interactive=False)
+                    with gr.Tab("Session History"):
+                        session_history_table = gr.DataFrame(
+                            value=pd.DataFrame(columns=session_history_columns),
+                            label="Per-step Classification History",
+                            headers=session_history_columns,
+                            col_count=(len(session_history_columns), "fixed"),
+                            datatype=["number", "str", "str", "str", "number"],
+                            interactive=False,
+                        )
+                        session_history_cues = gr.Markdown("### Match Results\n_No actions yet._")
+                        session_total_reward = gr.Markdown("", visible=False)
+                    with gr.Accordion("LLM Baseline Result", open=False):
+                        baseline_run_btn = gr.Button(BASELINE_RUN_BUTTON_LABEL, variant="secondary")
+                        baseline_status_disp = gr.Markdown("### Baseline agent idle.")
+                        baseline_result_md = gr.Markdown(BASELINE_RESULT_PLACEHOLDER)
+                
+                dominant_features_table = gr.DataFrame(
+                    value=pd.DataFrame(columns=dominant_feature_columns),
+                    label="Feature Focus Table",
+                    headers=dominant_feature_columns,
+                    column_count=len(dominant_feature_columns),
+                    column_limits=(len(dominant_feature_columns), len(dominant_feature_columns)),
+                    datatype=["str", "number", "str", "str"],
+                    interactive=False,
+                    visible=False
+                )
+
+        gr.Markdown("---")
+        gr.Markdown("### Propose Strategic Refinement")
+        with gr.Group():
+            with gr.Tabs(selected="easy") as tiered_input_tabs:
+                with gr.Tab("Easy: Definition Refining", id="easy"):
+                    easy_label_input = gr.Radio(
+                        choices=CLASS_LABEL_CHOICES,
+                        value="BORDERLINE",
+                        label="Predicted Label",
+                    )
+                    easy_observation_input = gr.Textbox(label="Key Observation", lines=2)
+                with gr.Tab("Medium: Gap Detection", id="medium"):
+                    medium_label_input = gr.Radio(
+                        choices=CLASS_LABEL_CHOICES,
+                        value="BORDERLINE",
+                        label="Predicted Label",
+                    )
+                    medium_primary_signal_input = gr.Textbox(label="Primary Signal", lines=1)
+                    medium_conflicting_signal_input = gr.Textbox(label="Conflicting Signal", lines=1)
+                    medium_reasoning_input = gr.TextArea(label="Reasoning", lines=4)
+                with gr.Tab("Hard: Full System Evolution", id="hard"):
+                    hard_label_input = gr.Radio(
+                        choices=CLASS_LABEL_CHOICES,
+                        value="BORDERLINE",
+                        label="Predicted Label",
+                    )
+                    hard_tradeoff_summary_input = gr.TextArea(label="Trade-off Summary", lines=4)
+                    with gr.Row():
+                        hard_confidence_justification_input = gr.TextArea(label="Confidence Justification", lines=3, scale=2)
+                        hard_confidence_input = gr.Slider(
+                            minimum=0.0,
+                            maximum=1.0,
+                            value=0.5,
+                            step=0.01,
+                            label="Confidence",
+                            scale=1
+                        )
+            
+            clip_id_input = gr.Textbox(label="Clip ID override (optional)")
+            hint_btn = gr.Button(QUALITY_HINT_BUTTON_LABEL, variant="secondary")
+
+        step_btn = gr.Button("Execute Strategic Step", variant="primary")
+
+        def sync_input_tab_for_task(selected_task_id: str):
+            tab_id = _input_tab_for_task(selected_task_id)
+            return _input_tab_update_for_task(selected_task_id), tab_id
+
+        def on_tab_select(evt: gr.SelectData) -> str:
+            return str(evt.value) if evt and evt.value else "easy"
+
+        task_id.change(
+            sync_input_tab_for_task,
+            inputs=[task_id],
+            outputs=[tiered_input_tabs, selected_tab_state],
+        )
+        tiered_input_tabs.select(on_tab_select, inputs=None, outputs=[selected_tab_state])
+
+        reset_btn.click(
+            handle_reset,
+            inputs=[env_state, task_id],
+            outputs=[
+                env_state,
+                corpus_table,
+                policy_display,
+                dominant_features_table,
+                best_score_disp,
+                steps_left_disp,
+                episode_disp,
+                corpus_count_disp,
+                reward_outcome_disp,
+                session_history_table,
+                session_history_cues,
+                session_total_reward,
+                raw_json_box,
+            ],
+        )
+        step_btn.click(
+            handle_step,
+            inputs=[
+                env_state,
+                task_id,
+                selected_tab_state,
+                easy_label_input,
+                easy_observation_input,
+                medium_label_input,
+                medium_primary_signal_input,
+                medium_conflicting_signal_input,
+                medium_reasoning_input,
+                hard_label_input,
+                hard_tradeoff_summary_input,
+                hard_confidence_justification_input,
+                hard_confidence_input,
+                clip_id_input,
+            ],
+            outputs=[
+                env_state,
+                corpus_table,
+                policy_display,
+                dominant_features_table,
+                best_score_disp,
+                steps_left_disp,
+                episode_disp,
+                corpus_count_disp,
+                reward_outcome_disp,
+                session_history_table,
+                session_history_cues,
+                session_total_reward,
+                raw_json_box,
+            ],
+        )
+        hint_btn.click(
+            handle_quality_hint,
+            inputs=[
+                env_state,
+                task_id,
+                selected_tab_state,
+                easy_observation_input,
+                medium_reasoning_input,
+                hard_tradeoff_summary_input,
+            ],
+            outputs=[
+                env_state,
+                easy_observation_input,
+                medium_reasoning_input,
+                hard_tradeoff_summary_input,
+            ],
+        )
+        baseline_run_btn.click(
+            _start_baseline_ui_run,
+            inputs=[task_id],
+            outputs=[baseline_run_id_state, baseline_status_disp, baseline_result_md, baseline_run_btn, baseline_poll_timer],
+        )
+        baseline_poll_timer.tick(
+            _poll_baseline_ui_run,
+            inputs=[baseline_run_id_state],
+            outputs=[baseline_run_id_state, baseline_status_disp, baseline_result_md, baseline_run_btn, baseline_poll_timer],
+        )
+
+    return demo
+
+
+custom_demo = build_custom_ui()
+app = gr.mount_gradio_app(app, custom_demo, path="/dashboard/")
+
+
+def main(host: str = "0.0.0.0", port: int = 8000) -> None:
+    uvicorn.run("server.app:app", host=host, port=port, reload=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    if args.port == 8000:
+        main()
+    else:
+        main(port=args.port)

server/baseline_runs.py

@@ -0,0 +1,160 @@
+"""Thread-safe in-memory tracker for async baseline runs."""
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+from copy import deepcopy
+from typing import Any, Callable, Literal, TypedDict
+
+RunStatus = Literal["running", "completed", "failed"]
+DEFAULT_RUN_TTL_SECONDS = 10 * 60
+
+
+class BaselineRun(TypedDict):
+    run_id: str
+    status: RunStatus
+    created_at: float
+    updated_at: float
+    expires_at: float
+    started_at: float
+    completed_at: float | None
+    failed_at: float | None
+    partial: Any
+    result: Any
+    error: Any
+
+
+class BaselineRunTracker:
+    """Manage async baseline runs in-memory with TTL-based cleanup."""
+
+    def __init__(
+        self,
+        ttl_seconds: float = DEFAULT_RUN_TTL_SECONDS,
+        time_fn: Callable[[], float] | None = None,
+    ) -> None:
+        if ttl_seconds <= 0:
+            raise ValueError("ttl_seconds must be > 0")
+        self._ttl_seconds = float(ttl_seconds)
+        self._time_fn = time_fn or time.time
+        self._lock = threading.RLock()
+        self._runs: dict[str, BaselineRun] = {}
+
+    def _now(self) -> float:
+        return float(self._time_fn())
+
+    def _is_expired(self, run: BaselineRun, now: float) -> bool:
+        return float(run["expires_at"]) <= now
+
+    def _touch(self, run: BaselineRun, now: float) -> None:
+        run["updated_at"] = now
+        run["expires_at"] = now + self._ttl_seconds
+
+    def _get_for_update(self, run_id: str, now: float) -> BaselineRun:
+        run = self._runs.get(run_id)
+        if run is None:
+            raise KeyError(f"Unknown run_id: {run_id}")
+        if self._is_expired(run, now):
+            del self._runs[run_id]
+            raise KeyError(f"Expired run_id: {run_id}")
+        return run
+
+    def _merge_partial(self, run: BaselineRun, payload: Any) -> None:
+        current = run.get("partial")
+        if isinstance(current, dict) and isinstance(payload, dict):
+            merged = deepcopy(current)
+            merged.update(deepcopy(payload))
+            run["partial"] = merged
+            return
+        run["partial"] = deepcopy(payload)
+
+    def create_run(self) -> str:
+        with self._lock:
+            now = self._now()
+            run_id = uuid.uuid4().hex
+            while run_id in self._runs:
+                run_id = uuid.uuid4().hex
+            self._runs[run_id] = BaselineRun(
+                run_id=run_id,
+                status="running",
+                created_at=now,
+                updated_at=now,
+                expires_at=now + self._ttl_seconds,
+                started_at=now,
+                completed_at=None,
+                failed_at=None,
+                partial={},
+                result=None,
+                error=None,
+            )
+            return run_id
+
+    def mark_running(self, run_id: str, payload: Any | None = None) -> BaselineRun:
+        with self._lock:
+            now = self._now()
+            run = self._get_for_update(run_id, now)
+            run["status"] = "running"
+            run["result"] = None
+            run["error"] = None
+            run["completed_at"] = None
+            run["failed_at"] = None
+            if payload is not None:
+                self._merge_partial(run, payload)
+            self._touch(run, now)
+            return deepcopy(run)
+
+    def update_partial(self, run_id: str, payload: Any) -> BaselineRun:
+        return self.mark_running(run_id, payload=payload)
+
+    def mark_complete(self, run_id: str, result: Any) -> BaselineRun:
+        with self._lock:
+            now = self._now()
+            run = self._get_for_update(run_id, now)
+            run["status"] = "completed"
+            run["result"] = deepcopy(result)
+            run["error"] = None
+            run["completed_at"] = now
+            run["failed_at"] = None
+            self._touch(run, now)
+            return deepcopy(run)
+
+    def mark_failed(self, run_id: str, error: Any) -> BaselineRun:
+        with self._lock:
+            now = self._now()
+            run = self._get_for_update(run_id, now)
+            run["status"] = "failed"
+            run["result"] = None
+            run["error"] = deepcopy(error)
+            run["completed_at"] = None
+            run["failed_at"] = now
+            self._touch(run, now)
+            return deepcopy(run)
+
+    def get_run(self, run_id: str) -> BaselineRun | None:
+        with self._lock:
+            now = self._now()
+            run = self._runs.get(run_id)
+            if run is None:
+                return None
+            if self._is_expired(run, now):
+                del self._runs[run_id]
+                return None
+            return deepcopy(run)
+
+    def cleanup_expired(self) -> int:
+        with self._lock:
+            now = self._now()
+            expired_ids = [run_id for run_id, run in self._runs.items() if self._is_expired(run, now)]
+            for run_id in expired_ids:
+                del self._runs[run_id]
+            return len(expired_ids)
+
+
+baseline_run_tracker = BaselineRunTracker()
+
+__all__ = [
+    "BaselineRun",
+    "BaselineRunTracker",
+    "DEFAULT_RUN_TTL_SECONDS",
+    "baseline_run_tracker",
+]

server/clip_quality_environment.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from clip_quality_env.env import ClipQualityEnvironment as BaseClipQualityEnvironment
+
+
+class ClipQualityEnvironment(BaseClipQualityEnvironment):
+    """Server runtime wrapper preserving expected import path."""
+
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True

server/environment.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from clip_quality_env.env import ClipQualityEnvironment
+
+# Backward-compatibility alias (deprecated).
+PolicyEvolverEnvironment = ClipQualityEnvironment
+
+__all__ = ["ClipQualityEnvironment"]

server/grader.py

@@ -0,0 +1,133 @@
+"""Deterministic clip-quality grader used by `/grader`."""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from clip_quality_env.difficulty import calibrate_total_score, normalize_difficulty
+from clip_quality_env.grader import grade as clip_grade
+from clip_quality_env.ground_truth import GTStore
+from clip_quality_env.models import Action
+from clip_quality_env.rubric import RubricState
+from server.tasks import TASK_REGISTRY
+
+
+_RUBRIC = RubricState()
+_GT = GTStore()
+_VALID_LABELS = {"KEEP", "BORDERLINE", "REJECT"}
+_TEXT_TOKEN_RE = re.compile(r"[a-z0-9_]+")
+
+
+def _clamp01(value: float) -> float:
+    return max(0.0, min(1.0, float(value)))
+
+
+def _resolve_clip(action_dict: dict[str, Any], task_id: str) -> dict[str, Any]:
+    task = TASK_REGISTRY.get(task_id, {})
+    corpus = task.get("data_corpus", [])
+    if not corpus:
+        return {}
+
+    requested_clip_id = action_dict.get("clip_id")
+    if requested_clip_id:
+        for clip in corpus:
+            if str(clip.get("clip_id", "")) == str(requested_clip_id):
+                return dict(clip)
+    return dict(corpus[0])
+
+
+def _normalize_label(raw: Any, fallback: str = "BORDERLINE") -> str:
+    label = str(raw or fallback).strip().upper()
+    return label if label in _VALID_LABELS else fallback
+
+
+def _normalize_confidence(raw: Any) -> float:
+    try:
+        return _clamp01(float(raw))
+    except (TypeError, ValueError):
+        return 0.5
+
+
+def _normalize_reasoning(action_dict: dict[str, Any]) -> str:
+    text_fields = [
+        action_dict.get("reasoning"),
+        action_dict.get("justification"),
+        action_dict.get("suggested_definition"),
+        action_dict.get("new_rule"),
+        action_dict.get("think"),
+    ]
+    parts = [str(part).strip() for part in text_fields if str(part or "").strip()]
+    if parts:
+        return " ".join(parts)
+    return "Reasoning references face_confidence, motion_score, audio_snr_db, and lighting_uniformity."
+
+
+def _normalize_action(action_dict: dict[str, Any], clip: dict[str, Any]) -> Action:
+    clip_id = str(clip.get("clip_id", ""))
+    fallback_label = _normalize_label(_GT.lookup(clip_id), fallback="")
+    if fallback_label not in _VALID_LABELS:
+        fallback_label = _normalize_label(_RUBRIC.derive_label(clip), fallback="BORDERLINE")
+    payload = {
+        "label": _normalize_label(
+            action_dict.get("label")
+            or action_dict.get("predicted_label")
+            or action_dict.get("decision")
+            or action_dict.get("review_status"),
+            fallback=fallback_label,
+        ),
+        "reasoning": _normalize_reasoning(action_dict),
+        "confidence": _normalize_confidence(action_dict.get("confidence", 0.5)),
+        "clip_id": str(action_dict.get("clip_id") or clip.get("clip_id") or ""),
+    }
+    return Action.model_validate(payload)
+
+
+def _mentions_cue(reasoning: str, cue: str) -> bool:
+    text_tokens = set(_TEXT_TOKEN_RE.findall(reasoning.lower()))
+    cue_tokens = [token for token in _TEXT_TOKEN_RE.findall(cue.lower()) if len(token) >= 4]
+    if not cue_tokens:
+        return False
+    overlap = sum(1 for token in cue_tokens if token in text_tokens)
+    needed = 2 if len(cue_tokens) >= 2 else 1
+    return overlap >= needed
+
+
+def _cue_bonus(reasoning: str, clip: dict[str, Any]) -> float:
+    cues = clip.get("quality_cues")
+    if not isinstance(cues, list):
+        return 0.0
+    valid_cues = [str(cue).strip() for cue in cues if str(cue).strip()]
+    if not valid_cues:
+        return 0.0
+    hits = sum(1 for cue in valid_cues if _mentions_cue(reasoning, cue))
+    if hits >= 2:
+        return 0.10
+    if hits == 1:
+        return 0.05
+    return 0.0
+
+
+def grade(action_dict: dict[str, Any], task_id: str, temperature: float = 0.0, seed: int = 42) -> float:
+    del temperature, seed
+    if task_id not in TASK_REGISTRY:
+        return 0.0
+    try:
+        clip = _resolve_clip(action_dict, task_id)
+        if not clip:
+            return 0.0
+
+        action = _normalize_action(action_dict, clip)
+        reward = clip_grade(action, clip, _RUBRIC, _GT)
+
+        format_score = float(reward.format_score)
+        label_score = float(reward.label_score)
+        reasoning_score = float(reward.reasoning_score)
+
+        reasoning_score = min(0.30, reasoning_score + _cue_bonus(str(action.reasoning), clip))
+        task = TASK_REGISTRY.get(task_id, {})
+        difficulty = normalize_difficulty(str(task.get("difficulty", "")))
+        total = _clamp01(format_score + label_score + reasoning_score)
+        total = calibrate_total_score(total, difficulty=difficulty)
+        return round(total, 4)
+    except Exception:
+        return 0.0

server/requirements.txt

@@ -0,0 +1,8 @@
+openenv-core>=0.2.3
+fastapi>=0.104.0
+uvicorn>=0.24.0
+pydantic>=2.0.0
+requests>=2.25.0
+websockets>=12.0
+gradio>=4.0.0
+pandas>=2.0.0

server/tasks/__init__.py

@@ -0,0 +1,13 @@
+from .task_easy import EASY_TASK
+from .task_medium import MEDIUM_TASK
+from .task_hard import HARD_TASK
+
+TASK_REGISTRY = {
+    "task_easy": EASY_TASK,
+    "task_medium": MEDIUM_TASK,
+    "task_hard": HARD_TASK,
+}
+
+TASK_IDS = tuple(TASK_REGISTRY.keys())
+
+__all__ = ["EASY_TASK", "MEDIUM_TASK", "HARD_TASK", "TASK_REGISTRY", "TASK_IDS"]

server/tasks/task_easy.py

@@ -0,0 +1,159 @@
+EASY_TASK = {
+    "task_id": "task_easy",
+    "difficulty": "easy",
+    "description": (
+        "Classify clips with dominant quality evidence. "
+        "Return one label in {KEEP, BORDERLINE, REJECT} with brief metadata-grounded reasoning."
+    ),
+    "data_corpus": [
+        {
+            "id": "easy_001",
+            "clip_id": "clip_0001",
+            "duration_s": 8.4,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.43,
+            "face_confidence": 0.94,
+            "head_pose_yaw_deg": 6.2,
+            "head_pose_pitch_deg": 4.3,
+            "motion_score": 0.09,
+            "bg_complexity": "solid_dark",
+            "bg_complexity_score": 0.05,
+            "mouth_open_ratio": 0.36,
+            "blink_rate_hz": 0.22,
+            "audio_snr_db": 27.4,
+            "transcript_word_count": 28,
+            "transcript_confidence": 0.89,
+            "lighting_uniformity": 0.88,
+            "occlusion_present": False,
+            "environment_tag": "studio",
+            "framing": "front",
+            "expected_label": "KEEP",
+            "quality_cues": [
+                "high face_confidence and large face_area_ratio",
+                "low motion_score and clean audio_snr_db",
+                "uniform lighting with no occlusion",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "easy_002",
+            "clip_id": "clip_0004",
+            "duration_s": 7.1,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.38,
+            "face_confidence": 0.90,
+            "head_pose_yaw_deg": 7.8,
+            "head_pose_pitch_deg": 5.0,
+            "motion_score": 0.12,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.09,
+            "mouth_open_ratio": 0.33,
+            "blink_rate_hz": 0.21,
+            "audio_snr_db": 24.1,
+            "transcript_word_count": 24,
+            "transcript_confidence": 0.86,
+            "lighting_uniformity": 0.80,
+            "occlusion_present": False,
+            "environment_tag": "office",
+            "framing": "front",
+            "expected_label": "KEEP",
+            "quality_cues": [
+                "face_confidence is high",
+                "motion_score is low",
+                "audio_snr_db is comfortably above keep threshold",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "easy_003",
+            "clip_id": "clip_0007",
+            "duration_s": 8.0,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.25,
+            "face_confidence": 0.80,
+            "head_pose_yaw_deg": 17.5,
+            "head_pose_pitch_deg": 8.2,
+            "motion_score": 0.26,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.16,
+            "mouth_open_ratio": 0.25,
+            "blink_rate_hz": 0.29,
+            "audio_snr_db": 19.4,
+            "transcript_word_count": 19,
+            "transcript_confidence": 0.76,
+            "lighting_uniformity": 0.65,
+            "occlusion_present": False,
+            "environment_tag": "home_office",
+            "framing": "front",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "face_area_ratio sits on the keep boundary",
+                "audio_snr_db and motion_score are borderline",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "easy_004",
+            "clip_id": "clip_0014",
+            "duration_s": 3.4,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.16,
+            "face_confidence": 0.56,
+            "head_pose_yaw_deg": 23.0,
+            "head_pose_pitch_deg": 11.2,
+            "motion_score": 0.54,
+            "bg_complexity": "busy_room",
+            "bg_complexity_score": 0.45,
+            "mouth_open_ratio": 0.15,
+            "blink_rate_hz": 0.38,
+            "audio_snr_db": 11.0,
+            "transcript_word_count": 8,
+            "transcript_confidence": 0.44,
+            "lighting_uniformity": 0.38,
+            "occlusion_present": True,
+            "environment_tag": "corridor",
+            "framing": "offgaze",
+            "expected_label": "REJECT",
+            "quality_cues": [
+                "occlusion_present is true",
+                "face_confidence is too low and motion_score is too high",
+                "audio_snr_db and lighting_uniformity are below reject thresholds",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "easy_005",
+            "clip_id": "clip_0016",
+            "duration_s": 5.0,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.18,
+            "face_confidence": 0.63,
+            "head_pose_yaw_deg": 25.5,
+            "head_pose_pitch_deg": 12.4,
+            "motion_score": 0.47,
+            "bg_complexity": "busy_indoor",
+            "bg_complexity_score": 0.42,
+            "mouth_open_ratio": 0.17,
+            "blink_rate_hz": 0.35,
+            "audio_snr_db": 13.1,
+            "transcript_word_count": 10,
+            "transcript_confidence": 0.51,
+            "lighting_uniformity": 0.42,
+            "occlusion_present": False,
+            "environment_tag": "street_interview",
+            "framing": "left",
+            "expected_label": "REJECT",
+            "quality_cues": [
+                "face_confidence is below keep range",
+                "motion_score is reject-level",
+                "audio_snr_db is noisy and lighting is poor",
+            ],
+            "review_status": "pending",
+        },
+    ],
+}

server/tasks/task_hard.py

@@ -0,0 +1,160 @@
+HARD_TASK = {
+    "task_id": "task_hard",
+    "difficulty": "hard",
+    "description": (
+        "Classify difficult clips with conflicting quality cues. "
+        "Reasoning should explain trade-offs across visual and audio metadata."
+    ),
+    "data_corpus": [
+        {
+            "id": "hard_001",
+            "clip_id": "clip_0012",
+            "duration_s": 8.6,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.31,
+            "face_confidence": 0.88,
+            "head_pose_yaw_deg": 15.5,
+            "head_pose_pitch_deg": 9.0,
+            "motion_score": 0.43,
+            "bg_complexity": "busy_room",
+            "bg_complexity_score": 0.34,
+            "mouth_open_ratio": 0.31,
+            "blink_rate_hz": 0.33,
+            "audio_snr_db": 14.2,
+            "transcript_word_count": 26,
+            "transcript_confidence": 0.79,
+            "lighting_uniformity": 0.60,
+            "occlusion_present": False,
+            "environment_tag": "event_hall",
+            "framing": "front",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "strong face_confidence conflicts with noisy audio_snr_db",
+                "motion_score is near reject but not over threshold",
+                "lighting is acceptable but not robust",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "hard_002",
+            "clip_id": "clip_0013",
+            "duration_s": 6.9,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.29,
+            "face_confidence": 0.73,
+            "head_pose_yaw_deg": 21.8,
+            "head_pose_pitch_deg": 11.0,
+            "motion_score": 0.46,
+            "bg_complexity": "busy_room",
+            "bg_complexity_score": 0.38,
+            "mouth_open_ratio": 0.22,
+            "blink_rate_hz": 0.36,
+            "audio_snr_db": 16.6,
+            "transcript_word_count": 18,
+            "transcript_confidence": 0.71,
+            "lighting_uniformity": 0.49,
+            "occlusion_present": False,
+            "environment_tag": "street_interview",
+            "framing": "offgaze",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "motion_score is high but audio_snr_db remains salvageable",
+                "face_confidence and lighting_uniformity are weak",
+                "multiple borderline cues create mixed confidence",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "hard_003",
+            "clip_id": "clip_0017",
+            "duration_s": 5.6,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.22,
+            "face_confidence": 0.66,
+            "head_pose_yaw_deg": 18.2,
+            "head_pose_pitch_deg": 9.4,
+            "motion_score": 0.33,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.20,
+            "mouth_open_ratio": 0.27,
+            "blink_rate_hz": 0.32,
+            "audio_snr_db": 20.2,
+            "transcript_word_count": 19,
+            "transcript_confidence": 0.79,
+            "lighting_uniformity": 0.57,
+            "occlusion_present": False,
+            "environment_tag": "home_office",
+            "framing": "left",
+            "expected_label": "REJECT",
+            "quality_cues": [
+                "face_confidence is near reject threshold",
+                "face_area_ratio is too small and motion is elevated",
+                "combined weak visual signals push toward reject",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "hard_004",
+            "clip_id": "clip_0018",
+            "duration_s": 11.8,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.18,
+            "face_confidence": 0.64,
+            "head_pose_yaw_deg": 24.4,
+            "head_pose_pitch_deg": 12.1,
+            "motion_score": 0.41,
+            "bg_complexity": "busy_indoor",
+            "bg_complexity_score": 0.41,
+            "mouth_open_ratio": 0.19,
+            "blink_rate_hz": 0.39,
+            "audio_snr_db": 13.9,
+            "transcript_word_count": 14,
+            "transcript_confidence": 0.66,
+            "lighting_uniformity": 0.44,
+            "occlusion_present": False,
+            "environment_tag": "conference_room",
+            "framing": "offgaze",
+            "expected_label": "REJECT",
+            "quality_cues": [
+                "face_confidence is below reject ceiling",
+                "audio_snr_db is noisy and lighting is poor",
+                "duration_s long but quality remains low",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "hard_005",
+            "clip_id": "clip_0006",
+            "duration_s": 7.7,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.30,
+            "face_confidence": 0.84,
+            "head_pose_yaw_deg": 10.3,
+            "head_pose_pitch_deg": 6.2,
+            "motion_score": 0.25,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.13,
+            "mouth_open_ratio": 0.31,
+            "blink_rate_hz": 0.24,
+            "audio_snr_db": 21.1,
+            "transcript_word_count": 24,
+            "transcript_confidence": 0.86,
+            "lighting_uniformity": 0.72,
+            "occlusion_present": False,
+            "environment_tag": "studio",
+            "framing": "front",
+            "expected_label": "KEEP",
+            "quality_cues": [
+                "face metrics are above keep threshold",
+                "audio_snr_db is clean and motion_score is controlled",
+                "consistent lighting and framing support keep decision",
+            ],
+            "review_status": "pending",
+        },
+    ],
+}

server/tasks/task_medium.py

@@ -0,0 +1,159 @@
+MEDIUM_TASK = {
+    "task_id": "task_medium",
+    "difficulty": "medium",
+    "description": (
+        "Classify clips with mixed or borderline evidence. "
+        "Use metadata trade-offs to justify KEEP/BORDERLINE/REJECT."
+    ),
+    "data_corpus": [
+        {
+            "id": "med_001",
+            "clip_id": "clip_0008",
+            "duration_s": 6.2,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.24,
+            "face_confidence": 0.78,
+            "head_pose_yaw_deg": 14.4,
+            "head_pose_pitch_deg": 9.3,
+            "motion_score": 0.29,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.18,
+            "mouth_open_ratio": 0.23,
+            "blink_rate_hz": 0.30,
+            "audio_snr_db": 18.5,
+            "transcript_word_count": 20,
+            "transcript_confidence": 0.75,
+            "lighting_uniformity": 0.62,
+            "occlusion_present": False,
+            "environment_tag": "office",
+            "framing": "left",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "face_confidence is slightly below keep threshold",
+                "audio_snr_db and motion_score are borderline",
+                "no hard reject signal",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "med_002",
+            "clip_id": "clip_0009",
+            "duration_s": 10.6,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.23,
+            "face_confidence": 0.77,
+            "head_pose_yaw_deg": 16.1,
+            "head_pose_pitch_deg": 9.8,
+            "motion_score": 0.31,
+            "bg_complexity": "simple_room",
+            "bg_complexity_score": 0.19,
+            "mouth_open_ratio": 0.24,
+            "blink_rate_hz": 0.31,
+            "audio_snr_db": 17.4,
+            "transcript_word_count": 23,
+            "transcript_confidence": 0.73,
+            "lighting_uniformity": 0.59,
+            "occlusion_present": False,
+            "environment_tag": "podcast_corner",
+            "framing": "front",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "duration_s is near upper keep band edge",
+                "face_area_ratio is borderline small",
+                "audio_snr_db is weak but not catastrophic",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "med_003",
+            "clip_id": "clip_0010",
+            "duration_s": 7.4,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.27,
+            "face_confidence": 0.82,
+            "head_pose_yaw_deg": 11.2,
+            "head_pose_pitch_deg": 7.7,
+            "motion_score": 0.24,
+            "bg_complexity": "office",
+            "bg_complexity_score": 0.14,
+            "mouth_open_ratio": 0.29,
+            "blink_rate_hz": 0.26,
+            "audio_snr_db": 20.1,
+            "transcript_word_count": 22,
+            "transcript_confidence": 0.81,
+            "lighting_uniformity": 0.66,
+            "occlusion_present": False,
+            "environment_tag": "workstation",
+            "framing": "front",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "face_confidence is good but face_area_ratio remains only moderate",
+                "motion_score and mouth_open_ratio are near boundary values",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "med_004",
+            "clip_id": "clip_0011",
+            "duration_s": 4.3,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.21,
+            "face_confidence": 0.69,
+            "head_pose_yaw_deg": 19.1,
+            "head_pose_pitch_deg": 10.6,
+            "motion_score": 0.39,
+            "bg_complexity": "busy_room",
+            "bg_complexity_score": 0.23,
+            "mouth_open_ratio": 0.21,
+            "blink_rate_hz": 0.34,
+            "audio_snr_db": 15.0,
+            "transcript_word_count": 13,
+            "transcript_confidence": 0.69,
+            "lighting_uniformity": 0.54,
+            "occlusion_present": False,
+            "environment_tag": "classroom",
+            "framing": "offgaze",
+            "expected_label": "BORDERLINE",
+            "quality_cues": [
+                "duration_s is barely acceptable",
+                "audio_snr_db and lighting_uniformity are borderline",
+                "motion_score is elevated but below hard reject cutoff",
+            ],
+            "review_status": "pending",
+        },
+        {
+            "id": "med_005",
+            "clip_id": "clip_0003",
+            "duration_s": 8.7,
+            "fps": 25,
+            "resolution": "1280x720",
+            "face_area_ratio": 0.36,
+            "face_confidence": 0.91,
+            "head_pose_yaw_deg": 8.1,
+            "head_pose_pitch_deg": 5.4,
+            "motion_score": 0.14,
+            "bg_complexity": "solid_dark",
+            "bg_complexity_score": 0.08,
+            "mouth_open_ratio": 0.34,
+            "blink_rate_hz": 0.24,
+            "audio_snr_db": 23.9,
+            "transcript_word_count": 29,
+            "transcript_confidence": 0.88,
+            "lighting_uniformity": 0.81,
+            "occlusion_present": False,
+            "environment_tag": "studio",
+            "framing": "front",
+            "expected_label": "KEEP",
+            "quality_cues": [
+                "strong face_confidence and face_area_ratio",
+                "clean audio and stable motion",
+                "well lit with no occlusion",
+            ],
+            "review_status": "pending",
+        },
+    ],
+}

spaces_app.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import os
+
+import gradio as gr
+
+from models import Action
+from server.environment import ClipQualityEnvironment
+from server.tasks import TASK_REGISTRY
+
+
+def _run_step(task_id: str, label: str, reasoning: str, confidence: float, clip_id: str) -> str:
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id=task_id)
+    payload = {
+        "label": label,
+        "reasoning": (reasoning or "No reasoning provided.").strip(),
+        "confidence": float(confidence),
+    }
+    if clip_id and clip_id.strip():
+        payload["clip_id"] = clip_id.strip()
+
+    next_obs = env.step(Action.model_validate(payload))
+    current_clip = next_obs.clip_metadata.clip_id
+    expected = next_obs.clip_metadata.expected_label
+    return (
+        f"Clip-quality task: {obs.task_id}\n"
+        f"Classified clip: {current_clip}\n"
+        f"Expected label: {expected}\n"
+        f"Step: {next_obs.step_count}\n"
+        f"Reward: {next_obs.reward:.4f}\n"
+        f"Done: {next_obs.done}\n"
+        f"Best quality score: {next_obs.info.get('best_score', 0.0):.4f}"
+    )
+
+
+def build_demo() -> gr.Blocks:
+    with gr.Blocks(title="CLIP Quality Analyzer Dashboard") as demo:
+        gr.Markdown("# CLIP Quality Analyzer Dashboard")
+        gr.Markdown("Reference-style dashboard for clip-quality classification.")
+
+        task_id = gr.Dropdown(choices=list(TASK_REGISTRY.keys()), value="task_easy", label="Clip-Quality Scenario")
+        label = gr.Radio(
+            choices=["KEEP", "BORDERLINE", "REJECT"],
+            value="BORDERLINE",
+            label="Predicted Label",
+        )
+        reasoning = gr.Textbox(label="Reasoning (reference clip metadata)")
+        confidence = gr.Slider(minimum=0.0, maximum=1.0, value=0.5, step=0.01, label="Confidence")
+        clip_id = gr.Textbox(label="Clip ID override (optional)")
+        run_btn = gr.Button("Submit Classification Action")
+        output = gr.Textbox(label="Classification Result", lines=8)
+        run_btn.click(_run_step, inputs=[task_id, label, reasoning, confidence, clip_id], outputs=[output])
+    return demo
+
+
+def _run_demo() -> None:
+    demo = build_demo()
+    demo.queue()
+    demo.launch(server_name=os.environ.get("GRADIO_SERVER_NAME", "0.0.0.0"), server_port=int(os.environ.get("PORT", "7860")))
+
+
+if __name__ == "__main__":
+    _run_demo()

state/ground_truth.json

@@ -0,0 +1,198 @@
+{
+  "clip_0001": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0002": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0003": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0004": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0005": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0006": {
+    "confidence": null,
+    "episode": 0,
+    "label": "KEEP",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0007": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0008": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0009": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0010": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0011": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0012": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0013": {
+    "confidence": null,
+    "episode": 0,
+    "label": "BORDERLINE",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0014": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0015": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0016": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0017": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0018": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_0019": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_002": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "REJECT",
+    "reward": 0.85,
+    "source": "agent_promoted"
+  },
+  "clip_0020": {
+    "confidence": null,
+    "episode": 0,
+    "label": "REJECT",
+    "reward": null,
+    "source": "seed"
+  },
+  "clip_003": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "REJECT",
+    "reward": 0.85,
+    "source": "agent_promoted"
+  },
+  "clip_004": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "REJECT",
+    "reward": 0.85,
+    "source": "agent_promoted"
+  },
+  "clip_008": {
+    "confidence": 0.85,
+    "episode": 21,
+    "label": "REJECT",
+    "reward": 0.856,
+    "source": "agent_promoted"
+  },
+  "clip_010": {
+    "confidence": 0.96,
+    "episode": 9,
+    "label": "REJECT",
+    "reward": 0.95,
+    "source": "agent_promoted"
+  },
+  "clip_012": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "REJECT",
+    "reward": 0.95,
+    "source": "agent_promoted"
+  },
+  "clip_019": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "KEEP",
+    "reward": 0.95,
+    "source": "agent_promoted"
+  },
+  "clip_020": {
+    "confidence": 0.82,
+    "episode": 1,
+    "label": "REJECT",
+    "reward": 0.95,
+    "source": "agent_promoted"
+  }
+}

state/history.jsonl

@@ -0,0 +1,98 @@
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_e70a0096", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.25}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_333f90f4", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.9}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_2e4845a7", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.455, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_ac48dccf", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_7830fef5", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_b9961414", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_b373bc97", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_39a6f2aa", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_4858f1a1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_b89db641", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_46ec41c6", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_61e09b7f", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.2, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_195c392b", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_65e03fc7", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_aace83de", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_9bbc2e2c", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_2fff2df8", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_cde64aa0", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 6, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_652d1f4e", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_e8c2e65a", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_2284b99a", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 7, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_92a79235", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_ad4dec94", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_597f5b02", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 8, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_c70d500a", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_b630f24a", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_4cf710c4", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 9, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_ba850490", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_1a2941bb", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_0dd7abbc", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 10, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_50f3f686", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_aafe5c00", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_c109cd91", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_24b3c30c", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_1f0d711b", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_83b8f2a8", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_0170e61e", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_446460c1", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_60287218", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_6f73370f", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_3c1e3756", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_1caa4633", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.2, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_0001", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "real_medium_0001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_0001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_0001", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "real_medium_0001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_0001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.32, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.4125, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_007", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "REJECT", "expected_label": "REJECT", "reward": 0.8}], "ep_reward": 0.8925, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_015", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.87, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_021", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_003", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.64, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.9}, {"step": 2, "difficulty": "medium", "clip_id": "clip_014", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_002", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.62, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.4125, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "clip_007", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_015", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.8, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_021", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_003", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_014", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_002", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 6, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_023", "label": "REJECT", "expected_label": "REJECT", "reward": 0.8}], "ep_reward": 0.8, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 7, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_011", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_005", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.6825, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 8, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "clip_013", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.6125, "gt_promoted": false, "gt_size": 20, "rubric_version": 1}
+{"episode": 9, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_022", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}], "ep_reward": 0.9775, "gt_promoted": true, "gt_size": 21, "rubric_version": 1}
+{"episode": 10, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "clip_016", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.62, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.32, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "clip_007", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.41, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_015", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.8, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_021", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_003", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_014", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_002", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 6, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_023", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 7, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "clip_011", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_005", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 8, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_013", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 9, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_022", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.32, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 10, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_016", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.59, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.4125, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "REJECT", "expected_label": "REJECT", "reward": 0.9}, {"step": 2, "difficulty": "medium", "clip_id": "clip_007", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}], "ep_reward": 0.94, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.4}], "ep_reward": 0.5025, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_009", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.35}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.3725, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "clip_007", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.95}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}], "ep_reward": 0.9825, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_015", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.87, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_021", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "clip_003", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.6, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_014", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_002", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 6, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_020", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_001", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_023", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 7, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_012", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}, {"step": 2, "difficulty": "medium", "clip_id": "clip_011", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_005", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.47, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 8, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_019", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_013", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "clip_004", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.53, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 9, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_017", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_022", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_010", "label": "BORDERLINE", "expected_label": "REJECT", "reward": 0.2}], "ep_reward": 0.32, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 10, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "clip_018", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}, {"step": 2, "difficulty": "medium", "clip_id": "clip_016", "label": "BORDERLINE", "expected_label": "KEEP", "reward": 0.2}, {"step": 3, "difficulty": "hard", "clip_id": "clip_008", "label": "BORDERLINE", "expected_label": "BORDERLINE", "reward": 0.8}], "ep_reward": 0.59, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_02bc3f2a", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_e7440a32", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_467b78ba", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}], "ep_reward": 0.8275, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_81082325", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_91e25287", "label": "REJECT", "expected_label": "REJECT", "reward": 0.8}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_133040f5", "label": "REJECT", "expected_label": "REJECT", "reward": 0.8}], "ep_reward": 0.84, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_041b3590", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_cb64ce2a", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_e99ab288", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}], "ep_reward": 0.85, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 4, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_5950673c", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_7291cb9c", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_24f67412", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 5, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_a24a4892", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_fc846ffd", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_f717eeec", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}], "ep_reward": 0.85, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 6, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_5e6cc0b9", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_8629bbf7", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_139a4dbd", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}], "ep_reward": 0.86, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 7, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_ac0fbcc2", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_6c05fde8", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_4feb4a68", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 8, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_528650e8", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_7ad11dd3", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_2170965a", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.68, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 9, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_7d7b74b8", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_1ea581d7", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_763cc786", "label": "REJECT", "expected_label": "REJECT", "reward": 0.95}], "ep_reward": 0.8275, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 10, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "syn_easy_278c97de", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "syn_med_4b1e494a", "label": "REJECT", "expected_label": "REJECT", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "syn_hard_a774a48f", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.45}], "ep_reward": 0.7425, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 1, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 2, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}
+{"episode": 3, "steps": [{"step": 1, "difficulty": "easy", "clip_id": "real_easy_1", "label": "KEEP", "expected_label": "KEEP", "reward": 0.95}, {"step": 2, "difficulty": "medium", "clip_id": "real_med_1", "label": "KEEP", "expected_label": "KEEP", "reward": 1.0}, {"step": 3, "difficulty": "hard", "clip_id": "real_hard_1", "label": "KEEP", "expected_label": "BORDERLINE", "reward": 0.6}], "ep_reward": 0.81, "gt_promoted": false, "gt_size": 21, "rubric_version": 1}

tests/test_baseline_runs.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from server.baseline_runs import BaselineRunTracker, DEFAULT_RUN_TTL_SECONDS
+
+
+class _FakeClock:
+    def __init__(self, start: float = 0.0) -> None:
+        self.value = float(start)
+
+    def time(self) -> float:
+        return self.value
+
+    def advance(self, seconds: float) -> None:
+        self.value += float(seconds)
+
+
+def test_create_run_defaults_to_running_state():
+    clock = _FakeClock(start=100.0)
+    tracker = BaselineRunTracker(time_fn=clock.time)
+
+    run_id = tracker.create_run()
+    run = tracker.get_run(run_id)
+
+    assert run is not None
+    assert run["run_id"] == run_id
+    assert run["status"] == "running"
+    assert run["partial"] == {}
+    assert run["result"] is None
+    assert run["error"] is None
+    assert run["created_at"] == 100.0
+    assert run["updated_at"] == 100.0
+    assert run["started_at"] == 100.0
+    assert run["completed_at"] is None
+    assert run["failed_at"] is None
+    assert run["expires_at"] == 100.0 + DEFAULT_RUN_TTL_SECONDS
+
+
+def test_update_partial_merges_and_refreshes_ttl():
+    clock = _FakeClock(start=10.0)
+    tracker = BaselineRunTracker(ttl_seconds=10.0, time_fn=clock.time)
+
+    run_id = tracker.create_run()
+    clock.advance(3.0)
+    tracker.update_partial(run_id, {"processed": 1})
+
+    first = tracker.get_run(run_id)
+    assert first is not None
+    assert first["status"] == "running"
+    assert first["partial"] == {"processed": 1}
+    assert first["expires_at"] == 23.0
+
+    clock.advance(2.0)
+    tracker.update_partial(run_id, {"total": 5})
+    second = tracker.get_run(run_id)
+
+    assert second is not None
+    assert second["partial"] == {"processed": 1, "total": 5}
+    assert second["expires_at"] == 25.0
+
+
+def test_status_transitions_failed_to_running_to_complete():
+    clock = _FakeClock(start=200.0)
+    tracker = BaselineRunTracker(time_fn=clock.time)
+
+    run_id = tracker.create_run()
+    clock.advance(1.0)
+    tracker.mark_failed(run_id, {"message": "transient error"})
+
+    failed = tracker.get_run(run_id)
+    assert failed is not None
+    assert failed["status"] == "failed"
+    assert failed["error"] == {"message": "transient error"}
+    assert failed["failed_at"] == 201.0
+
+    clock.advance(1.0)
+    tracker.mark_running(run_id, {"retry": 1})
+    running = tracker.get_run(run_id)
+    assert running is not None
+    assert running["status"] == "running"
+    assert running["error"] is None
+    assert running["failed_at"] is None
+    assert running["partial"] == {"retry": 1}
+
+    clock.advance(1.0)
+    tracker.mark_complete(run_id, {"average_score": 0.87})
+    completed = tracker.get_run(run_id)
+    assert completed is not None
+    assert completed["status"] == "completed"
+    assert completed["result"] == {"average_score": 0.87}
+    assert completed["error"] is None
+    assert completed["completed_at"] == 203.0
+
+
+def test_cleanup_expired_respects_ttl_and_touch_updates():
+    clock = _FakeClock(start=0.0)
+    tracker = BaselineRunTracker(ttl_seconds=10.0, time_fn=clock.time)
+
+    stale_run = tracker.create_run()
+    refreshed_run = tracker.create_run()
+
+    clock.advance(6.0)
+    tracker.update_partial(refreshed_run, {"progress": 0.5})
+
+    clock.advance(5.0)
+    removed = tracker.cleanup_expired()
+    assert removed == 1
+    assert tracker.get_run(stale_run) is None
+    assert tracker.get_run(refreshed_run) is not None
+
+    clock.advance(6.0)
+    assert tracker.cleanup_expired() == 1
+    assert tracker.get_run(refreshed_run) is None

tests/test_environment.py

@@ -0,0 +1,354 @@
+from __future__ import annotations
+
+import json
+from statistics import mean
+
+from clip_quality_env.env import ClipQualityEnvironment
+from clip_quality_env.ground_truth import GTStore
+from clip_quality_env.models import Action
+from clip_quality_env.rubric import RubricState
+
+
+def _action_for_clip(clip_id: str, label: str = "BORDERLINE") -> Action:
+    return Action.model_validate(
+        {
+            "label": label,
+            "reasoning": f"{label} decision for {clip_id} using clip metadata cues.",
+            "confidence": 0.8,
+            "clip_id": clip_id,
+        }
+    )
+
+
+def test_environment_reset_returns_task_observation():
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_easy")
+    assert obs.task_id == "task_easy"
+    assert obs.step_count == 0
+    assert obs.corpus_size == obs.corpus_shown
+    assert len(obs.data_corpus) == obs.corpus_size
+    assert obs.clip_metadata.clip_id
+    assert obs.max_steps == 5
+    assert obs.info["steps_remaining"] == 5
+
+
+def test_environment_step_updates_state_and_reward():
+    env = ClipQualityEnvironment()
+    env.reset(task_id="task_easy")
+    action = Action.model_validate(
+        {
+            "label": "KEEP",
+            "reasoning": "face_confidence and lighting_uniformity are high, motion_score is low, so keep.",
+            "confidence": 0.9,
+        }
+    )
+    next_obs = env.step(action)
+    assert next_obs.step_count == 1
+    assert 0.0 <= next_obs.reward <= 1.0
+    assert next_obs.history
+    state = env.state
+    assert state.step_count == 1
+    assert state.actions_taken[-1] in {"KEEP", "BORDERLINE", "REJECT"}
+
+
+def test_environment_observation_exposes_reward_decomposition():
+    env = ClipQualityEnvironment()
+    reset_obs = env.reset(task_id="task_easy")
+
+    assert reset_obs.info["format_score"] == 0.0
+    assert reset_obs.info["label_score"] == 0.0
+    assert reset_obs.info["reasoning_score"] == 0.0
+    assert reset_obs.info["raw_total"] == 0.0
+    assert reset_obs.info["calibrated_total"] == 0.0
+    assert reset_obs.info["reward_total"] == 0.0
+    assert reset_obs.info["total_reward"] == 0.0
+    assert reset_obs.info["reward_breakdown"] == {
+        "format_score": 0.0,
+        "label_score": 0.0,
+        "reasoning_score": 0.0,
+        "raw_total": 0.0,
+        "calibrated_total": 0.0,
+        "total_reward": 0.0,
+    }
+
+    action = Action.model_validate(
+        {
+            "label": "KEEP",
+            "reasoning": "face_confidence and lighting_uniformity are high, motion_score is low, so keep.",
+            "confidence": 0.9,
+        }
+    )
+    step_obs = env.step(action)
+
+    assert step_obs.info["format_score"] in {0.0, 0.1}
+    assert step_obs.info["label_score"] in {0.0, 0.25, 0.6}
+    assert 0.0 <= step_obs.info["reasoning_score"] <= 0.3
+    assert abs(
+        float(step_obs.info["raw_total"])
+        - (
+            float(step_obs.info["format_score"])
+            + float(step_obs.info["label_score"])
+            + float(step_obs.info["reasoning_score"])
+        )
+    ) < 1e-9
+    assert abs(float(step_obs.info["calibrated_total"]) - float(step_obs.reward)) < 1e-9
+    assert abs(float(step_obs.info["reward_total"]) - float(step_obs.reward)) < 1e-9
+    assert abs(float(step_obs.info["total_reward"]) - float(env.state.total_reward)) < 1e-9
+    assert step_obs.info["reward_breakdown"] == {
+        "format_score": float(step_obs.info["format_score"]),
+        "label_score": float(step_obs.info["label_score"]),
+        "reasoning_score": float(step_obs.info["reasoning_score"]),
+        "raw_total": float(step_obs.info["raw_total"]),
+        "calibrated_total": float(step_obs.info["calibrated_total"]),
+        "total_reward": float(step_obs.info["reward_total"]),
+    }
+
+
+def test_environment_derives_expected_label_when_manifest_value_is_null(monkeypatch, tmp_path):
+    manifest_path = tmp_path / "manifest_null_expected.jsonl"
+    manifest_row = {
+        "difficulty": "easy",
+        "clip_id": "manifest_null_expected",
+        "expected_label": None,
+        "face_confidence": 0.88,
+        "motion_score": 0.12,
+        "audio_snr_db": 24.0,
+        "lighting_uniformity": 0.8,
+        "duration_s": 8.0,
+    }
+    manifest_path.write_text(json.dumps(manifest_row) + "\n", encoding="utf-8")
+
+    monkeypatch.setenv("REAL_CLIPS_MANIFEST", str(manifest_path))
+    monkeypatch.setenv("CLIP_CORPUS_SOURCE", "manifest")
+
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_easy", seed=2026)
+
+    assert obs.info["corpus_source"] == "manifest:easy"
+    assert str(obs.clip_metadata.expected_label).upper() in {"KEEP", "BORDERLINE", "REJECT"}
+    assert str(obs.clip_metadata.expected_label).upper() != "NONE"
+
+
+def test_environment_can_force_task_registry_source(monkeypatch):
+    monkeypatch.setenv("CLIP_CORPUS_SOURCE", "task_registry")
+
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_easy", seed=2026)
+
+    assert obs.info["corpus_mode"] == "task_registry"
+    assert obs.info["corpus_source"] == "task_registry:task_easy"
+
+
+def test_environment_observation_includes_full_unsliced_corpus():
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_medium")
+    synthetic_corpus = []
+    template = dict(obs.data_corpus[0]) if obs.data_corpus else {
+        "expected_label": "BORDERLINE",
+        "review_status": "pending",
+    }
+    for idx in range(12):
+        item = dict(template)
+        item["id"] = f"synthetic_{idx:03d}"
+        item["clip_id"] = f"clip_synth_{idx:03d}"
+        item["expected_label"] = str(item.get("expected_label") or "BORDERLINE")
+        item["review_status"] = "pending"
+        synthetic_corpus.append(item)
+    env._episode_corpus[obs.task_id] = synthetic_corpus
+
+    full_obs = env._state_to_observation(reward=0.0, done=False)
+
+    assert full_obs.corpus_size == len(synthetic_corpus)
+    assert full_obs.corpus_shown == len(synthetic_corpus)
+    assert len(full_obs.data_corpus) == len(synthetic_corpus)
+    assert [item["clip_id"] for item in full_obs.data_corpus] == [
+        item["clip_id"] for item in synthetic_corpus
+    ]
+
+
+def test_environment_step_updates_submitted_clip_review_status():
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_medium")
+    current_clip_id = obs.clip_metadata.clip_id
+
+    current_row = next(item for item in obs.data_corpus if item["clip_id"] == current_clip_id)
+    assert str(current_row.get("review_status", "")).lower() == "pending"
+
+    action = Action.model_validate(
+        {
+            "label": "REJECT",
+            "reasoning": "multiple weak cues indicate this clip should be rejected.",
+            "confidence": 0.81,
+            "clip_id": current_clip_id,
+        }
+    )
+    next_obs = env.step(action)
+    updated_row = next(item for item in next_obs.data_corpus if item["clip_id"] == current_clip_id)
+
+    assert updated_row["review_status"] == "REJECT"
+
+
+def test_environment_instances_do_not_share_runtime_state():
+    env_a = ClipQualityEnvironment()
+    env_b = ClipQualityEnvironment()
+
+    env_a.reset(task_id="task_easy")
+    action = Action.model_validate(
+        {
+            "label": "KEEP",
+            "reasoning": "face_confidence and lighting_uniformity are high, motion_score is low, so keep.",
+            "confidence": 0.9,
+        }
+    )
+    env_a.step(action)
+
+    assert env_a is not env_b
+    assert env_a.state.step_count == 1
+    assert env_b.state.step_count == 0
+    assert env_b.state.actions_taken == []
+
+
+def test_environment_reset_plans_five_clips_from_selected_corpus():
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_medium", seed=1234)
+    expected_source = obs.info["corpus_source"]
+    expected_data = list(obs.data_corpus)
+    expected_ids = {str(item["clip_id"]) for item in expected_data}
+    first_plan_ids = [str(item.clip.get("clip_id", "")) for item in env._episode_plan]
+
+    assert obs.max_steps == 5
+    assert obs.info["steps_remaining"] == 5
+    assert len(env._episode_plan) == 5
+    assert {item.task_id for item in env._episode_plan} == {"task_medium"}
+    assert set(first_plan_ids).issubset(expected_ids)
+
+    repeated = env.reset(task_id="task_medium", seed=1234)
+    repeated_plan_ids = [str(item.clip.get("clip_id", "")) for item in env._episode_plan]
+    assert repeated.max_steps == 5
+    assert repeated.info["steps_remaining"] == 5
+    assert repeated.info["corpus_source"] == expected_source
+    assert repeated.data_corpus == expected_data
+    assert repeated_plan_ids == first_plan_ids
+
+
+def test_environment_updates_review_status_across_queue_and_summary():
+    env = ClipQualityEnvironment()
+    obs = env.reset(task_id="task_hard", seed=2026)
+
+    assert obs.info["steps_remaining"] == 5
+    assert "episode_summary" not in obs.info
+
+    submitted: list[tuple[str, str]] = []
+    final_obs = obs
+    for step_index in range(1, 6):
+        current_clip_id = final_obs.clip_metadata.clip_id
+        label = "KEEP" if step_index % 2 else "BORDERLINE"
+        submitted.append((current_clip_id, label))
+        final_obs = env.step(_action_for_clip(current_clip_id, label=label))
+        queue_row = next(item for item in final_obs.data_corpus if item.get("clip_id") == current_clip_id)
+        assert str(queue_row.get("review_status")) == label
+        assert final_obs.info["steps_remaining"] == max(0, 5 - step_index)
+
+    assert final_obs.done is True
+    assert final_obs.step_count == 5
+    assert "episode_summary" in final_obs.info
+    summary = final_obs.info["episode_summary"]
+    assert summary["steps_completed"] == 5
+    assert summary["max_steps"] == 5
+    assert abs(float(summary["total_reward"]) - round(float(env.state.total_reward), 4)) < 1e-9
+    assert abs(float(final_obs.info["total_reward"]) - float(env.state.total_reward)) < 1e-9
+    assert abs(float(summary["average_reward"]) - round(float(env.state.total_reward) / 5.0, 4)) < 1e-9
+
+    final_status_map = {str(item["clip_id"]): str(item["review_status"]) for item in final_obs.data_corpus}
+    for clip_id, label in submitted:
+        assert final_status_map[clip_id] == label
+
+
+def test_environment_task_averages_follow_hard_medium_easy_order(monkeypatch, tmp_path):
+    monkeypatch.setenv("REAL_CLIPS_MANIFEST", str(tmp_path / "missing_manifest.jsonl"))
+
+    def run_task_average(task_id: str) -> float:
+        env = ClipQualityEnvironment()
+        env._rubric = RubricState(path=str(tmp_path / f"rubric_{task_id}.json"))
+        env._gt_store = GTStore(
+            seed_path="data/seed_gt.json",
+            state_path=str(tmp_path / f"ground_truth_{task_id}.json"),
+        )
+
+        obs = env.reset(task_id=task_id, seed=2026)
+        step_rewards: list[float] = []
+        while True:
+            expected_label = str(obs.clip_metadata.expected_label or "BORDERLINE")
+            action = Action.model_validate(
+                {
+                    "label": expected_label,
+                    "reasoning": (
+                        "face_confidence, motion_score, audio_snr_db, and lighting_uniformity "
+                        "support this decision."
+                    ),
+                    "confidence": 0.9,
+                    "clip_id": obs.clip_metadata.clip_id,
+                }
+            )
+            obs = env.step(action)
+            step_rewards.append(float(obs.reward))
+            if obs.done:
+                break
+        return float(mean(step_rewards))
+
+    easy_avg = run_task_average("task_easy")
+    medium_avg = run_task_average("task_medium")
+    hard_avg = run_task_average("task_hard")
+
+    assert hard_avg > medium_avg > easy_avg
+
+
+def test_environment_difficulty_score_ranges_are_strictly_ordered(monkeypatch, tmp_path):
+    monkeypatch.setenv("REAL_CLIPS_MANIFEST", str(tmp_path / "missing_manifest.jsonl"))
+
+    def run_task_range(task_id: str) -> tuple[float, float]:
+        task_scores: list[float] = []
+        labels = ("KEEP", "BORDERLINE", "REJECT")
+        reasoning_cases = (
+            "x",
+            "face_confidence, motion_score, audio_snr_db, and lighting_uniformity support this decision.",
+        )
+        confidence_cases = (0.0, 0.5, 1.0)
+
+        for label in labels:
+            for reasoning in reasoning_cases:
+                for confidence in confidence_cases:
+                    env = ClipQualityEnvironment()
+                    env._rubric = RubricState(path=str(tmp_path / f"rubric_range_{task_id}.json"))
+                    env._gt_store = GTStore(
+                        seed_path="data/seed_gt.json",
+                        state_path=str(tmp_path / f"ground_truth_range_{task_id}.json"),
+                    )
+
+                    obs = env.reset(task_id=task_id, seed=2026)
+                    while True:
+                        action = Action.model_validate(
+                            {
+                                "label": label,
+                                "reasoning": reasoning,
+                                "confidence": confidence,
+                                "clip_id": obs.clip_metadata.clip_id,
+                            }
+                        )
+                        obs = env.step(action)
+                        task_scores.append(float(obs.reward))
+                        if obs.done:
+                            break
+
+        return min(task_scores), max(task_scores)
+
+    easy_min, easy_max = run_task_range("task_easy")
+    medium_min, medium_max = run_task_range("task_medium")
+    hard_min, hard_max = run_task_range("task_hard")
+
+    assert easy_max < medium_min
+    assert medium_max < hard_min
+
+    assert 0.0 <= easy_min <= easy_max <= 1.0
+    assert 0.0 <= medium_min <= medium_max <= 1.0
+    assert 0.0 <= hard_min <= hard_max <= 1.0

tests/test_grader.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+from statistics import mean
+
+import pytest
+
+import server.grader as grader_module
+from clip_quality_env.difficulty import DIFFICULTY_TOTAL_BANDS
+from clip_quality_env.ground_truth import GTStore
+from clip_quality_env.rubric import RubricState
+from server.tasks import TASK_REGISTRY
+from server.grader import grade
+
+
+@pytest.fixture
+def isolated_grader_state(monkeypatch, tmp_path):
+    rubric = RubricState(path=str(tmp_path / "rubric.json"))
+    gt = GTStore(seed_path="data/seed_gt.json", state_path=str(tmp_path / "ground_truth.json"))
+    monkeypatch.setattr(grader_module, "_RUBRIC", rubric)
+    monkeypatch.setattr(grader_module, "_GT", gt)
+    return rubric, gt
+
+
+def test_grade_easy_clip_quality_scores_in_easy_band():
+    action = {
+        "label": "KEEP",
+        "clip_id": "clip_0001",
+        "reasoning": "face_confidence and audio_snr_db are high while motion_score is low, so this clip should be kept.",
+        "confidence": 0.9,
+    }
+    score = grade(action, "task_easy")
+    low, high = DIFFICULTY_TOTAL_BANDS["easy"]
+    assert low <= score <= high
+
+
+def test_grade_medium_clip_quality_scores_in_range():
+    action = {
+        "label": "BORDERLINE",
+        "clip_id": "clip_0008",
+        "reasoning": "face_area_ratio is borderline and motion_score is elevated, so borderline is safest.",
+        "confidence": 0.74,
+    }
+    score = grade(action, "task_medium")
+    low, high = DIFFICULTY_TOTAL_BANDS["medium"]
+    assert low <= score <= high
+
+
+def test_grade_hard_clip_quality_scores_in_range():
+    action = {
+        "label": "REJECT",
+        "clip_id": "clip_0017",
+        "reasoning": "face_confidence is weak, face_area_ratio is small, and motion_score is high enough to reject.",
+        "confidence": 0.83,
+    }
+    score = grade(action, "task_hard")
+    low, high = DIFFICULTY_TOTAL_BANDS["hard"]
+    assert low <= score <= high
+
+
+def test_grade_legacy_payload_is_supported_and_bounded():
+    action = {
+        "action_type": "propose_clarification",
+        "ambiguous_term": "appropriate",
+        "suggested_definition": "Keep clips with stable framing and clear speech; reject clips with severe occlusion.",
+        "justification": "Makes decisions consistent for borderline metadata combinations.",
+    }
+    score = grade(action, "task_easy")
+    assert 0.0 <= score <= 1.0
+
+
+def test_grade_task_averages_follow_hard_medium_easy_order(isolated_grader_state):
+    del isolated_grader_state
+
+    def task_average(task_id: str) -> float:
+        scores: list[float] = []
+        for clip in TASK_REGISTRY[task_id]["data_corpus"]:
+            action = {
+                "label": str(clip.get("expected_label", "BORDERLINE")),
+                "clip_id": str(clip.get("clip_id", "")),
+                "reasoning": (
+                    "face_confidence, motion_score, audio_snr_db, and lighting_uniformity "
+                    "support this decision."
+                ),
+                "confidence": 0.9,
+            }
+            scores.append(grade(action, task_id))
+        return float(mean(scores))
+
+    easy_avg = task_average("task_easy")
+    medium_avg = task_average("task_medium")
+    hard_avg = task_average("task_hard")
+
+    assert hard_avg > medium_avg > easy_avg
+
+
+def test_grade_difficulty_bands_do_not_overlap(isolated_grader_state):
+    del isolated_grader_state
+
+    labels = ("KEEP", "BORDERLINE", "REJECT")
+    reasoning_cases = (
+        "x",
+        "face_confidence, motion_score, audio_snr_db, and lighting_uniformity support this decision.",
+    )
+    confidence_cases = (0.0, 0.5, 1.0)
+
+    ranges: dict[str, tuple[float, float]] = {}
+    for task_id in ("task_easy", "task_medium", "task_hard"):
+        task_scores: list[float] = []
+        for clip in TASK_REGISTRY[task_id]["data_corpus"]:
+            clip_id = str(clip.get("clip_id", ""))
+            for label in labels:
+                for reasoning in reasoning_cases:
+                    for confidence in confidence_cases:
+                        task_scores.append(
+                            grade(
+                                {
+                                    "label": label,
+                                    "clip_id": clip_id,
+                                    "reasoning": reasoning,
+                                    "confidence": confidence,
+                                },
+                                task_id,
+                            )
+                        )
+        ranges[task_id] = (min(task_scores), max(task_scores))
+
+    easy_min, easy_max = ranges["task_easy"]
+    medium_min, medium_max = ranges["task_medium"]
+    hard_min, hard_max = ranges["task_hard"]
+
+    assert easy_min >= DIFFICULTY_TOTAL_BANDS["easy"][0]
+    assert easy_max <= DIFFICULTY_TOTAL_BANDS["easy"][1]
+    assert medium_min >= DIFFICULTY_TOTAL_BANDS["medium"][0]
+    assert medium_max <= DIFFICULTY_TOTAL_BANDS["medium"][1]
+    assert hard_min >= DIFFICULTY_TOTAL_BANDS["hard"][0]
+    assert hard_max <= DIFFICULTY_TOTAL_BANDS["hard"][1]
+
+    assert easy_max < medium_min
+    assert medium_max < hard_min

tests/test_inference.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import Any
+
+import inference
+
+
+def test_inference_extract_json_handles_fenced_blocks():
+    raw = "```json\n{\"x\": 1}\n```"
+    parsed = inference._extract_json(raw)
+    assert parsed == {"x": 1}
+
+
+def test_run_baseline_works_without_token(monkeypatch):
+    monkeypatch.delenv("HF_TOKEN", raising=False)
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    result = inference.run_baseline(task="task_easy")
+    assert result["detail"][0]["task_id"] == "task_easy"
+    assert result["detail"][0]["mode"] == "fallback"
+    assert result["detail"][0]["steps"] == 5
+    assert 0.0 <= result["detail"][0]["total_reward"] <= 5.0
+    assert 0.0 <= result["detail"][0]["final_reward"] <= 1.0
+    assert 0.0 <= result["detail"][0]["reward"] <= 1.0
+    assert 0.0 <= result["baseline_scores"]["overall_avg"] <= 1.0
+
+
+class _DummyMessage:
+    def __init__(self, content: str):
+        self.content = content
+
+
+class _DummyChoice:
+    def __init__(self, content: str):
+        self.message = _DummyMessage(content)
+
+
+class _DummyResponse:
+    def __init__(self, content: str):
+        self.choices = [_DummyChoice(content)]
+
+
+class _DummyCompletions:
+    def __init__(self, content: str):
+        self._content = content
+
+    def create(self, **_: Any):
+        return _DummyResponse(self._content)
+
+
+class _DummyChat:
+    def __init__(self, content: str):
+        self.completions = _DummyCompletions(content)
+
+
+class _DummyClient:
+    def __init__(self, content: str):
+        self.chat = _DummyChat(content)
+
+
+def test_run_episode_preserves_logging_and_structure(capsys):
+    client = _DummyClient('{"label":"KEEP","reasoning":"face_confidence and motion_score indicate keep","confidence":0.9}')
+    result = inference.run_episode("task_easy", client, "dummy-model")
+    output = capsys.readouterr().out
+    assert "[START]" in output and "[STEP]" in output and "[END]" in output
+    assert "total_reward=" in output and "final_reward=" in output
+    assert result["task_id"] == "task_easy"
+    assert result["mode"] == "llm"
+    assert result["steps"] == 5
+    assert 0.0 <= result["final_reward"] <= 1.0
+    assert 0.0 <= result["total_reward"] <= 5.0
+    assert 0.0 <= result["reward"] <= 1.0
+    assert abs(result["reward"] * result["steps"] - result["total_reward"]) < 1e-9