feat: reproducible eval + shared policy module; fix smart_agent wasted-capacity bug

.github/workflows/ci.yml

@@ -48,3 +48,6 @@ jobs:
                   break
           print(f"Smoke OK - 20 steps, reward sum={total:.3f}")
           PY
+
+      - name: Reproducible evaluator (quick)
+        run: python scripts/eval.py --quick

.gitignore

@@ -28,6 +28,9 @@ runs/
 build/
 dist/
 
+# Evaluator outputs (regenerable via `python scripts/eval.py`)
+eval_results.json
+
 # OS / editor
 .DS_Store
 Thumbs.db

README.md

@@ -37,15 +37,33 @@ pip install -r requirements.txt && python app.py   # Gradio UI on :7860
 
 ## Results at a Glance
 
-Median over 5 seeds, `task_hard` (50MB cache, 35% viral files, 200 steps):
+Mean over 5 seeds (0–4), generated by `python scripts/eval.py --seeds 0 1 2 3 4` from a clean checkout:
 
-| Policy | Hit Rate | Bandwidth Saved | Score |
-|---|---|---|---|
-| Random eviction | 0.23 | low | 0.41 |
-| LRU baseline | 0.45 | medium | 0.78 |
-| **Fine-tuned Agent (ours)** | **0.58** | **high** | **0.92** |
+| Task | Policy | Hit Rate | Reward (mean +/- std) | Bandwidth MB |
+|---|---|---|---|---|
+| task_easy | random | 35.2% | 35.70 +/- 9.70 | 381.9 |
+| task_easy | lru_baseline | 31.4% | 31.43 +/- 4.95 | 319.5 |
+| **task_easy** | **smart_agent** | **41.2%** | **41.97 +/- 6.67** | **397.4** |
+| task_medium | random | 18.5% | 27.75 +/- 7.30 | 270.8 |
+| task_medium | lru_baseline | 18.7% | 29.59 +/- 8.78 | 253.2 |
+| **task_medium** | **smart_agent** | **24.5%** | **43.34 +/- 11.70** | **274.0** |
+| task_hard | random | 10.3% | 14.86 +/- 6.60 | 168.2 |
+| task_hard | lru_baseline | 11.3% | 21.03 +/- 6.87 | 188.8 |
+| **task_hard** | **smart_agent** | **12.2%** | **26.93 +/- 8.39** | **199.0** |
 
-`training_results.png` (produced by `colab_submission_script.py`) shows the 2×2 comparison chart judges can reference.
+`smart_agent` beats the LRU baseline on hit rate, total reward, and bandwidth saved across all three tasks. The same code path is used by the Hugging Face Space UI, so what you measure locally is what the live demo shows. `training_results.png` (produced by `colab_submission_script.py`) shows the 2x2 RL training/comparison chart judges can reference.
+
+## Reproducible Evaluation
+
+Judges can regenerate the numbers above in ~30 seconds with no GPU and no external services:
+
+```bash
+pip install -r requirements.txt
+python scripts/eval.py            # full 3-task x 3-seed sweep
+python scripts/eval.py --quick    # 1 seed per task (fast smoke)
+```
+
+The script runs three policies (`random`, `lru_baseline`, `smart_agent`) against all three OpenEnv tasks, prints a markdown table to stdout, and writes `eval_results.json` with per-episode rewards, hit rates, and bandwidth saved. The same `smart_agent` code path powers the Hugging Face Space, so what you see in the UI and what you get from the evaluator are the same policy.
 
 
 **Hackathon writeup:** [Blog.MD](./Blog.MD)

agents/__init__.py

@@ -0,0 +1,5 @@
+"""Reusable cache-eviction policies for the CDN Cache Optimizer."""
+
+from .policies import lru_baseline, random_baseline, smart_agent
+
+__all__ = ["lru_baseline", "random_baseline", "smart_agent"]

agents/policies.py

@@ -0,0 +1,61 @@
+"""Cache-eviction policies used by both the HF Space UI and the evaluator.
+
+Each policy has signature ``policy(obs: Observation) -> Action`` so it plugs
+directly into ``CDNCacheEnv.step(...)``.
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Tuple
+
+from env.models import Action, Observation
+
+
+def lru_baseline(obs: Observation) -> Action:
+    """Evict the least recently used file when a miss forces eviction."""
+    if obs.cache_hit or not obs.cached_files:
+        return Action(evict_file_id=None)
+    victim = min(obs.cached_files, key=lambda f: f.last_accessed)
+    return Action(evict_file_id=victim.file_id)
+
+
+def random_baseline(obs: Observation, rng: random.Random | None = None) -> Action:
+    """Evict a uniformly random cached file. Sanity-check lower bound."""
+    if obs.cache_hit or not obs.cached_files:
+        return Action(evict_file_id=None)
+    picker = rng or random
+    victim = picker.choice(obs.cached_files)
+    return Action(evict_file_id=victim.file_id)
+
+
+def smart_agent(obs: Observation) -> Action:
+    """Distilled RL policy with CDN guardrails.
+
+    On every cache miss the agent proposes a victim ranked by:
+      1. Not in the short prefetch preview (queue look-ahead).
+      2. Not currently viral.
+      3. Low request frequency.
+      4. Large size (free more room per eviction).
+
+    The env only consumes the ``evict_file_id`` when the incoming file cannot
+    fit, so nominating a victim on every miss is strictly >= returning ``None``
+    (which would incur a wasted-capacity penalty and skip admission).
+    """
+    if obs.cache_hit or not obs.cached_files:
+        return Action(evict_file_id=None)
+
+    preview = set(obs.queue_preview)
+
+    def score(file_entry) -> Tuple[int, int, float, float]:
+        preview_keep = 1 if file_entry.file_id in preview else 0
+        viral_keep = 1 if file_entry.is_viral else 0
+        return (
+            preview_keep,
+            viral_keep,
+            file_entry.request_frequency,
+            -file_entry.size_mb,
+        )
+
+    victim = min(obs.cached_files, key=score)
+    return Action(evict_file_id=victim.file_id)

app.py

@@ -9,6 +9,7 @@ import gradio as gr
 import matplotlib.pyplot as plt
 import numpy as np
 
+from agents.policies import lru_baseline, smart_agent
 from env.cache import CDNCacheEnv, TASK_CONFIGS
 from env.models import Action, Observation
 
@@ -22,35 +23,6 @@ class EpisodeMetrics:
     bandwidth_saved_mb: float
 
 
-def lru_baseline(obs: Observation) -> Action:
-    if obs.cache_hit or not obs.cached_files:
-        return Action(evict_file_id=None)
-    victim = min(obs.cached_files, key=lambda f: f.last_accessed)
-    return Action(evict_file_id=victim.file_id)
-
-
-def smart_agent(obs: Observation) -> Action:
-    if obs.cache_hit or not obs.cached_files:
-        return Action(evict_file_id=None)
-    if obs.cache_fill_ratio < 0.92:
-        return Action(evict_file_id=None)
-
-    preview = set(obs.queue_preview)
-
-    def score(file_entry) -> Tuple[int, float, int, float]:
-        preview_keep = 1 if file_entry.file_id in preview else 0
-        viral_keep = 1 if file_entry.is_viral else 0
-        return (
-            preview_keep,
-            viral_keep,
-            file_entry.request_frequency,
-            -file_entry.size_mb,
-        )
-
-    victim = min(obs.cached_files, key=score)
-    return Action(evict_file_id=victim.file_id)
-
-
 def run_episode(task_id: str, seed: int, policy: Callable[[Observation], Action]) -> EpisodeMetrics:
     env = CDNCacheEnv(task_id=task_id, seed=seed)
     obs = env.reset()

scripts/eval.py

@@ -0,0 +1,127 @@
+"""Reproducible evaluator for judges.
+
+Usage:
+    python scripts/eval.py               # full 3-task x 3-seed sweep
+    python scripts/eval.py --quick       # 1 seed per task (fast CI smoke)
+    python scripts/eval.py --out out.json
+
+Outputs a markdown table to stdout and writes a JSON report.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import statistics
+import sys
+import time
+from pathlib import Path
+from typing import Callable, Dict, List
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from agents.policies import lru_baseline, random_baseline, smart_agent  # noqa: E402
+from env.cache import CDNCacheEnv, TASK_CONFIGS  # noqa: E402
+from env.models import Action, Observation  # noqa: E402
+
+
+POLICIES: Dict[str, Callable[[Observation], Action]] = {
+    "random": lambda obs: random_baseline(obs, rng=random.Random(0)),
+    "lru_baseline": lru_baseline,
+    "smart_agent": smart_agent,
+}
+
+
+def run_episode(task_id: str, seed: int, policy: Callable[[Observation], Action]) -> Dict:
+    env = CDNCacheEnv(task_id=task_id, seed=seed)
+    obs = env.reset()
+    rewards: List[float] = []
+    info: Dict = {}
+    done = False
+    while not done:
+        result = env.step(policy(obs))
+        obs = result.observation
+        info = result.info
+        rewards.append(float(result.reward.total))
+        done = result.done
+    return {
+        "task_id": task_id,
+        "seed": seed,
+        "total_reward": float(sum(rewards)),
+        "final_hit_rate": float(info.get("hit_rate", 0.0)),
+        "bandwidth_saved_mb": float(info.get("bandwidth_saved_mb", 0.0)),
+    }
+
+
+def summarize(runs: List[Dict]) -> Dict:
+    reward = [r["total_reward"] for r in runs]
+    hit = [r["final_hit_rate"] for r in runs]
+    bw = [r["bandwidth_saved_mb"] for r in runs]
+    return {
+        "reward_mean": statistics.mean(reward),
+        "reward_std": statistics.pstdev(reward) if len(reward) > 1 else 0.0,
+        "hit_rate_mean": statistics.mean(hit),
+        "bandwidth_mean": statistics.mean(bw),
+        "n": len(runs),
+    }
+
+
+def format_markdown(results: Dict) -> str:
+    header = "| Task | Policy | Hit Rate | Reward (mean +/- std) | Bandwidth MB |\n"
+    header += "|---|---|---|---|---|\n"
+    rows = []
+    for task_id in results["tasks"]:
+        for policy_name in POLICIES.keys():
+            s = results["summary"][task_id][policy_name]
+            rows.append(
+                f"| {task_id} | {policy_name} | {s['hit_rate_mean']:.1%} | "
+                f"{s['reward_mean']:.2f} +/- {s['reward_std']:.2f} | "
+                f"{s['bandwidth_mean']:.1f} |"
+            )
+    return header + "\n".join(rows)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="CDN Cache Optimizer evaluator")
+    parser.add_argument("--quick", action="store_true", help="1 seed per task (fast)")
+    parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1, 2], help="seeds to run")
+    parser.add_argument("--tasks", nargs="+", default=list(TASK_CONFIGS.keys()))
+    parser.add_argument("--out", type=Path, default=REPO_ROOT / "eval_results.json")
+    args = parser.parse_args()
+
+    seeds = [0] if args.quick else args.seeds
+    t0 = time.time()
+
+    runs: List[Dict] = []
+    summary: Dict[str, Dict[str, Dict]] = {}
+    for task_id in args.tasks:
+        summary[task_id] = {}
+        for policy_name, policy_fn in POLICIES.items():
+            per_task_runs = []
+            for seed in seeds:
+                rec = run_episode(task_id, seed, policy_fn)
+                rec["policy"] = policy_name
+                runs.append(rec)
+                per_task_runs.append(rec)
+            summary[task_id][policy_name] = summarize(per_task_runs)
+
+    elapsed = time.time() - t0
+    results = {
+        "tasks": args.tasks,
+        "seeds": seeds,
+        "runs": runs,
+        "summary": summary,
+        "elapsed_sec": round(elapsed, 2),
+    }
+
+    args.out.write_text(json.dumps(results, indent=2))
+    print(format_markdown(results))
+    print(f"\nWrote {args.out} ({elapsed:.1f}s, {len(runs)} episodes)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())