Add GVL robot-judge Gradio app and requirements

app.py

@@ -0,0 +1,170 @@
+"""
+Build Small hackathon starter: sub-32B VLM as a training-free judge for robot manipulation.
+
+What it does
+------------
+Given a manipulation episode (a list of frames + a task description), it uses MiniCPM-V to
+estimate per-frame task progress (0-100) using the GVL trick (shuffle frames so the model
+can't just copy temporal order), then:
+  - plots the progress curve,
+  - derives a success verdict from late-stage progress,
+  - reports Value-Order Correlation (VOC) = rank corr between predicted progress and true
+    chronological order. High VOC means the model actually understands task progression.
+
+Method reference: Ma et al., "Vision Language Models are In-Context Value Learners" (GVL),
+arXiv 2411.04549. VOC and the shuffled-frame formulation are from that paper.
+
+Notes
+-----
+- The MiniCPM-V call signature can change between versions. Verify against the current model
+  card at https://huggingface.co/openbmb/MiniCPM-V-4_5 before relying on it.
+- Use MiniCPM-V-4.6 for local dev on 8GB VRAM, MiniCPM-V-4_5 (8B) on ZeroGPU/Modal
+  for the real numbers. The 4.5-vs-4.6 gap is your headline finding.
+- This is a scaffold. The parts that matter (GVL scoring, VOC) are correct; the model I/O is
+  where you should expect to adjust.
+"""
+
+import random
+import numpy as np
+from PIL import Image
+from scipy.stats import spearmanr
+import gradio as gr
+
+# ZeroGPU: the decorator moves work onto a GPU slice on Spaces. Harmless to import locally.
+try:
+    import spaces
+
+    GPU = spaces.GPU
+except Exception:  # local dev without the spaces package
+
+    def GPU(*a, **k):
+        def wrap(fn):
+            return fn
+
+        return wrap if (a and callable(a[0]) is False) else (a[0] if a else None)
+
+
+import torch
+from transformers import AutoModel, AutoTokenizer
+
+MODEL_ID = "openbmb/MiniCPM-V-4_5"  # swap to openbmb/MiniCPM-V-4.6 for the smaller local run
+N_FRAMES = 16
+SUCCESS_THRESHOLD = (
+    70.0  # late-stage progress above this counts as success; tune on your data
+)
+
+_model = None
+_tokenizer = None
+
+
+def _load():
+    global _model, _tokenizer
+    if _model is None:
+        _tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
+        _model = (
+            AutoModel.from_pretrained(
+                MODEL_ID,
+                trust_remote_code=True,
+                torch_dtype=torch.bfloat16,
+                attn_implementation="sdpa",
+            )
+            .eval()
+            .cuda()
+        )
+    return _model, _tokenizer
+
+
+def sample_frames(frames, n=N_FRAMES):
+    """Uniformly sample n frames from a list of PIL images."""
+    if len(frames) <= n:
+        return list(range(len(frames))), frames
+    idx = np.linspace(0, len(frames) - 1, n).astype(int).tolist()
+    return idx, [frames[i] for i in idx]
+
+
+def _ask_progress(model, tokenizer, task, init_frame, query_frame):
+    """One VLM call: given the initial frame as the 0% reference, score one query frame."""
+    prompt = (
+        f"A robot is performing this task: {task}.\n"
+        f"The first image is the initial state (0% complete).\n"
+        f"The second image is a frame from somewhere in the episode (order unknown).\n"
+        f"Estimate how complete the task is in the second image, as a single integer 0-100. "
+        f"Reply with the number only."
+    )
+    msgs = [{"role": "user", "content": [init_frame, query_frame, prompt]}]
+    out = model.chat(
+        image=None, msgs=msgs, tokenizer=tokenizer
+    )  # verify signature per model card
+    digits = "".join(c for c in str(out) if c.isdigit())
+    if not digits:
+        return 0.0
+    return float(max(0, min(100, int(digits[:3]))))
+
+
+@GPU(duration=120)
+def score_episode(frames, task):
+    """GVL-style scoring. Returns (chronological progress list, VOC)."""
+    model, tokenizer = _load()
+    _, sampled = sample_frames(frames)
+    init = sampled[0]
+    rest = list(enumerate(sampled))[1:]  # (true_chrono_index, frame)
+
+    shuffled = rest[:]
+    random.shuffle(shuffled)  # the GVL trick: break temporal adjacency
+
+    scored = {}
+    for true_idx, frame in shuffled:
+        scored[true_idx] = _ask_progress(model, tokenizer, task, init, frame)
+
+    progress = [0.0] + [scored[i] for i, _ in rest]  # restore chronological order
+    true_order = list(range(len(progress)))
+    voc, _ = spearmanr(progress, true_order)
+    return progress, float(voc)
+
+
+def run(video_frames_dir_or_list, task):
+    """Gradio entry. Expects a list of PIL frames; adapt to your loader (lerobot / decord)."""
+    frames = video_frames_dir_or_list  # TODO: plug your LeRobot/LIBERO/OXE episode loader here
+    if not frames:
+        return None, "No frames loaded.", None
+    progress, voc = score_episode(frames, task)
+    late = np.mean(progress[-3:]) if len(progress) >= 3 else progress[-1]
+    verdict = "SUCCESS" if late >= SUCCESS_THRESHOLD else "FAILURE"
+
+    import matplotlib.pyplot as plt
+
+    fig, ax = plt.subplots(figsize=(6, 3))
+    ax.plot(range(len(progress)), progress, marker="o")
+    ax.set_xlabel("frame (chronological)")
+    ax.set_ylabel("predicted progress (%)")
+    ax.set_ylim(-5, 105)
+    ax.set_title(f"VOC = {voc:.2f}")
+    fig.tight_layout()
+
+    summary = f"Verdict: {verdict}\nLate-stage progress: {late:.0f}%\nVOC: {voc:.2f}"
+    return fig, summary, voc
+
+
+with gr.Blocks(title="Can a small VLM judge robot success?") as demo:
+    gr.Markdown(
+        "## Small VLM as a training-free robot judge\n"
+        "Upload or pick a manipulation episode. The model estimates task progress per frame "
+        "(GVL shuffled-frame method) and reports VOC and a success verdict."
+    )
+    with gr.Row():
+        with gr.Column():
+            task_in = gr.Textbox(
+                label="Task description", value="put the red block in the bowl"
+            )
+            frames_in = gr.State(
+                []
+            )  # replace with a gr.File / example picker wired to your loader
+            run_btn = gr.Button("Judge episode", variant="primary")
+        with gr.Column():
+            plot_out = gr.Plot(label="Progress curve")
+            summary_out = gr.Textbox(label="Result")
+            voc_out = gr.Number(label="VOC")
+    run_btn.click(run, [frames_in, task_in], [plot_out, summary_out, voc_out])
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,14 @@
+# ZeroGPU images already include CUDA torch/torchvision, so do not pin them here.
+# Match the transformers pin (and any extras like timm) to the MiniCPM-V model card
+# at https://huggingface.co/openbmb/MiniCPM-V-4_5 to avoid trust_remote_code load errors.
+
+gradio
+spaces
+transformers
+accelerate
+sentencepiece
+pillow
+scipy
+numpy
+matplotlib
+decord