Upload quick_validate.py

quick_validate.py

@@ -0,0 +1,309 @@
+"""
+Quick single-instance cascade validation.
+
+Runs the cascade agent + verification on ONE instance.
+This is the minimal proof that the cascade works end-to-end.
+
+Usage:
+    python quick_validate.py
+    python quick_validate.py --instance django__django-11815
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+import traceback
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple
+
+from huggingface_hub import InferenceClient
+
+
+# ============================================================
+# Pick the easiest instance first
+# ============================================================
+DEFAULT_INSTANCE = "django__django-14315"  # django bug with clean fix
+
+# For T1/T2 models (free HF inference)
+T1_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
+T2_MODEL = "meta-llama/Llama-3.3-70B-Instruct"
+
+
+def run(cmd, cwd=None, timeout=120):
+    result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout, shell=True)
+    return result.returncode, result.stdout, result.stderr
+
+
+def call_model(client, messages, max_tokens=4096):
+    """Call HF inference, return (text, input_tokens, output_tokens)."""
+    try:
+        completion = client.chat.completions.create(
+            model=client.model,
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=0.2,
+        )
+        text = completion.choices[0].message.content
+        itok = completion.usage.prompt_tokens if hasattr(completion, 'usage') and completion.usage else 0
+        otok = completion.usage.completion_tokens if hasattr(completion, 'usage') and completion.usage else 0
+        return text, itok, otok
+    except Exception as e:
+        return f"[ERROR: {e}]", 0, 0
+
+
+def extract_patch(text):
+    """Extract a diff/patch from model output."""
+    for tag in ['patch', 'diff']:
+        m = re.search(rf'<{tag}>(.*?)</{tag}>', text, re.DOTALL)
+        if m:
+            return m.group(1).strip()
+    for block in ['diff', 'patch']:
+        m = re.search(rf'```{block}\s*\n(.*?)```', text, re.DOTALL)
+        if m:
+            return m.group(1).strip()
+    diff_match = re.search(r'(diff --git a/.*?(?:\n(?:@@|\+\+\+|diff --git|```|</).*)*)', text, re.DOTALL)
+    if diff_match:
+        return diff_match.group(1).strip()
+    return None
+
+
+def run_cascade(instance, repo_dir):
+    """Run T1 then T2. Returns {patch, tier, turns, tokens}."""
+    
+    problem = instance.get("problem_statement", "")
+    
+    system = f"""You are fixing a bug in {instance['repo']}. Repository is at {repo_dir}.
+
+Output format:
+- Bash commands: <bash>command</bash>
+- Final patch: <patch>your diff here</patch>
+- Done: <submit>Done</submit>
+
+First explore the codebase to understand the issue, then make a minimal fix and verify it."""
+
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": f"PROBLEM:\n{problem}\n\nStart by exploring the repository."}
+    ]
+    
+    tiers = [
+        ("T1", T1_MODEL, 30),
+        ("T2", T2_MODEL, 30),
+    ]
+    
+    for tier_name, model_id, max_turns in tiers:
+        print(f"\n[{tier_name}] Running {model_id}...")
+        client = InferenceClient(model_id)
+        tier_turns = 0
+        tier_itok = 0
+        tier_otok = 0
+        
+        for turn in range(max_turns):
+            text, itok, otok = call_model(client, messages, max_tokens=4096)
+            tier_turns += 1
+            tier_itok += itok
+            tier_otok += otok
+            messages.append({"role": "assistant", "content": text})
+            
+            # Extract patch
+            patch = extract_patch(text)
+            if patch:
+                print(f"  ✅ Patch found ({len(patch)} chars) at turn {turn+1}")
+                return {"patch": patch, "tier": tier_name, "turns": tier_turns, "input_tokens": tier_itok, "output_tokens": tier_otok}
+            
+            # Execute bash commands
+            cmds = re.findall(r'<bash>(.*?)</bash>', text, re.DOTALL)
+            for cmd in cmds:
+                cmd = cmd.strip()
+                print(f"  $ {cmd[:80]}...")
+                rc, stdout, stderr = run(cmd, cwd=str(repo_dir), timeout=30)
+                output = (stdout + stderr)[:1500]
+                if rc != 0:
+                    output += f" [EXIT:{rc}]"
+                messages.append({"role": "user", "content": f"<output>\n{output}\n</output>"})
+            
+            if "<submit>" in text:
+                break
+    
+    return {"patch": None, "tier": None, "turns": 0, "input_tokens": 0, "output_tokens": 0}
+
+
+def verify_patch(instance, model_patch, repo_dir, env_name=None):
+    """Apply patch + test_patch, run FAIL_TO_PASS tests."""
+    base_commit = instance.get("base_commit", "")
+    test_patch = instance.get("test_patch", "")
+    f2p = instance.get("FAIL_TO_PASS", [])
+    
+    if not base_commit or not test_patch or not f2p:
+        return {"resolved": False, "error": "missing base_commit/test_patch/FAIL_TO_PASS"}
+    
+    # Reset
+    run(f"git checkout -f {base_commit}", cwd=str(repo_dir))
+    
+    # Apply model patch
+    patch_file = repo_dir / "_aco.patch"
+    patch_file.write_text(model_patch)
+    rc, out, err = run(f"git apply --check {patch_file}", cwd=str(repo_dir))
+    if rc != 0:
+        return {"resolved": False, "error": f"patch --check: {err[:200]}"}
+    rc, out, err = run(f"git apply {patch_file}", cwd=str(repo_dir))
+    if rc != 0:
+        rc, out, err = run(f"git apply --reject {patch_file}", cwd=str(repo_dir))
+        if rc != 0:
+            return {"resolved": False, "error": f"patch apply: {err[:200]}"}
+    
+    # Apply test patch
+    test_file = repo_dir / "_aco_test.patch"
+    test_file.write_text(test_patch)
+    rc, out, err = run(f"git apply --check {test_file}", cwd=str(repo_dir))
+    if rc == 0:
+        run(f"git apply {test_file}", cwd=str(repo_dir))
+    
+    # Run FAIL_TO_PASS
+    cmd_prefix = f"conda run -n {env_name} " if env_name else ""
+    cmd = f"{cmd_prefix}python -m pytest -v --tb=short -x {' '.join(f2p)}"
+    print(f"  Running: pytest {' '.join(f2p[:2])}...")
+    rc, out, err = run(cmd, cwd=str(repo_dir), timeout=300)
+    
+    if rc == 0:
+        # Check regressions
+        p2p = instance.get("PASS_TO_PASS", [])
+        if p2p:
+            cmd2 = f"{cmd_prefix}python -m pytest -v --tb=short -x {' '.join(p2p[:15])}"
+            rc2, out2, err2 = run(cmd2, cwd=str(repo_dir), timeout=300)
+            if rc2 == 0:
+                return {"resolved": True, "regressions": False}
+            else:
+                return {"resolved": False, "error": f"regression: {(out2+err2)[:200]}", "regressions": True}
+        return {"resolved": True, "regressions": False}
+    
+    # Count failures
+    failures = [l.strip() for l in (out+err).split('\n') if 'FAILED' in l]
+    return {"resolved": False, "error": f"{len(failures)} F2P failures", "failures": failures[:5]}
+
+
+def main():
+    from datasets import load_dataset
+    
+    instance_id = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_INSTANCE
+    print(f"Validating: {instance_id}")
+    
+    ds = load_dataset("princeton-nlp/SWE-bench_Verified", split="test")
+    instance = None
+    for row in ds:
+        if row["instance_id"] == instance_id:
+            instance = dict(row)
+            break
+    
+    if not instance:
+        print(f"Instance {instance_id} not found!")
+        sys.exit(1)
+    
+    print(f"  Repo: {instance['repo']}")
+    print(f"  Base: {instance['base_commit'][:12]}")
+    print(f"  F2P: {len(instance.get('FAIL_TO_PASS', []))} tests")
+    
+    with tempfile.TemporaryDirectory(prefix=f"aco_quick_") as tmpdir:
+        repo_dir = Path(tmpdir) / "repo"
+        env_name = f"aco_q_{instance_id.replace('__','_').replace('-','_')[:30]}"
+        
+        # Clone
+        repo = instance["repo"]
+        url = f"https://github.com/{repo}.git"
+        print(f"\n[CLONE] {url}")
+        rc, out, err = run(f"git clone --depth 50 {url} {repo_dir}", timeout=180)
+        if rc != 0:
+            rc, out, err = run(f"git clone {url} {repo_dir}", timeout=600)
+        if rc != 0:
+            print(f"CLONE FAILED: {err[:300]}")
+            sys.exit(1)
+        
+        # Set up conda env
+        env_commit = instance.get("environment_setup_commit", "")
+        if env_commit:
+            run(f"cd {repo_dir} && git fetch origin {env_commit}", timeout=60)
+            run(f"cd {repo_dir} && git checkout {env_commit}", timeout=30)
+        
+        env_yml = None
+        for c in ["environment.yml", "dev/environment.yml", ".github/environment.yml"]:
+            if (repo_dir / c).exists():
+                env_yml = c
+                break
+        
+        print(f"\n[ENV] Creating conda env '{env_name}'...")
+        if env_yml:
+            rc, out, err = run(f"cd {repo_dir} && conda env create -f {env_yml} -n {env_name} --quiet", timeout=600)
+        else:
+            rc, out, err = run(f"conda create -n {env_name} python=3.10 pip -y --quiet", timeout=300)
+        
+        if rc != 0:
+            print(f"ENV SETUP FAILED: {err[:300]}")
+            sys.exit(1)
+        
+        # Install repo at base_commit
+        base_commit = instance["base_commit"]
+        run(f"cd {repo_dir} && git fetch origin {base_commit}", timeout=60)
+        run(f"cd {repo_dir} && git checkout {base_commit}", timeout=30)
+        rc, out, err = run(f"cd {repo_dir} && conda run -n {env_name} pip install -e . --quiet", timeout=300)
+        if rc != 0:
+            print(f"PIP INSTALL FAILED (continuing): {err[:200]}")
+        
+        print(f"\n[CASCADE] Running agent...")
+        t0 = time.time()
+        agent_result = run_cascade(instance, repo_dir)
+        agent_time = time.time() - t0
+        
+        print(f"  Patch: {'FOUND' if agent_result['patch'] else 'NOT FOUND'}")
+        print(f"  Tier: {agent_result['tier']}")
+        print(f"  Time: {agent_time:.1f}s")
+        
+        if not agent_result["patch"]:
+            print("FAILED: No patch produced")
+            sys.exit(1)
+        
+        print(f"\n[VERIFY] Testing patch...")
+        verify_result = verify_patch(instance, agent_result["patch"], repo_dir, env_name)
+        
+        print(f"\n{'='*60}")
+        print(f"RESULT: {'✅ RESOLVED' if verify_result['resolved'] else '❌ NOT RESOLVED'}")
+        print(f"{'='*60}")
+        print(f"  Instance: {instance_id}")
+        print(f"  Tier: {agent_result['tier']}")
+        print(f"  Turns: {agent_result['turns']}")
+        print(f"  Tokens: {agent_result['input_tokens']} in / {agent_result['output_tokens']} out")
+        print(f"  Agent time: {agent_time:.1f}s")
+        if not verify_result["resolved"]:
+            print(f"  Error: {verify_result.get('error', 'unknown')}")
+        
+        # Save result
+        final = {
+            "instance_id": instance_id,
+            "repo": instance["repo"],
+            "timestamp": datetime.now().isoformat(),
+            "resolved": verify_result["resolved"],
+            "tier": agent_result["tier"],
+            "turns": agent_result["turns"],
+            "input_tokens": agent_result["input_tokens"],
+            "output_tokens": agent_result["output_tokens"],
+            "agent_time_seconds": agent_time,
+            "error": verify_result.get("error"),
+        }
+        
+        result_path = f"quick_validate_{instance_id}.json"
+        with open(result_path, "w") as f:
+            json.dump(final, f, indent=2)
+        print(f"\n  Saved: {result_path}")
+        
+        # Cleanup
+        run(f"conda env remove -n {env_name} -y --quiet", timeout=30)
+    
+    return 0 if verify_result["resolved"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())