Phase A: Add v2.0 3-level test harness (L1 raw, L2 solo, L3 integrated + master matrix)

tests/v2/__init__.py

@@ -0,0 +1 @@
+# BAYAN v2.0 Test Suite

tests/v2/benchmark_matrix.py

@@ -0,0 +1,210 @@
+"""
+BAYAN v2.0 — Benchmark Matrix Runner
+======================================
+Master script that runs all 3 test levels and produces a side-by-side
+comparison matrix showing raw model vs solo API vs integrated pipeline.
+
+Usage:
+    # Full matrix (all 3 levels)
+    python tests/v2/benchmark_matrix.py --url URL
+
+    # Single level only
+    python tests/v2/benchmark_matrix.py --url URL --level 1
+    python tests/v2/benchmark_matrix.py --url URL --level 3
+
+    # Single dataset
+    python tests/v2/benchmark_matrix.py --url URL --dataset spelling
+
+    # Compare saved results
+    python tests/v2/benchmark_matrix.py --compare
+"""
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+REPORT_DIR = Path(__file__).parent / "reports"
+
+
+def run_level(level: int, url: str, dataset: str = None):
+    """Run a specific test level."""
+    import subprocess
+    scripts = {
+        1: "tests/v2/test_level1_raw.py",
+        2: "tests/v2/test_level2_solo.py",
+        3: "tests/v2/test_level3_integrated.py",
+    }
+    script = scripts[level]
+    cmd = [sys.executable, script, "--url", url]
+    if dataset:
+        cmd.extend(["--dataset", dataset])
+
+    print(f"\n{'#'*70}")
+    print(f"# RUNNING LEVEL {level}: {script}")
+    print(f"{'#'*70}\n")
+
+    result = subprocess.run(cmd, cwd=str(Path(__file__).parent.parent.parent))
+    return result.returncode == 0
+
+
+def load_report(level: int) -> dict:
+    """Load saved report for a level."""
+    filenames = {
+        1: "level1_raw_results.json",
+        2: "level2_solo_results.json",
+        3: "level3_integrated_results.json",
+    }
+    path = REPORT_DIR / filenames[level]
+    if path.exists():
+        with open(path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    return None
+
+
+def print_comparison_matrix():
+    """Load all 3 reports and print side-by-side comparison."""
+    l1 = load_report(1)
+    l2 = load_report(2)
+    l3 = load_report(3)
+
+    print(f"\n{'='*70}")
+    print("BAYAN v2.0 — BENCHMARK COMPARISON MATRIX")
+    print(f"{'='*70}")
+
+    # Timestamps
+    for level, report, name in [(1, l1, "L1-Raw"), (2, l2, "L2-Solo"), (3, l3, "L3-Pipeline")]:
+        if report:
+            ts = report.get("timestamp", "N/A")
+            print(f"  {name}: {ts}")
+        else:
+            print(f"  {name}: NOT RUN")
+
+    # ── Level 1 Summary ──
+    if l1:
+        print(f"\n{'─'*70}")
+        print("Level 1: RAW MODEL OUTPUT (no filters)")
+        print(f"{'─'*70}")
+        a = l1.get("analysis", {})
+        by_model = a.get("by_model", {})
+        for model, data in by_model.items():
+            changed = data.get("changed", 0)
+            unchanged = data.get("unchanged", 0)
+            total = changed + unchanged + data.get("errors", 0)
+            rate = changed / total * 100 if total else 0
+            print(f"  {model:<12}: {changed}/{total} texts modified ({rate:.1f}%)")
+
+    # ── Level 2 Summary ──
+    if l2:
+        print(f"\n{'─'*70}")
+        print("Level 2: SOLO API (single model + filters, no integration)")
+        print(f"{'─'*70}")
+        a = l2.get("analysis", {})
+        by_model = a.get("by_model", {})
+        print(f"  {'Model':<12} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4} {'Pass%':>7}")
+        print(f"  {'-'*12} {'-'*4} {'-'*4} {'-'*4} {'-'*4} {'-'*7}")
+        for model, data in by_model.items():
+            pr = data.get("pass_rate", 0) * 100
+            print(f"  {model:<12} {data.get('TP',0):>4} {data.get('TN',0):>4} "
+                  f"{data.get('FP',0):>4} {data.get('FN',0):>4} {pr:>6.1f}%")
+
+    # ── Level 3 Summary ──
+    if l3:
+        print(f"\n{'─'*70}")
+        print("Level 3: INTEGRATED PIPELINE (full Spelling→Grammar→Punctuation)")
+        print(f"{'─'*70}")
+        a = l3.get("analysis", {})
+        agg = a.get("aggregate", {})
+        total = a.get("total", 0)
+        pr = agg.get("pass_rate", 0) * 100
+        print(f"  Overall: {pr:.1f}% pass ({total} tests)")
+        print(f"  TP={agg.get('TP',0)} TN={agg.get('TN',0)} "
+              f"FP={agg.get('FP',0)} FN={agg.get('FN',0)}")
+
+        print(f"\n  {'Dataset':<14} {'Pass%':>7} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4}")
+        print(f"  {'-'*14} {'-'*7} {'-'*4} {'-'*4} {'-'*4} {'-'*4}")
+        by_ds = a.get("by_dataset", {})
+        for ds in sorted(by_ds.keys()):
+            d = by_ds[ds]
+            dp = d.get("pass_rate", 0) * 100
+            print(f"  {ds:<14} {dp:>6.1f}% {d.get('TP',0):>4} {d.get('TN',0):>4} "
+                  f"{d.get('FP',0):>4} {d.get('FN',0):>4}")
+
+    # ── Cross-Level Comparison ──
+    if l2 and l3:
+        print(f"\n{'─'*70}")
+        print("CROSS-LEVEL COMPARISON: Solo vs Integrated")
+        print(f"{'─'*70}")
+
+        l2_ds = l2.get("analysis", {}).get("by_dataset", {})
+        l3_ds = l3.get("analysis", {}).get("by_dataset", {})
+
+        all_datasets = sorted(set(list(l2_ds.keys()) + list(l3_ds.keys())))
+
+        print(f"  {'Dataset':<14} {'L2-Spell':>10} {'L2-Gram':>10} {'L2-Punct':>10} {'L3-Pipeline':>12} {'Delta':>7}")
+        print(f"  {'-'*14} {'-'*10} {'-'*10} {'-'*10} {'-'*12} {'-'*7}")
+
+        for ds in all_datasets:
+            l2d = l2_ds.get(ds, {})
+            l3d = l3_ds.get(ds, {})
+
+            l2_s = l2d.get("spelling", {}).get("pass_rate", 0) * 100
+            l2_g = l2d.get("grammar", {}).get("pass_rate", 0) * 100
+            l2_p = l2d.get("punctuation", {}).get("pass_rate", 0) * 100
+            l3_p = l3d.get("pass_rate", 0) * 100
+
+            # Best solo model vs pipeline
+            best_solo = max(l2_s, l2_g, l2_p)
+            delta = l3_p - best_solo
+
+            delta_str = f"{delta:+.1f}%"
+            print(f"  {ds:<14} {l2_s:>9.1f}% {l2_g:>9.1f}% {l2_p:>9.1f}% {l3_p:>11.1f}% {delta_str:>7}")
+
+    # Save comparison
+    comparison = {
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "l1_timestamp": l1.get("timestamp") if l1 else None,
+        "l2_timestamp": l2.get("timestamp") if l2 else None,
+        "l3_timestamp": l3.get("timestamp") if l3 else None,
+        "l2_summary": l2.get("analysis", {}).get("by_model") if l2 else None,
+        "l3_summary": l3.get("analysis", {}).get("aggregate") if l3 else None,
+    }
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+    with open(REPORT_DIR / "benchmark_matrix.json", 'w', encoding='utf-8') as f:
+        json.dump(comparison, f, ensure_ascii=False, indent=2)
+    print(f"\n[MATRIX] Comparison → {REPORT_DIR / 'benchmark_matrix.json'}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="BAYAN v2.0 Benchmark Matrix")
+    parser.add_argument("--url", default="https://bayan10-bayan-api.hf.space")
+    parser.add_argument("--level", type=int, default=None, help="Run specific level (1/2/3)")
+    parser.add_argument("--dataset", default=None, help="Filter to single dataset")
+    parser.add_argument("--compare", action="store_true", help="Compare saved results only")
+    args = parser.parse_args()
+
+    if args.compare:
+        print_comparison_matrix()
+        return
+
+    levels = [args.level] if args.level else [1, 2, 3]
+
+    print(f"\n{'#'*70}")
+    print(f"# BAYAN v2.0 — BENCHMARK MATRIX")
+    print(f"# Levels: {levels}")
+    print(f"# Target: {args.url}")
+    print(f"{'#'*70}")
+
+    for level in levels:
+        success = run_level(level, args.url, args.dataset)
+        if not success:
+            print(f"\n❌ Level {level} failed!")
+            sys.exit(1)
+        print(f"\n✅ Level {level} complete!")
+
+    # Print comparison
+    print_comparison_matrix()
+
+
+if __name__ == "__main__":
+    main()

tests/v2/test_level1_raw.py

@@ -0,0 +1,278 @@
+"""
+BAYAN v2.0 — Level 1: Raw Model Tests
+======================================
+Tests each ML model DIRECTLY — no filters, no pipeline, no StageLocker.
+Measures the raw model ceiling: what's the best each model can do?
+
+Usage:
+    python tests/v2/test_level1_raw.py --url URL [--dataset DATASET]
+"""
+import argparse
+import json
+import re
+import time
+import sys
+from pathlib import Path
+from dataclasses import dataclass, field, asdict
+from typing import List, Dict, Optional
+import requests
+
+# Datasets
+DATASETS_DIR = Path(__file__).parent.parent / "phase10" / "gold_datasets"
+
+REPORT_DIR = Path(__file__).parent / "reports"
+
+
+@dataclass
+class RawModelResult:
+    id: str
+    dataset: str
+    category: str
+    input_text: str
+    expected: str          # What the benchmark expects
+    severity: str
+    # Raw model outputs
+    spelling_raw: str = ""
+    spelling_ms: int = 0
+    grammar_raw: str = ""
+    grammar_ms: int = 0
+    punctuation_raw: str = ""
+    punctuation_ms: int = 0
+    # Analysis
+    spelling_changed: bool = False
+    grammar_changed: bool = False
+    punctuation_changed: bool = False
+
+
+class APIClient:
+    """Minimal client to call individual model endpoints."""
+    def __init__(self, base_url):
+        self.base = base_url.rstrip('/')
+        self.session = requests.Session()
+        self.session.headers['Content-Type'] = 'application/json'
+
+    def _post(self, endpoint, text, timeout=120):
+        t0 = time.time()
+        try:
+            r = self.session.post(
+                f"{self.base}{endpoint}",
+                json={"text": text},
+                timeout=timeout
+            )
+            ms = int((time.time() - t0) * 1000)
+            data = r.json()
+            return data, ms
+        except Exception as e:
+            ms = int((time.time() - t0) * 1000)
+            return {"error": str(e)}, ms
+
+    def spelling_raw(self, text):
+        """Call /api/spelling — raw spelling model through API."""
+        data, ms = self._post("/api/spelling", text)
+        corrected = data.get("corrected_text", data.get("corrected", text))
+        return corrected, ms
+
+    def grammar_raw(self, text):
+        """Call /api/grammar — raw grammar model through API."""
+        data, ms = self._post("/api/grammar", text)
+        corrected = data.get("corrected_text", data.get("corrected", text))
+        return corrected, ms
+
+    def punctuation_raw(self, text):
+        """Call /api/punctuation — raw punctuation model through API."""
+        data, ms = self._post("/api/punctuation", text)
+        corrected = data.get("corrected_text", data.get("corrected", text))
+        return corrected, ms
+
+
+def load_datasets(dataset_filter=None):
+    """Load all gold datasets."""
+    datasets = {}
+    for f in sorted(DATASETS_DIR.glob("*.json")):
+        name = f.stem
+        if dataset_filter and name != dataset_filter:
+            continue
+        with open(f, 'r', encoding='utf-8') as fh:
+            data = json.load(fh)
+            datasets[name] = data
+    return datasets
+
+
+def strip_diacritics(text):
+    return re.sub(r'[\u064B-\u065F\u0670]', '', text)
+
+
+def normalize(text):
+    t = strip_diacritics(text)
+    t = re.sub(r'\s+', ' ', t).strip()
+    return t
+
+
+def run_level1(api: APIClient, datasets: dict) -> List[RawModelResult]:
+    """Run each test case through all 3 raw models."""
+    results = []
+    total = sum(len(v) for v in datasets.values())
+    idx = 0
+
+    for ds_name, cases in datasets.items():
+        print(f"\n{'='*60}")
+        print(f"DATASET: {ds_name.upper()} ({len(cases)} samples)")
+        print(f"{'='*60}")
+
+        for case in cases:
+            idx += 1
+            cid = case.get('id', f'{ds_name}_{idx}')
+            cat = case.get('category', '')
+            inp = case.get('input', '')
+            expected = case.get('expected', case.get('input', ''))
+            severity = case.get('severity', '')
+
+            r = RawModelResult(
+                id=cid, dataset=ds_name, category=cat,
+                input_text=inp, expected=expected, severity=severity
+            )
+
+            print(f"  [{idx}/{total}] {cid} ({cat})...", end=" ", flush=True)
+
+            # ── Spelling ──
+            try:
+                r.spelling_raw, r.spelling_ms = api.spelling_raw(inp)
+                r.spelling_changed = (normalize(r.spelling_raw) != normalize(inp))
+            except Exception as e:
+                r.spelling_raw = f"ERROR: {e}"
+
+            # ── Grammar ──
+            try:
+                r.grammar_raw, r.grammar_ms = api.grammar_raw(inp)
+                r.grammar_changed = (normalize(r.grammar_raw) != normalize(inp))
+            except Exception as e:
+                r.grammar_raw = f"ERROR: {e}"
+
+            # ── Punctuation ──
+            try:
+                r.punctuation_raw, r.punctuation_ms = api.punctuation_raw(inp)
+                r.punctuation_changed = (normalize(r.punctuation_raw) != normalize(inp))
+            except Exception as e:
+                r.punctuation_raw = f"ERROR: {e}"
+
+            total_ms = r.spelling_ms + r.grammar_ms + r.punctuation_ms
+            changes = []
+            if r.spelling_changed: changes.append("S")
+            if r.grammar_changed: changes.append("G")
+            if r.punctuation_changed: changes.append("P")
+            change_str = "+".join(changes) if changes else "none"
+            print(f"[{change_str}] ({total_ms}ms)")
+
+            results.append(r)
+
+    return results
+
+
+def analyze_results(results: List[RawModelResult]) -> dict:
+    """Produce per-model analysis of raw outputs."""
+    analysis = {
+        "total": len(results),
+        "by_model": {
+            "spelling": {"changed": 0, "unchanged": 0, "errors": 0},
+            "grammar": {"changed": 0, "unchanged": 0, "errors": 0},
+            "punctuation": {"changed": 0, "unchanged": 0, "errors": 0},
+        },
+        "by_dataset": {},
+    }
+
+    for r in results:
+        ds = r.dataset
+        if ds not in analysis["by_dataset"]:
+            analysis["by_dataset"][ds] = {
+                "total": 0,
+                "spelling_changed": 0,
+                "grammar_changed": 0,
+                "punctuation_changed": 0,
+            }
+        analysis["by_dataset"][ds]["total"] += 1
+
+        # Spelling
+        if "ERROR" in r.spelling_raw:
+            analysis["by_model"]["spelling"]["errors"] += 1
+        elif r.spelling_changed:
+            analysis["by_model"]["spelling"]["changed"] += 1
+            analysis["by_dataset"][ds]["spelling_changed"] += 1
+        else:
+            analysis["by_model"]["spelling"]["unchanged"] += 1
+
+        # Grammar
+        if "ERROR" in r.grammar_raw:
+            analysis["by_model"]["grammar"]["errors"] += 1
+        elif r.grammar_changed:
+            analysis["by_model"]["grammar"]["changed"] += 1
+            analysis["by_dataset"][ds]["grammar_changed"] += 1
+        else:
+            analysis["by_model"]["grammar"]["unchanged"] += 1
+
+        # Punctuation
+        if "ERROR" in r.punctuation_raw:
+            analysis["by_model"]["punctuation"]["errors"] += 1
+        elif r.punctuation_changed:
+            analysis["by_model"]["punctuation"]["changed"] += 1
+            analysis["by_dataset"][ds]["punctuation_changed"] += 1
+        else:
+            analysis["by_model"]["punctuation"]["unchanged"] += 1
+
+    return analysis
+
+
+def print_analysis(analysis: dict):
+    print(f"\n{'='*60}")
+    print("LEVEL 1: RAW MODEL ANALYSIS")
+    print(f"{'='*60}")
+
+    print(f"\n## Per-Model Summary ({analysis['total']} tests)")
+    print(f"| Model       | Changed | Unchanged | Errors |")
+    print(f"|-------------|---------|-----------|--------|")
+    for model, data in analysis["by_model"].items():
+        print(f"| {model:<11} | {data['changed']:>7} | {data['unchanged']:>9} | {data['errors']:>6} |")
+
+    print(f"\n## Change Rate by Dataset")
+    print(f"| Dataset      | Total | S-Changed | G-Changed | P-Changed |")
+    print(f"|--------------|-------|-----------|-----------|-----------|")
+    for ds, data in sorted(analysis["by_dataset"].items()):
+        print(f"| {ds:<12} | {data['total']:>5} | {data['spelling_changed']:>9} | {data['grammar_changed']:>9} | {data['punctuation_changed']:>9} |")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Level 1: Raw Model Tests")
+    parser.add_argument("--url", default="https://bayan10-bayan-api.hf.space")
+    parser.add_argument("--dataset", default=None, help="Filter to single dataset")
+    args = parser.parse_args()
+
+    api = APIClient(args.url)
+    datasets = load_datasets(args.dataset)
+
+    print(f"\n{'='*60}")
+    print("BAYAN v2.0 — Level 1: Raw Model Tests")
+    print(f"{'='*60}")
+    print(f"  Target:   {args.url}")
+    print(f"  Datasets: {list(datasets.keys())}")
+    print(f"  Total:    {sum(len(v) for v in datasets.values())} tests")
+    print(f"{'='*60}")
+
+    results = run_level1(api, datasets)
+    analysis = analyze_results(results)
+    print_analysis(analysis)
+
+    # Save results
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = REPORT_DIR / "level1_raw_results.json"
+    report = {
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "target": args.url,
+        "analysis": analysis,
+        "results": [asdict(r) for r in results],
+    }
+    with open(out_path, 'w', encoding='utf-8') as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+    print(f"\n[L1] Results → {out_path}")
+
+
+if __name__ == "__main__":
+    main()

tests/v2/test_level2_solo.py

@@ -0,0 +1,289 @@
+"""
+BAYAN v2.0 — Level 2: Solo API Tests
+=====================================
+Tests each model through its INDIVIDUAL API endpoint (/api/spelling, /api/grammar,
+/api/punctuation). This measures what each stage produces in isolation — with
+any endpoint-level preprocessing but WITHOUT pipeline integration (StageLocker,
+OffsetMapper, cross-stage interaction).
+
+Compares with Level 1 raw results to measure filter impact:
+- If L2 passes more tests than L1 → filters are helping
+- If L2 passes fewer tests than L1 → filters are over-filtering
+
+Usage:
+    python tests/v2/test_level2_solo.py --url URL [--dataset DATASET]
+"""
+import argparse
+import json
+import re
+import time
+from pathlib import Path
+from dataclasses import dataclass, asdict
+from typing import List
+import requests
+
+DATASETS_DIR = Path(__file__).parent.parent / "phase10" / "gold_datasets"
+REPORT_DIR = Path(__file__).parent / "reports"
+
+
+def normalize(text):
+    t = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+    t = re.sub(r'\s+', ' ', t).strip()
+    return t
+
+
+@dataclass
+class SoloResult:
+    id: str
+    dataset: str
+    category: str
+    input_text: str
+    expected: str
+    severity: str
+    # Solo API outputs (each model called independently on the SAME input)
+    spelling_solo: str = ""
+    spelling_ms: int = 0
+    grammar_solo: str = ""
+    grammar_ms: int = 0
+    punctuation_solo: str = ""
+    punctuation_ms: int = 0
+    # Verdict per model
+    spelling_verdict: str = ""  # TP, TN, FP, FN
+    grammar_verdict: str = ""
+    punctuation_verdict: str = ""
+
+
+class APIClient:
+    def __init__(self, base_url):
+        self.base = base_url.rstrip('/')
+        self.session = requests.Session()
+        self.session.headers['Content-Type'] = 'application/json'
+
+    def call(self, endpoint, text, timeout=120):
+        t0 = time.time()
+        try:
+            r = self.session.post(
+                f"{self.base}{endpoint}",
+                json={"text": text},
+                timeout=timeout
+            )
+            ms = int((time.time() - t0) * 1000)
+            data = r.json()
+            corrected = data.get("corrected_text", data.get("corrected", text))
+            return corrected, ms
+        except Exception as e:
+            ms = int((time.time() - t0) * 1000)
+            return f"ERROR: {e}", ms
+
+
+def classify_result(input_text, output_text, expected_text, dataset):
+    """Classify a single model output as TP/TN/FP/FN.
+    
+    For datasets that test CORRECTION (spelling, grammar, punctuation):
+      - TP: model corrected AND the correction is in the expected direction
+      - FN: model did NOT correct (output == input) but should have
+      - FP: model corrected but incorrectly (changed text that was correct or wrong direction)
+      - TN: model correctly left unchanged (output == input AND input was correct)
+    
+    For datasets that test PRESERVATION (entities, religious, structured, hallucination):
+      - TN: model left text unchanged → PASS
+      - FP: model modified text → FAIL
+    """
+    inp_n = normalize(input_text)
+    out_n = normalize(output_text)
+    exp_n = normalize(expected_text)
+    
+    is_preservation = dataset in ('entities', 'religious', 'structured', 'hallucination')
+    text_changed = (out_n != inp_n)
+    
+    if is_preservation:
+        # For preservation tests, the expected output == input (don't change)
+        if not text_changed:
+            return "TN"  # Correctly preserved
+        else:
+            return "FP"  # Incorrectly modified
+    else:
+        # For correction tests
+        needs_correction = (inp_n != exp_n)
+        
+        if needs_correction:
+            if text_changed:
+                # Check if output matches expected (or is closer to expected)
+                if out_n == exp_n:
+                    return "TP"  # Perfect correction
+                elif _edit_distance(out_n, exp_n) < _edit_distance(inp_n, exp_n):
+                    return "TP"  # Partial but improving correction
+                else:
+                    return "FP"  # Changed but not in right direction
+            else:
+                return "FN"  # Should have corrected but didn't
+        else:
+            if text_changed:
+                return "FP"  # Changed text that was already correct
+            else:
+                return "TN"  # Correctly left unchanged
+
+
+def _edit_distance(a, b):
+    """Simple Levenshtein edit distance."""
+    if len(a) < len(b):
+        return _edit_distance(b, a)
+    if len(b) == 0:
+        return len(a)
+    
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a):
+        curr = [i + 1]
+        for j, cb in enumerate(b):
+            cost = 0 if ca == cb else 1
+            curr.append(min(curr[j] + 1, prev[j + 1] + 1, prev[j] + cost))
+        prev = curr
+    return prev[len(b)]
+
+
+def load_datasets(dataset_filter=None):
+    datasets = {}
+    for f in sorted(DATASETS_DIR.glob("*.json")):
+        name = f.stem
+        if dataset_filter and name != dataset_filter:
+            continue
+        with open(f, 'r', encoding='utf-8') as fh:
+            datasets[name] = json.load(fh)
+    return datasets
+
+
+def run_level2(api: APIClient, datasets: dict) -> List[SoloResult]:
+    results = []
+    total = sum(len(v) for v in datasets.values())
+    idx = 0
+
+    for ds_name, cases in datasets.items():
+        print(f"\n{'='*60}")
+        print(f"DATASET: {ds_name.upper()} ({len(cases)} samples)")
+        print(f"{'='*60}")
+
+        for case in cases:
+            idx += 1
+            cid = case.get('id', f'{ds_name}_{idx}')
+            cat = case.get('category', '')
+            inp = case.get('input', '')
+            expected = case.get('expected', case.get('input', ''))
+            severity = case.get('severity', '')
+
+            r = SoloResult(
+                id=cid, dataset=ds_name, category=cat,
+                input_text=inp, expected=expected, severity=severity
+            )
+
+            print(f"  [{idx}/{total}] {cid} ({cat})...", end=" ", flush=True)
+
+            # Test each model independently on the SAME original input
+            r.spelling_solo, r.spelling_ms = api.call("/api/spelling", inp)
+            r.grammar_solo, r.grammar_ms = api.call("/api/grammar", inp)
+            r.punctuation_solo, r.punctuation_ms = api.call("/api/punctuation", inp)
+
+            # Classify each model's result
+            r.spelling_verdict = classify_result(inp, r.spelling_solo, expected, ds_name)
+            r.grammar_verdict = classify_result(inp, r.grammar_solo, expected, ds_name)
+            r.punctuation_verdict = classify_result(inp, r.punctuation_solo, expected, ds_name)
+
+            total_ms = r.spelling_ms + r.grammar_ms + r.punctuation_ms
+            print(f"S={r.spelling_verdict} G={r.grammar_verdict} P={r.punctuation_verdict} ({total_ms}ms)")
+
+            results.append(r)
+
+    return results
+
+
+def analyze_and_print(results: List[SoloResult]) -> dict:
+    analysis = {"total": len(results), "by_model": {}, "by_dataset": {}}
+    
+    for model in ("spelling", "grammar", "punctuation"):
+        verdicts = {"TP": 0, "TN": 0, "FP": 0, "FN": 0}
+        for r in results:
+            v = getattr(r, f"{model}_verdict", "")
+            if v in verdicts:
+                verdicts[v] += 1
+        total = sum(verdicts.values())
+        pass_count = verdicts["TP"] + verdicts["TN"]
+        analysis["by_model"][model] = {
+            **verdicts,
+            "pass_rate": round(pass_count / total, 4) if total else 0,
+        }
+
+    # Per-dataset breakdown
+    for ds in set(r.dataset for r in results):
+        ds_results = [r for r in results if r.dataset == ds]
+        ds_analysis = {}
+        for model in ("spelling", "grammar", "punctuation"):
+            verdicts = {"TP": 0, "TN": 0, "FP": 0, "FN": 0}
+            for r in ds_results:
+                v = getattr(r, f"{model}_verdict", "")
+                if v in verdicts:
+                    verdicts[v] += 1
+            total = sum(verdicts.values())
+            pass_count = verdicts["TP"] + verdicts["TN"]
+            ds_analysis[model] = {
+                **verdicts,
+                "pass_rate": round(pass_count / total, 4) if total else 0,
+            }
+        analysis["by_dataset"][ds] = {"total": len(ds_results), **ds_analysis}
+
+    # Print
+    print(f"\n{'='*60}")
+    print("LEVEL 2: SOLO API ANALYSIS")
+    print(f"{'='*60}")
+
+    print(f"\n## Per-Model Summary ({analysis['total']} tests)")
+    print(f"| Model       | TP  | TN  | FP  | FN  | Pass%  |")
+    print(f"|-------------|-----|-----|-----|-----|--------|")
+    for model, data in analysis["by_model"].items():
+        print(f"| {model:<11} | {data['TP']:>3} | {data['TN']:>3} | {data['FP']:>3} | {data['FN']:>3} | {data['pass_rate']*100:5.1f}% |")
+
+    print(f"\n## Per-Dataset × Model Pass Rate")
+    print(f"| Dataset      | Spelling | Grammar | Punctuation |")
+    print(f"|--------------|----------|---------|-------------|")
+    for ds in sorted(analysis["by_dataset"].keys()):
+        d = analysis["by_dataset"][ds]
+        s = d["spelling"]["pass_rate"] * 100
+        g = d["grammar"]["pass_rate"] * 100
+        p = d["punctuation"]["pass_rate"] * 100
+        print(f"| {ds:<12} | {s:6.1f}%  | {g:5.1f}%  | {p:9.1f}%  |")
+
+    return analysis
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Level 2: Solo API Tests")
+    parser.add_argument("--url", default="https://bayan10-bayan-api.hf.space")
+    parser.add_argument("--dataset", default=None)
+    args = parser.parse_args()
+
+    api = APIClient(args.url)
+    datasets = load_datasets(args.dataset)
+
+    print(f"\n{'='*60}")
+    print("BAYAN v2.0 — Level 2: Solo API Tests")
+    print(f"{'='*60}")
+    print(f"  Target:   {args.url}")
+    print(f"  Datasets: {list(datasets.keys())}")
+    print(f"  Total:    {sum(len(v) for v in datasets.values())} tests")
+
+    results = run_level2(api, datasets)
+    analysis = analyze_and_print(results)
+
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = REPORT_DIR / "level2_solo_results.json"
+    report = {
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "target": args.url,
+        "analysis": analysis,
+        "results": [asdict(r) for r in results],
+    }
+    with open(out_path, 'w', encoding='utf-8') as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+    print(f"\n[L2] Results → {out_path}")
+
+
+if __name__ == "__main__":
+    main()

tests/v2/test_level3_integrated.py

@@ -0,0 +1,297 @@
+"""
+BAYAN v2.0 — Level 3: Integrated Pipeline Tests
+=================================================
+Tests the FULL integrated pipeline through /api/analyze.
+This is the end-to-end test: Spelling → Grammar → Punctuation
+with all filters, StageLocker, OffsetMapper, PatchSet.
+
+Reuses the exact same verdict logic as the existing benchmark_runner.py
+to ensure comparability.
+
+Usage:
+    python tests/v2/test_level3_integrated.py --url URL [--dataset DATASET]
+"""
+import argparse
+import json
+import re
+import time
+from pathlib import Path
+from dataclasses import dataclass, asdict
+from typing import List
+import requests
+
+DATASETS_DIR = Path(__file__).parent.parent / "phase10" / "gold_datasets"
+REPORT_DIR = Path(__file__).parent / "reports"
+
+
+def normalize(text):
+    t = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+    t = re.sub(r'\s+', ' ', t).strip()
+    return t
+
+
+@dataclass
+class IntegratedResult:
+    id: str
+    dataset: str
+    category: str
+    input_text: str
+    expected: str
+    severity: str
+    # Pipeline output
+    pipeline_corrected: str = ""
+    pipeline_suggestions: int = 0
+    pipeline_ms: int = 0
+    spelling_ms: int = 0
+    grammar_ms: int = 0
+    punctuation_ms: int = 0
+    # Verdict
+    verdict: str = ""      # TP, TN, FP, FN
+    detail: str = ""
+
+
+class APIClient:
+    def __init__(self, base_url):
+        self.base = base_url.rstrip('/')
+        self.session = requests.Session()
+        self.session.headers['Content-Type'] = 'application/json'
+
+    def analyze(self, text, timeout=120):
+        t0 = time.time()
+        try:
+            r = self.session.post(
+                f"{self.base}/api/analyze",
+                json={"text": text},
+                timeout=timeout
+            )
+            ms = int((time.time() - t0) * 1000)
+            return r.json(), ms
+        except Exception as e:
+            ms = int((time.time() - t0) * 1000)
+            return {"error": str(e)}, ms
+
+
+def classify_pipeline(input_text, corrected_text, expected_text, dataset, entity=None):
+    """Classify pipeline output.
+    
+    For correction datasets (spelling, grammar, punctuation, collision):
+      Input has errors → should be corrected to match expected.
+    
+    For preservation datasets (entities, religious, structured, hallucination):
+      Input is correct → should NOT be modified.
+      For entity tests: specifically check that the entity string is preserved.
+    """
+    inp_n = normalize(input_text)
+    out_n = normalize(corrected_text)
+    exp_n = normalize(expected_text)
+    
+    is_preservation = dataset in ('entities', 'religious', 'structured', 'hallucination')
+    text_changed = (out_n != inp_n)
+    
+    if dataset == 'entities' and entity:
+        # Entity tests: check if entity is preserved in output
+        entity_n = normalize(entity)
+        if entity_n in out_n:
+            return "TN", "Entity preserved"
+        elif not text_changed:
+            return "TN", "Text unchanged"
+        else:
+            return "FP", f"ENTITY CORRUPTED: '{entity}' missing from output"
+    
+    if is_preservation:
+        if not text_changed:
+            return "TN", "Text correctly preserved"
+        else:
+            # Check what changed
+            inp_words = inp_n.split()
+            out_words = out_n.split()
+            changes = []
+            for iw, ow in zip(inp_words, out_words):
+                if iw != ow:
+                    changes.append(f"{iw}→{ow}")
+            detail = f"Text modified: {changes[:5]}"
+            return "FP", detail
+    else:
+        # Correction dataset
+        needs_correction = (inp_n != exp_n)
+        
+        if needs_correction:
+            if out_n == exp_n:
+                return "TP", "Exact match"
+            elif text_changed and _closer(out_n, inp_n, exp_n):
+                return "TP", "Partial improvement"
+            elif not text_changed:
+                return "FN", "No correction applied"
+            else:
+                return "FP", f"Wrong correction"
+        else:
+            if not text_changed:
+                return "TN", "Correctly unchanged"
+            else:
+                return "FP", f"Modified correct text"
+
+
+def _closer(output, input_text, expected):
+    """Is output closer to expected than input was?"""
+    d_out = _edit_distance(output, expected)
+    d_inp = _edit_distance(input_text, expected)
+    return d_out < d_inp
+
+
+def _edit_distance(a, b):
+    if len(a) < len(b):
+        return _edit_distance(b, a)
+    if len(b) == 0:
+        return len(a)
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a):
+        curr = [i + 1]
+        for j, cb in enumerate(b):
+            cost = 0 if ca == cb else 1
+            curr.append(min(curr[j] + 1, prev[j + 1] + 1, prev[j] + cost))
+        prev = curr
+    return prev[len(b)]
+
+
+def load_datasets(dataset_filter=None):
+    datasets = {}
+    for f in sorted(DATASETS_DIR.glob("*.json")):
+        name = f.stem
+        if dataset_filter and name != dataset_filter:
+            continue
+        with open(f, 'r', encoding='utf-8') as fh:
+            datasets[name] = json.load(fh)
+    return datasets
+
+
+def run_level3(api: APIClient, datasets: dict) -> List[IntegratedResult]:
+    results = []
+    total = sum(len(v) for v in datasets.values())
+    idx = 0
+
+    for ds_name, cases in datasets.items():
+        print(f"\n{'='*60}")
+        print(f"DATASET: {ds_name.upper()} ({len(cases)} samples)")
+        print(f"{'='*60}")
+
+        tp = tn = fp = fn = 0
+        for case in cases:
+            idx += 1
+            cid = case.get('id', f'{ds_name}_{idx}')
+            cat = case.get('category', '')
+            inp = case.get('input', '')
+            expected = case.get('expected', case.get('input', ''))
+            severity = case.get('severity', '')
+            entity = case.get('entity', None)
+
+            r = IntegratedResult(
+                id=cid, dataset=ds_name, category=cat,
+                input_text=inp, expected=expected, severity=severity
+            )
+
+            print(f"  [{idx}/{total}] {cid} ({cat})...", end=" ", flush=True)
+
+            data, ms = api.analyze(inp)
+            r.pipeline_ms = ms
+            r.pipeline_corrected = data.get('corrected', inp)
+            r.pipeline_suggestions = len(data.get('suggestions', []))
+            timing = data.get('timing_ms', {})
+            r.spelling_ms = timing.get('spelling_ms', 0)
+            r.grammar_ms = timing.get('grammar_ms', 0)
+            r.punctuation_ms = timing.get('punctuation_ms', 0)
+
+            r.verdict, r.detail = classify_pipeline(
+                inp, r.pipeline_corrected, expected, ds_name, entity
+            )
+
+            icon = {"TP": "✅", "TN": "✅", "FP": "❌", "FN": "⚠️"}.get(r.verdict, "?")
+            print(f"{icon} {r.verdict} ({r.pipeline_ms}ms)")
+
+            if r.verdict == "TP": tp += 1
+            elif r.verdict == "TN": tn += 1
+            elif r.verdict == "FP": fp += 1
+            elif r.verdict == "FN": fn += 1
+
+            results.append(r)
+
+        total_ds = tp + tn + fp + fn
+        pass_pct = (tp + tn) / total_ds * 100 if total_ds else 0
+        print(f"\n  Pass={pass_pct:.1f}% TP={tp} TN={tn} FP={fp} FN={fn}")
+
+    return results
+
+
+def analyze_and_print(results: List[IntegratedResult]) -> dict:
+    verdicts = {"TP": 0, "TN": 0, "FP": 0, "FN": 0}
+    by_dataset = {}
+
+    for r in results:
+        verdicts[r.verdict] = verdicts.get(r.verdict, 0) + 1
+        if r.dataset not in by_dataset:
+            by_dataset[r.dataset] = {"TP": 0, "TN": 0, "FP": 0, "FN": 0, "total": 0}
+        by_dataset[r.dataset][r.verdict] += 1
+        by_dataset[r.dataset]["total"] += 1
+
+    total = sum(verdicts.values())
+    pass_count = verdicts["TP"] + verdicts["TN"]
+
+    analysis = {
+        "total": total,
+        "aggregate": {
+            **verdicts,
+            "pass_rate": round(pass_count / total, 4) if total else 0,
+        },
+        "by_dataset": {},
+    }
+
+    print(f"\n{'='*60}")
+    print("LEVEL 3: INTEGRATED PIPELINE ANALYSIS")
+    print(f"{'='*60}")
+    print(f"\n  Total: {total} | Pass: {pass_count}/{total} ({analysis['aggregate']['pass_rate']*100:.1f}%)")
+    print(f"  TP={verdicts['TP']} TN={verdicts['TN']} FP={verdicts['FP']} FN={verdicts['FN']}")
+
+    print(f"\n| Dataset      | Total | TP  | TN  | FP  | FN  | Pass%  |")
+    print(f"|--------------|-------|-----|-----|-----|-----|--------|")
+    for ds in sorted(by_dataset.keys()):
+        d = by_dataset[ds]
+        p = (d["TP"] + d["TN"]) / d["total"] * 100 if d["total"] else 0
+        print(f"| {ds:<12} | {d['total']:>5} | {d['TP']:>3} | {d['TN']:>3} | {d['FP']:>3} | {d['FN']:>3} | {p:5.1f}% |")
+        analysis["by_dataset"][ds] = {**d, "pass_rate": round(p / 100, 4)}
+
+    return analysis
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Level 3: Integrated Pipeline Tests")
+    parser.add_argument("--url", default="https://bayan10-bayan-api.hf.space")
+    parser.add_argument("--dataset", default=None)
+    args = parser.parse_args()
+
+    api = APIClient(args.url)
+    datasets = load_datasets(args.dataset)
+
+    print(f"\n{'='*60}")
+    print("BAYAN v2.0 — Level 3: Integrated Pipeline Tests")
+    print(f"{'='*60}")
+    print(f"  Target:   {args.url}")
+    print(f"  Datasets: {list(datasets.keys())}")
+    print(f"  Total:    {sum(len(v) for v in datasets.values())} tests")
+
+    results = run_level3(api, datasets)
+    analysis = analyze_and_print(results)
+
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+    out_path = REPORT_DIR / "level3_integrated_results.json"
+    report = {
+        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "target": args.url,
+        "analysis": analysis,
+        "results": [asdict(r) for r in results],
+    }
+    with open(out_path, 'w', encoding='utf-8') as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+    print(f"\n[L3] Results → {out_path}")
+
+
+if __name__ == "__main__":
+    main()