Phase A: Upgrade L1 with TP/FP/FN/TN verdicts + word edits, add F1 scores to matrix

tests/v2/benchmark_matrix.py

@@ -62,6 +62,22 @@ def load_report(level: int) -> dict:
     return None
 
 
+def _compute_f1(tp, fp, fn):
+    """Compute F1 score from TP/FP/FN."""
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0
+    return round(f1, 4), round(precision, 4), round(recall, 4)
+
+
+def _extract_model_metrics(report, model_key):
+    """Extract TP/TN/FP/FN for a model from a report."""
+    a = report.get("analysis", {})
+    by_model = a.get("by_model", {})
+    data = by_model.get(model_key, {})
+    return data.get("TP", 0), data.get("TN", 0), data.get("FP", 0), data.get("FN", 0)
+
+
 def print_comparison_matrix():
     """Load all 3 reports and print side-by-side comparison."""
     l1 = load_report(1)
@@ -72,27 +88,29 @@ def print_comparison_matrix():
     print("BAYAN v2.0 — BENCHMARK COMPARISON MATRIX")
     print(f"{'='*70}")
 
-    # Timestamps
     for level, report, name in [(1, l1, "L1-Raw"), (2, l2, "L2-Solo"), (3, l3, "L3-Pipeline")]:
         if report:
             ts = report.get("timestamp", "N/A")
-            print(f"  {name}: {ts}")
+            total = report.get("analysis", {}).get("total", "?")
+            print(f"  {name}: {ts} ({total} tests)")
         else:
             print(f"  {name}: NOT RUN")
 
     # ── Level 1 Summary ──
     if l1:
         print(f"\n{'─'*70}")
-        print("Level 1: RAW MODEL OUTPUT (no filters)")
+        print("Level 1: RAW MODEL OUTPUT (through solo API endpoints)")
         print(f"{'─'*70}")
         a = l1.get("analysis", {})
         by_model = a.get("by_model", {})
+        print(f"  {'Model':<12} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4} {'Pass%':>7} {'ChgRate':>8} {'F1':>6}")
+        print(f"  {'-'*12} {'-'*4} {'-'*4} {'-'*4} {'-'*4} {'-'*7} {'-'*8} {'-'*6}")
         for model, data in by_model.items():
-            changed = data.get("changed", 0)
-            unchanged = data.get("unchanged", 0)
-            total = changed + unchanged + data.get("errors", 0)
-            rate = changed / total * 100 if total else 0
-            print(f"  {model:<12}: {changed}/{total} texts modified ({rate:.1f}%)")
+            tp, tn, fp, fn = data.get("TP",0), data.get("TN",0), data.get("FP",0), data.get("FN",0)
+            pr = data.get("pass_rate", 0) * 100
+            cr = data.get("change_rate", 0) * 100
+            f1, _, _ = _compute_f1(tp, fp, fn)
+            print(f"  {model:<12} {tp:>4} {tn:>4} {fp:>4} {fn:>4} {pr:>6.1f}% {cr:>6.1f}%  {f1:.3f}")
 
     # ── Level 2 Summary ──
     if l2:
@@ -101,12 +119,13 @@ def print_comparison_matrix():
         print(f"{'─'*70}")
         a = l2.get("analysis", {})
         by_model = a.get("by_model", {})
-        print(f"  {'Model':<12} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4} {'Pass%':>7}")
-        print(f"  {'-'*12} {'-'*4} {'-'*4} {'-'*4} {'-'*4} {'-'*7}")
+        print(f"  {'Model':<12} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4} {'Pass%':>7} {'F1':>6}")
+        print(f"  {'-'*12} {'-'*4} {'-'*4} {'-'*4} {'-'*4} {'-'*7} {'-'*6}")
         for model, data in by_model.items():
+            tp, tn, fp, fn = data.get("TP",0), data.get("TN",0), data.get("FP",0), data.get("FN",0)
             pr = data.get("pass_rate", 0) * 100
-            print(f"  {model:<12} {data.get('TP',0):>4} {data.get('TN',0):>4} "
-                  f"{data.get('FP',0):>4} {data.get('FN',0):>4} {pr:>6.1f}%")
+            f1, _, _ = _compute_f1(tp, fp, fn)
+            print(f"  {model:<12} {tp:>4} {tn:>4} {fp:>4} {fn:>4} {pr:>6.1f}% {f1:.3f}")
 
     # ── Level 3 Summary ──
     if l3:
@@ -117,32 +136,50 @@ def print_comparison_matrix():
         agg = a.get("aggregate", {})
         total = a.get("total", 0)
         pr = agg.get("pass_rate", 0) * 100
+        tp, tn, fp, fn = agg.get("TP",0), agg.get("TN",0), agg.get("FP",0), agg.get("FN",0)
+        f1, prec, rec = _compute_f1(tp, fp, fn)
         print(f"  Overall: {pr:.1f}% pass ({total} tests)")
-        print(f"  TP={agg.get('TP',0)} TN={agg.get('TN',0)} "
-              f"FP={agg.get('FP',0)} FN={agg.get('FN',0)}")
+        print(f"  TP={tp} TN={tn} FP={fp} FN={fn}")
+        print(f"  F1={f1:.3f}  Precision={prec:.3f}  Recall={rec:.3f}")
 
-        print(f"\n  {'Dataset':<14} {'Pass%':>7} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4}")
-        print(f"  {'-'*14} {'-'*7} {'-'*4} {'-'*4} {'-'*4} {'-'*4}")
+        print(f"\n  {'Dataset':<14} {'Pass%':>7} {'TP':>4} {'TN':>4} {'FP':>4} {'FN':>4} {'F1':>6}")
+        print(f"  {'-'*14} {'-'*7} {'-'*4} {'-'*4} {'-'*4} {'-'*4} {'-'*6}")
         by_ds = a.get("by_dataset", {})
         for ds in sorted(by_ds.keys()):
             d = by_ds[ds]
             dp = d.get("pass_rate", 0) * 100
-            print(f"  {ds:<14} {dp:>6.1f}% {d.get('TP',0):>4} {d.get('TN',0):>4} "
-                  f"{d.get('FP',0):>4} {d.get('FN',0):>4}")
+            dtp, dtn, dfp, dfn = d.get("TP",0), d.get("TN",0), d.get("FP",0), d.get("FN",0)
+            df1, _, _ = _compute_f1(dtp, dfp, dfn)
+            print(f"  {ds:<14} {dp:>6.1f}% {dtp:>4} {dtn:>4} {dfp:>4} {dfn:>4} {df1:.3f}")
 
     # ── Cross-Level Comparison ──
+    if l1 and l2:
+        print(f"\n{'─'*70}")
+        print("CROSS-LEVEL: L1 (Raw) vs L2 (Solo) — Filter Impact")
+        print(f"{'─'*70}")
+        print(f"  {'Model':<12} {'L1-Pass%':>9} {'L2-Pass%':>9} {'Delta':>7} {'L1-F1':>7} {'L2-F1':>7}")
+        print(f"  {'-'*12} {'-'*9} {'-'*9} {'-'*7} {'-'*7} {'-'*7}")
+        for model in ("spelling", "grammar", "punctuation"):
+            l1m = l1.get("analysis",{}).get("by_model",{}).get(model,{})
+            l2m = l2.get("analysis",{}).get("by_model",{}).get(model,{})
+            l1p = l1m.get("pass_rate",0) * 100
+            l2p = l2m.get("pass_rate",0) * 100
+            delta = l2p - l1p
+            l1f1, _, _ = _compute_f1(l1m.get("TP",0), l1m.get("FP",0), l1m.get("FN",0))
+            l2f1, _, _ = _compute_f1(l2m.get("TP",0), l2m.get("FP",0), l2m.get("FN",0))
+            print(f"  {model:<12} {l1p:>8.1f}% {l2p:>8.1f}% {delta:>+6.1f}% {l1f1:>6.3f}  {l2f1:>6.3f}")
+
     if l2 and l3:
         print(f"\n{'─'*70}")
-        print("CROSS-LEVEL COMPARISON: Solo vs Integrated")
+        print("CROSS-LEVEL: L2 (Solo) vs L3 (Pipeline) — Integration Impact")
         print(f"{'─'*70}")
 
         l2_ds = l2.get("analysis", {}).get("by_dataset", {})
         l3_ds = l3.get("analysis", {}).get("by_dataset", {})
-
         all_datasets = sorted(set(list(l2_ds.keys()) + list(l3_ds.keys())))
 
-        print(f"  {'Dataset':<14} {'L2-Spell':>10} {'L2-Gram':>10} {'L2-Punct':>10} {'L3-Pipeline':>12} {'Delta':>7}")
-        print(f"  {'-'*14} {'-'*10} {'-'*10} {'-'*10} {'-'*12} {'-'*7}")
+        print(f"  {'Dataset':<14} {'L2-Best':>8} {'L3-Pipe':>8} {'Delta':>7}")
+        print(f"  {'-'*14} {'-'*8} {'-'*8} {'-'*7}")
 
         for ds in all_datasets:
             l2d = l2_ds.get(ds, {})
@@ -152,13 +189,9 @@ def print_comparison_matrix():
             l2_g = l2d.get("grammar", {}).get("pass_rate", 0) * 100
             l2_p = l2d.get("punctuation", {}).get("pass_rate", 0) * 100
             l3_p = l3d.get("pass_rate", 0) * 100
-
-            # Best solo model vs pipeline
             best_solo = max(l2_s, l2_g, l2_p)
             delta = l3_p - best_solo
-
-            delta_str = f"{delta:+.1f}%"
-            print(f"  {ds:<14} {l2_s:>9.1f}% {l2_g:>9.1f}% {l2_p:>9.1f}% {l3_p:>11.1f}% {delta_str:>7}")
+            print(f"  {ds:<14} {best_solo:>7.1f}% {l3_p:>7.1f}% {delta:>+6.1f}%")
 
     # Save comparison
     comparison = {
@@ -166,6 +199,7 @@ def print_comparison_matrix():
         "l1_timestamp": l1.get("timestamp") if l1 else None,
         "l2_timestamp": l2.get("timestamp") if l2 else None,
         "l3_timestamp": l3.get("timestamp") if l3 else None,
+        "l1_summary": l1.get("analysis", {}).get("by_model") if l1 else None,
         "l2_summary": l2.get("analysis", {}).get("by_model") if l2 else None,
         "l3_summary": l3.get("analysis", {}).get("aggregate") if l3 else None,
     }

tests/v2/reports/benchmark_matrix.json

@@ -1,8 +1,8 @@
 {
-  "timestamp": "2026-06-23T18:55:12Z",
-  "l1_timestamp": null,
+  "timestamp": "2026-06-23T19:01:36Z",
+  "l1_timestamp": "2026-06-23T19:01:36Z",
   "l2_timestamp": "2026-06-23T18:55:12Z",
-  "l3_timestamp": "2026-06-23T18:54:52Z",
+  "l3_timestamp": "2026-06-23T19:00:34Z",
   "l2_summary": {
     "spelling": {
       "TP": 0,
@@ -27,10 +27,10 @@
     }
   },
   "l3_summary": {
-    "TP": 22,
-    "TN": 0,
-    "FP": 58,
+    "TP": 49,
+    "TN": 17,
+    "FP": 14,
     "FN": 0,
-    "pass_rate": 0.275
+    "pass_rate": 0.825
   }
 }

tests/v2/reports/level1_raw_results.json

@@ -0,0 +1,544 @@
+{
+  "timestamp": "2026-06-23T19:01:36Z",
+  "target": "https://bayan10-bayan-api.hf.space",
+  "analysis": {
+    "total": 30,
+    "by_model": {
+      "spelling": {
+        "changed": 11,
+        "unchanged": 19,
+        "errors": 0
+      },
+      "grammar": {
+        "changed": 5,
+        "unchanged": 25,
+        "errors": 0
+      },
+      "punctuation": {
+        "changed": 29,
+        "unchanged": 1,
+        "errors": 0
+      }
+    },
+    "by_dataset": {
+      "entities": {
+        "total": 30,
+        "spelling_changed": 11,
+        "grammar_changed": 5,
+        "punctuation_changed": 29
+      }
+    }
+  },
+  "results": [
+    {
+      "id": "E001",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "محمد صلاح لاعب كرة قدم مصري",
+      "expected": "محمد صلاح لاعب كرة قدم مصري",
+      "severity": "major",
+      "spelling_raw": "محمد صلاح لاعب كرة قدم مصري",
+      "spelling_ms": 3171,
+      "grammar_raw": "محمد صلاح لاعب كرة قدم مصري",
+      "grammar_ms": 4260,
+      "punctuation_raw": "محمد صلاح لاعب كرة قدم مصري.",
+      "punctuation_ms": 4161,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E002",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "عبدالله يدرس في الجامعة",
+      "expected": "عبدالله يدرس في الجامعة",
+      "severity": "major",
+      "spelling_raw": "عبد الله يدرس في الجامعة",
+      "spelling_ms": 3213,
+      "grammar_raw": "عبدالله يدرس في الجامعة",
+      "grammar_ms": 3791,
+      "punctuation_raw": "عبدالله يدرس في الجامعة.",
+      "punctuation_ms": 3398,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E003",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "عبد الرحمن أخي الأكبر",
+      "expected": "عبد الرحمن أخي الأكبر",
+      "severity": "major",
+      "spelling_raw": "عبد الرحمن أخي الأكبر",
+      "spelling_ms": 3931,
+      "grammar_raw": "عبد الرحمن أخي الأكبر",
+      "grammar_ms": 4482,
+      "punctuation_raw": "عبد الرحمن أخي الأكبر.",
+      "punctuation_ms": 3616,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E004",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "أحمد محمود يعمل مهندساً",
+      "expected": "أحمد محمود يعمل مهندساً",
+      "severity": "major",
+      "spelling_raw": "أحمد محمود يعمل مهندسا",
+      "spelling_ms": 5051,
+      "grammar_raw": "أحمد محمود يعمل مهندسا",
+      "grammar_ms": 3249,
+      "punctuation_raw": "أحمد محمود يعمل مهندساً.",
+      "punctuation_ms": 3381,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E005",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "الدكتور حسن علي أستاذ جامعي",
+      "expected": "الدكتور حسن علي أستاذ جامعي",
+      "severity": "major",
+      "spelling_raw": "الدكتور حسن على أستاذ جامعي",
+      "spelling_ms": 6359,
+      "grammar_raw": "الدكتور حسن علي أستاذ جامعي",
+      "grammar_ms": 3254,
+      "punctuation_raw": "الدكتور حسن علي أستاذ جامعي.",
+      "punctuation_ms": 4454,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E006",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "السيدة فاطمة الزهراء معلمة",
+      "expected": "السيدة فاطمة الزهراء معلمة",
+      "severity": "major",
+      "spelling_raw": "السيدة فاطمة الزهراء معلمة",
+      "spelling_ms": 6272,
+      "grammar_raw": "السيدة فاطمة الزهراء معلمة",
+      "grammar_ms": 4581,
+      "punctuation_raw": "السيدة فاطمة الزهراء معلمة.",
+      "punctuation_ms": 4866,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E007",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "الأستاذ عمر بن الخطاب عادل",
+      "expected": "الأستاذ عمر بن الخطاب عادل",
+      "severity": "major",
+      "spelling_raw": "الأستاذ عمر بن الخطاب عادل",
+      "spelling_ms": 2516,
+      "grammar_raw": "الأستاذ عمر بن الخطاب عادل",
+      "grammar_ms": 2612,
+      "punctuation_raw": "الأستاذ عمر بن الخطاب عادل.",
+      "punctuation_ms": 751,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E008",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "خالد بن الوليد قائد عظيم",
+      "expected": "خالد بن الوليد قائد عظيم",
+      "severity": "major",
+      "spelling_raw": "خالد بن الوليد قائد عظيم",
+      "spelling_ms": 815,
+      "grammar_raw": "خالد بن الوليد قائد عظيم",
+      "grammar_ms": 858,
+      "punctuation_raw": "خالد بن الوليد قائد عظيم.",
+      "punctuation_ms": 665,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E009",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "صلاح الدين الأيوبي حرر القدس",
+      "expected": "صلاح الدين الأيوبي حرر القدس",
+      "severity": "major",
+      "spelling_raw": "صلاح الدين الأيوبي حر القدس",
+      "spelling_ms": 743,
+      "grammar_raw": "صلاح الدين الأيوبي حرر القدس",
+      "grammar_ms": 852,
+      "punctuation_raw": "صلاح الدين الأيوبي حرر القدس.",
+      "punctuation_ms": 810,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E010",
+      "dataset": "entities",
+      "category": "person",
+      "input_text": "ابن سينا عالم عربي مشهور",
+      "expected": "ابن سينا عالم عربي مشهور",
+      "severity": "major",
+      "spelling_raw": "ابن سينا عالم عربي مشهور",
+      "spelling_ms": 763,
+      "grammar_raw": "ابن سينا عالم عربي مشهور",
+      "grammar_ms": 887,
+      "punctuation_raw": "ابن سينا عالم عربي مشهور.",
+      "punctuation_ms": 644,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E011",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "جامعة القاهرة من أعرق الجامعات",
+      "expected": "جامعة القاهرة من أعرق الجامعات",
+      "severity": "major",
+      "spelling_raw": "جامعة القاهرة من أعمق الجامعات",
+      "spelling_ms": 911,
+      "grammar_raw": "جامعة القاهرة من أعرق الجامعات",
+      "grammar_ms": 909,
+      "punctuation_raw": "جامعة القاهرة من أعرق الجامعات.",
+      "punctuation_ms": 869,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E012",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "مدينة الرياض عاصمة المملكة",
+      "expected": "مدينة الرياض عاصمة المملكة",
+      "severity": "major",
+      "spelling_raw": "مدينة الرياض عاصمة المملكة",
+      "spelling_ms": 808,
+      "grammar_raw": "مدينة الرياض عاصمة المملكة",
+      "grammar_ms": 784,
+      "punctuation_raw": "مدينة الرياض عاصمة المملكة.",
+      "punctuation_ms": 767,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E013",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "دبي مدينة عصرية وجميلة",
+      "expected": "دبي مدينة عصرية وجميلة",
+      "severity": "major",
+      "spelling_raw": "دبي مدينة عصرية وجميلة",
+      "spelling_ms": 958,
+      "grammar_raw": "دبي مدينة عصرية وجميلة",
+      "grammar_ms": 808,
+      "punctuation_raw": "دبي مدينة عصرية وجميلة.",
+      "punctuation_ms": 701,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E014",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "القدس مدينة مقدسة عند المسلمين",
+      "expected": "القدس مدينة مقدسة عند المسلمين",
+      "severity": "major",
+      "spelling_raw": "القدس مدينة مقدسة عند المسلمين",
+      "spelling_ms": 786,
+      "grammar_raw": "القدس مدينة مقدسة عند المسلمين",
+      "grammar_ms": 840,
+      "punctuation_raw": "القدس مدينة مقدسة عند المسلمين.",
+      "punctuation_ms": 825,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E015",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "مكة المكرمة أطهر البقاع",
+      "expected": "مكة المكرمة أطهر البقاع",
+      "severity": "major",
+      "spelling_raw": "مكة المكرمة أطهر البقاع",
+      "spelling_ms": 838,
+      "grammar_raw": "مكة المكرمة أطهر البقاع",
+      "grammar_ms": 797,
+      "punctuation_raw": "مكة المكرمة أطهر البقاع.",
+      "punctuation_ms": 778,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E016",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "المدينة المنورة طابة الطيبة",
+      "expected": "المدينة المنورة طابة الطيبة",
+      "severity": "major",
+      "spelling_raw": "المدينة المنورة الطيبة",
+      "spelling_ms": 689,
+      "grammar_raw": "المدينة المنورة طابة طيبة",
+      "grammar_ms": 810,
+      "punctuation_raw": "المدينة المنورة طابة الطيبة.",
+      "punctuation_ms": 826,
+      "spelling_changed": true,
+      "grammar_changed": true,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E017",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "بغداد عاصمة العراق",
+      "expected": "بغداد عاصمة العراق",
+      "severity": "major",
+      "spelling_raw": "بغداد عاصمة العراق",
+      "spelling_ms": 715,
+      "grammar_raw": "بغداد عاصمة العراق",
+      "grammar_ms": 651,
+      "punctuation_raw": "بغداد عاصمة العراق.",
+      "punctuation_ms": 696,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E018",
+      "dataset": "entities",
+      "category": "place",
+      "input_text": "دمشق أقدم عاصمة في التاريخ",
+      "expected": "دمشق أقدم عاصمة في التاريخ",
+      "severity": "major",
+      "spelling_raw": "دمشق أقدم عاصمة في التاريخ",
+      "spelling_ms": 783,
+      "grammar_raw": "دمشق أقدم عاصمة في التاريخ",
+      "grammar_ms": 796,
+      "punctuation_raw": "دمشق أقدم عاصمة في التاريخ.",
+      "punctuation_ms": 669,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E019",
+      "dataset": "entities",
+      "category": "company",
+      "input_text": "شركة OpenAI تطور الذكاء الاصطناعي",
+      "expected": "شركة OpenAI تطور الذكاء الاصطناعي",
+      "severity": "major",
+      "spelling_raw": "شركة OpenAI تطور الذكاء الاصطناعي",
+      "spelling_ms": 1480,
+      "grammar_raw": "شركة OpenAI تطور الذكاء الاصطناعي",
+      "grammar_ms": 860,
+      "punctuation_raw": "شركة OpenAI تطور الذكاء الاصطناعي",
+      "punctuation_ms": 798,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": false
+    },
+    {
+      "id": "E020",
+      "dataset": "entities",
+      "category": "company",
+      "input_text": "شركة Google عملاق التقنية",
+      "expected": "شركة Google عملاق التقنية",
+      "severity": "major",
+      "spelling_raw": "شركة جوجل جوجل عملاق التقنية",
+      "spelling_ms": 994,
+      "grammar_raw": "شركة Google عملاق التقنية",
+      "grammar_ms": 751,
+      "punctuation_raw": "شركة Google عملاق التقنية.",
+      "punctuation_ms": 623,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E021",
+      "dataset": "entities",
+      "category": "company",
+      "input_text": "شركة Microsoft تنتج البرمجيات",
+      "expected": "شركة Microsoft تنتج البرمجيات",
+      "severity": "major",
+      "spelling_raw": "شركة مايكروسوفت تنتج البرمجيات",
+      "spelling_ms": 1041,
+      "grammar_raw": "شركة Microsoft تنتج البرمجيات",
+      "grammar_ms": 700,
+      "punctuation_raw": "شركة Microsoft تنتج البرمجيات.",
+      "punctuation_ms": 812,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E022",
+      "dataset": "entities",
+      "category": "company",
+      "input_text": "منصة GitHub للمطورين",
+      "expected": "منصة GitHub للمطورين",
+      "severity": "major",
+      "spelling_raw": "منصة GitHub المطورين",
+      "spelling_ms": 1202,
+      "grammar_raw": "منصة GitHuب للمطورين",
+      "grammar_ms": 938,
+      "punctuation_raw": "منصة GitHub للمطورين.",
+      "punctuation_ms": 863,
+      "spelling_changed": true,
+      "grammar_changed": true,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E023",
+      "dataset": "entities",
+      "category": "company",
+      "input_text": "شركة Tesla للسيارات الكهربائية",
+      "expected": "شركة Tesla للسيارات الكهربائية",
+      "severity": "major",
+      "spelling_raw": "شركة تي اس لا للسيارات الكهربائية",
+      "spelling_ms": 960,
+      "grammar_raw": "شركة Teسla للسيارات الكهربائية",
+      "grammar_ms": 849,
+      "punctuation_raw": "شركة Tesla للسيارات الكهربائية.",
+      "punctuation_ms": 791,
+      "spelling_changed": true,
+      "grammar_changed": true,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E024",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "أستخدم Python في البرمجة",
+      "expected": "أستخدم Python في البرمجة",
+      "severity": "major",
+      "spelling_raw": "أستخدم Python في البرمجة",
+      "spelling_ms": 1042,
+      "grammar_raw": "أستخدمت Python في البرمجة",
+      "grammar_ms": 794,
+      "punctuation_raw": "أستخدم Python في البرمجة؟",
+      "punctuation_ms": 723,
+      "spelling_changed": false,
+      "grammar_changed": true,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E025",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "إطار TensorFlow مفيد للتعلم",
+      "expected": "إطار TensorFlow مفيد للتعلم",
+      "severity": "major",
+      "spelling_raw": "إطار TensorFlow مفيد للتعلم",
+      "spelling_ms": 1027,
+      "grammar_raw": "إطار TensorFlow مفيد للتعلم",
+      "grammar_ms": 989,
+      "punctuation_raw": "إطار TensorFlow، مفيد: للتعلم",
+      "punctuation_ms": 992,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E026",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "مكتبة PyTorch للتعلم العميق",
+      "expected": "مكتبة PyTorch للتعلم العميق",
+      "severity": "major",
+      "spelling_raw": "مكتبة PyTorch للتعلم العميق",
+      "spelling_ms": 1002,
+      "grammar_raw": "مكتبة PyTorch للتعلم العميق",
+      "grammar_ms": 891,
+      "punctuation_raw": "مكتبة PyTorch للتعلم العميق.",
+      "punctuation_ms": 1038,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E027",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "منصة Node.js للخوادم",
+      "expected": "منصة Node.js للخوادم",
+      "severity": "major",
+      "spelling_raw": "منصة Node. js للخوادم",
+      "spelling_ms": 1441,
+      "grammar_raw": "منصة Node ، jة للخوادم",
+      "grammar_ms": 1141,
+      "punctuation_raw": "منصة Node. js للخوادم:",
+      "punctuation_ms": 857,
+      "spelling_changed": true,
+      "grammar_changed": true,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E028",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "لغة JavaScript للويب",
+      "expected": "لغة JavaScript للويب",
+      "severity": "major",
+      "spelling_raw": "لغة JavaScript الويب",
+      "spelling_ms": 1346,
+      "grammar_raw": "لغة JavaScript للويب",
+      "grammar_ms": 813,
+      "punctuation_raw": "لغة JavaScript: للويب",
+      "punctuation_ms": 951,
+      "spelling_changed": true,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E029",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "قاعدة بيانات MongoDB جيدة",
+      "expected": "قاعدة بيانات MongoDB جيدة",
+      "severity": "major",
+      "spelling_raw": "قاعدة بيانات MongoDB جيدة",
+      "spelling_ms": 969,
+      "grammar_raw": "قاعدة بيانات MongoDB جيدة",
+      "grammar_ms": 903,
+      "punctuation_raw": "قاعدة بيانات MongoDB. جيدة",
+      "punctuation_ms": 862,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    },
+    {
+      "id": "E030",
+      "dataset": "entities",
+      "category": "tech",
+      "input_text": "خدمة Docker للحاويات",
+      "expected": "خدمة Docker للحاويات",
+      "severity": "major",
+      "spelling_raw": "خدمة Docker للحاويات",
+      "spelling_ms": 1411,
+      "grammar_raw": "خدمة Docker للحاويات",
+      "grammar_ms": 946,
+      "punctuation_raw": "خدمة Docker للحاويات.",
+      "punctuation_ms": 806,
+      "spelling_changed": false,
+      "grammar_changed": false,
+      "punctuation_changed": true
+    }
+  ]
+}

tests/v2/reports/level3_integrated_results.json

@@ -1,23 +1,23 @@
 {
-  "timestamp": "2026-06-23T18:54:52Z",
+  "timestamp": "2026-06-23T19:00:34Z",
   "target": "https://bayan10-bayan-api.hf.space",
   "analysis": {
     "total": 80,
     "aggregate": {
-      "TP": 22,
-      "TN": 0,
-      "FP": 58,
+      "TP": 49,
+      "TN": 17,
+      "FP": 14,
       "FN": 0,
-      "pass_rate": 0.275
+      "pass_rate": 0.825
     },
     "by_dataset": {
       "spelling": {
-        "TP": 22,
-        "TN": 0,
-        "FP": 58,
+        "TP": 49,
+        "TN": 17,
+        "FP": 14,
         "FN": 0,
         "total": 80,
-        "pass_rate": 0.275
+        "pass_rate": 0.825
       }
     }
   },
@@ -31,12 +31,12 @@
       "severity": "major",
       "pipeline_corrected": "أنا طالب في الجامعة.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 8033,
-      "spelling_ms": 3645,
-      "grammar_ms": 670,
-      "punctuation_ms": 2748,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2089,
+      "spelling_ms": 497,
+      "grammar_ms": 593,
+      "punctuation_ms": 412,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S002",
@@ -47,12 +47,12 @@
       "severity": "major",
       "pipeline_corrected": "إذا جاء الربيع تزهر الأشجار.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 5444,
-      "spelling_ms": 3501,
-      "grammar_ms": 586,
-      "punctuation_ms": 522,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 3020,
+      "spelling_ms": 786,
+      "grammar_ms": 594,
+      "punctuation_ms": 1421,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S003",
@@ -63,10 +63,10 @@
       "severity": "major",
       "pipeline_corrected": "ايضا هذا الأمر مهم جداً.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 5593,
-      "spelling_ms": 3361,
-      "grammar_ms": 654,
-      "punctuation_ms": 664,
+      "pipeline_ms": 2143,
+      "spelling_ms": 677,
+      "grammar_ms": 646,
+      "punctuation_ms": 605,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -79,12 +79,12 @@
       "severity": "major",
       "pipeline_corrected": "لأن الأمر يتعلق بالمستقبل.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 5150,
-      "spelling_ms": 3364,
-      "grammar_ms": 494,
-      "punctuation_ms": 569,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2039,
+      "spelling_ms": 661,
+      "grammar_ms": 497,
+      "punctuation_ms": 663,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S005",
@@ -95,12 +95,12 @@
       "severity": "major",
       "pipeline_corrected": "أين ذهبت أمس؟",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 3457,
-      "spelling_ms": 1889,
-      "grammar_ms": 436,
-      "punctuation_ms": 338,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1601,
+      "spelling_ms": 604,
+      "grammar_ms": 386,
+      "punctuation_ms": 394,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S006",
@@ -111,10 +111,10 @@
       "severity": "major",
       "pipeline_corrected": "اول مرة أزور هذا المكان.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 5318,
-      "spelling_ms": 3370,
-      "grammar_ms": 655,
-      "punctuation_ms": 558,
+      "pipeline_ms": 2175,
+      "spelling_ms": 796,
+      "grammar_ms": 547,
+      "punctuation_ms": 613,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -127,12 +127,12 @@
       "severity": "major",
       "pipeline_corrected": "هذا أو ذاك لا فرق.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 6027,
-      "spelling_ms": 4153,
-      "grammar_ms": 604,
-      "punctuation_ms": 701,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1792,
+      "spelling_ms": 637,
+      "grammar_ms": 536,
+      "punctuation_ms": 400,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S008",
@@ -143,12 +143,12 @@
       "severity": "major",
       "pipeline_corrected": "أكبر مدينة في العالم؟",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 8374,
-      "spelling_ms": 4040,
-      "grammar_ms": 610,
-      "punctuation_ms": 2543,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1641,
+      "spelling_ms": 464,
+      "grammar_ms": 493,
+      "punctuation_ms": 465,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S009",
@@ -159,12 +159,12 @@
       "severity": "major",
       "pipeline_corrected": "أصغر طالب في الصف؟",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4214,
-      "spelling_ms": 2207,
-      "grammar_ms": 595,
-      "punctuation_ms": 513,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1794,
+      "spelling_ms": 573,
+      "grammar_ms": 638,
+      "punctuation_ms": 364,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S010",
@@ -175,12 +175,12 @@
       "severity": "major",
       "pipeline_corrected": "أبناء الوطن يعملون بجد.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4294,
-      "spelling_ms": 2184,
-      "grammar_ms": 699,
-      "punctuation_ms": 620,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1696,
+      "spelling_ms": 536,
+      "grammar_ms": 531,
+      "punctuation_ms": 405,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S011",
@@ -191,12 +191,12 @@
       "severity": "major",
       "pipeline_corrected": "أطفال المدرسة يلعبون.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4345,
-      "spelling_ms": 2268,
-      "grammar_ms": 542,
-      "punctuation_ms": 567,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1582,
+      "spelling_ms": 481,
+      "grammar_ms": 477,
+      "punctuation_ms": 408,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S012",
@@ -207,12 +207,12 @@
       "severity": "major",
       "pipeline_corrected": "أخيراً وصلنا إلى الهدف.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 6255,
-      "spelling_ms": 4085,
-      "grammar_ms": 596,
-      "punctuation_ms": 729,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1954,
+      "spelling_ms": 660,
+      "grammar_ms": 535,
+      "punctuation_ms": 543,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S013",
@@ -223,12 +223,12 @@
       "severity": "major",
       "pipeline_corrected": "وقف أمام المدرسة.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2917,
-      "spelling_ms": 1380,
-      "grammar_ms": 441,
-      "punctuation_ms": 481,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1380,
+      "spelling_ms": 393,
+      "grammar_ms": 386,
+      "punctuation_ms": 386,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S014",
@@ -239,12 +239,12 @@
       "severity": "major",
       "pipeline_corrected": "أنت طالب مجتهد.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 3440,
-      "spelling_ms": 1755,
-      "grammar_ms": 535,
-      "punctuation_ms": 337,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1359,
+      "spelling_ms": 399,
+      "grammar_ms": 479,
+      "punctuation_ms": 260,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S015",
@@ -255,10 +255,10 @@
       "severity": "major",
       "pipeline_corrected": "اننا نحب الوطن.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 4129,
-      "spelling_ms": 2441,
-      "grammar_ms": 440,
-      "punctuation_ms": 463,
+      "pipeline_ms": 1566,
+      "spelling_ms": 540,
+      "grammar_ms": 433,
+      "punctuation_ms": 377,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -271,12 +271,12 @@
       "severity": "major",
       "pipeline_corrected": "إن شاء الله سنفوز.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 5342,
-      "spelling_ms": 3569,
-      "grammar_ms": 636,
-      "punctuation_ms": 533,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1891,
+      "spelling_ms": 670,
+      "grammar_ms": 529,
+      "punctuation_ms": 474,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S017",
@@ -287,12 +287,12 @@
       "severity": "none",
       "pipeline_corrected": "اقترب الموعد النهائي.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 4093,
-      "spelling_ms": 1957,
-      "grammar_ms": 754,
-      "punctuation_ms": 584,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 1622,
+      "spelling_ms": 483,
+      "grammar_ms": 478,
+      "punctuation_ms": 444,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S018",
@@ -303,12 +303,12 @@
       "severity": "major",
       "pipeline_corrected": "أرسل الرسالة فوراً.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4064,
-      "spelling_ms": 2103,
-      "grammar_ms": 651,
-      "punctuation_ms": 518,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1576,
+      "spelling_ms": 428,
+      "grammar_ms": 531,
+      "punctuation_ms": 395,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S019",
@@ -319,12 +319,12 @@
       "severity": "major",
       "pipeline_corrected": "آخر الأخبار في المساء.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4463,
-      "spelling_ms": 2236,
-      "grammar_ms": 549,
-      "punctuation_ms": 721,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1721,
+      "spelling_ms": 493,
+      "grammar_ms": 486,
+      "punctuation_ms": 526,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S020",
@@ -335,12 +335,12 @@
       "severity": "major",
       "pipeline_corrected": "أسرع في المشي قليلاً!",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 6099,
-      "spelling_ms": 3801,
-      "grammar_ms": 705,
-      "punctuation_ms": 588,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2011,
+      "spelling_ms": 554,
+      "grammar_ms": 634,
+      "punctuation_ms": 608,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S021",
@@ -351,12 +351,12 @@
       "severity": "major",
       "pipeline_corrected": "أجمل مكان في العالم؟",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4010,
-      "spelling_ms": 2218,
-      "grammar_ms": 540,
-      "punctuation_ms": 408,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1706,
+      "spelling_ms": 623,
+      "grammar_ms": 485,
+      "punctuation_ms": 378,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S022",
@@ -367,10 +367,10 @@
       "severity": "major",
       "pipeline_corrected": "اعلن الرئيس عن القرار.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 4267,
-      "spelling_ms": 2265,
-      "grammar_ms": 497,
-      "punctuation_ms": 598,
+      "pipeline_ms": 1783,
+      "spelling_ms": 648,
+      "grammar_ms": 478,
+      "punctuation_ms": 440,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -383,12 +383,12 @@
       "severity": "major",
       "pipeline_corrected": "أكد الوزير على الخطة.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4566,
-      "spelling_ms": 2472,
-      "grammar_ms": 554,
-      "punctuation_ms": 519,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1680,
+      "spelling_ms": 586,
+      "grammar_ms": 496,
+      "punctuation_ms": 382,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S024",
@@ -399,12 +399,12 @@
       "severity": "major",
       "pipeline_corrected": "أشار التقرير إلى ذلك.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4563,
-      "spelling_ms": 2521,
-      "grammar_ms": 590,
-      "punctuation_ms": 568,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1766,
+      "spelling_ms": 689,
+      "grammar_ms": 487,
+      "punctuation_ms": 370,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S025",
@@ -415,12 +415,12 @@
       "severity": "major",
       "pipeline_corrected": "أعتقد انه سيحضر غداً.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4144,
-      "spelling_ms": 2119,
-      "grammar_ms": 652,
-      "punctuation_ms": 578,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1719,
+      "spelling_ms": 561,
+      "grammar_ms": 530,
+      "punctuation_ms": 413,
+      "verdict": "TP",
+      "detail": "Partial improvement"
     },
     {
       "id": "S026",
@@ -431,12 +431,12 @@
       "severity": "major",
       "pipeline_corrected": "والأسعار مرتفعة جداً.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4880,
-      "spelling_ms": 2705,
-      "grammar_ms": 593,
-      "punctuation_ms": 585,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1905,
+      "spelling_ms": 589,
+      "grammar_ms": 592,
+      "punctuation_ms": 506,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S027",
@@ -447,12 +447,12 @@
       "severity": "major",
       "pipeline_corrected": "بالإضافة إلى ذلك.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4101,
-      "spelling_ms": 2414,
-      "grammar_ms": 438,
-      "punctuation_ms": 442,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 1555,
+      "spelling_ms": 538,
+      "grammar_ms": 429,
+      "punctuation_ms": 371,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S028",
@@ -463,10 +463,10 @@
       "severity": "major",
       "pipeline_corrected": "فالانسان يحتاج للعلم.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 5749,
-      "spelling_ms": 3787,
-      "grammar_ms": 484,
-      "punctuation_ms": 692,
+      "pipeline_ms": 1849,
+      "spelling_ms": 576,
+      "grammar_ms": 483,
+      "punctuation_ms": 575,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -479,10 +479,10 @@
       "severity": "major",
       "pipeline_corrected": "كلا الأطفال في اللعب.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 6739,
-      "spelling_ms": 4640,
-      "grammar_ms": 701,
-      "punctuation_ms": 540,
+      "pipeline_ms": 2995,
+      "spelling_ms": 1635,
+      "grammar_ms": 636,
+      "punctuation_ms": 413,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -495,12 +495,12 @@
       "severity": "major",
       "pipeline_corrected": "للأسف لم ينجح.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 4198,
-      "spelling_ms": 2413,
+      "pipeline_ms": 1590,
+      "spelling_ms": 498,
       "grammar_ms": 437,
-      "punctuation_ms": 476,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "punctuation_ms": 437,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S031",
@@ -511,12 +511,12 @@
       "severity": "major",
       "pipeline_corrected": "المدرسة كبيرة وجميلة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 3680,
-      "spelling_ms": 1927,
-      "grammar_ms": 494,
-      "punctuation_ms": 654,
+      "pipeline_ms": 2278,
+      "spelling_ms": 832,
+      "grammar_ms": 498,
+      "punctuation_ms": 730,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S032",
@@ -527,12 +527,12 @@
       "severity": "major",
       "pipeline_corrected": "الجامعة في القاهرة.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 6067,
-      "spelling_ms": 4080,
-      "grammar_ms": 537,
-      "punctuation_ms": 527,
+      "pipeline_ms": 1974,
+      "spelling_ms": 741,
+      "grammar_ms": 492,
+      "punctuation_ms": 520,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S033",
@@ -543,12 +543,12 @@
       "severity": "major",
       "pipeline_corrected": "السيارة سريعة جداً.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 4645,
-      "spelling_ms": 2783,
-      "grammar_ms": 595,
-      "punctuation_ms": 423,
+      "pipeline_ms": 1858,
+      "spelling_ms": 688,
+      "grammar_ms": 535,
+      "punctuation_ms": 418,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S034",
@@ -559,12 +559,12 @@
       "severity": "major",
       "pipeline_corrected": "الشجرة طويلة وجميلة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 3937,
-      "spelling_ms": 2031,
-      "grammar_ms": 496,
+      "pipeline_ms": 2099,
+      "spelling_ms": 865,
+      "grammar_ms": 495,
       "punctuation_ms": 519,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S035",
@@ -575,12 +575,12 @@
       "severity": "major",
       "pipeline_corrected": "الحياة صعبة في المدينة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 4159,
-      "spelling_ms": 2357,
-      "grammar_ms": 535,
-      "punctuation_ms": 577,
+      "pipeline_ms": 2175,
+      "spelling_ms": 742,
+      "grammar_ms": 594,
+      "punctuation_ms": 622,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S036",
@@ -591,12 +591,12 @@
       "severity": "major",
       "pipeline_corrected": "المكتبة قريبة من البيت.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 5008,
-      "spelling_ms": 3269,
-      "grammar_ms": 597,
-      "punctuation_ms": 503,
+      "pipeline_ms": 2074,
+      "spelling_ms": 701,
+      "grammar_ms": 536,
+      "punctuation_ms": 573,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S037",
@@ -607,12 +607,12 @@
       "severity": "major",
       "pipeline_corrected": "الغرفة نظيفة ومرتبة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 2678,
-      "spelling_ms": 811,
-      "grammar_ms": 653,
-      "punctuation_ms": 459,
+      "pipeline_ms": 4644,
+      "spelling_ms": 3274,
+      "grammar_ms": 610,
+      "punctuation_ms": 540,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S038",
@@ -623,12 +623,12 @@
       "severity": "major",
       "pipeline_corrected": "القصة مثيرة للاهتمام!",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 2345,
-      "spelling_ms": 526,
-      "grammar_ms": 536,
-      "punctuation_ms": 599,
+      "pipeline_ms": 3675,
+      "spelling_ms": 2465,
+      "grammar_ms": 483,
+      "punctuation_ms": 511,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S039",
@@ -639,12 +639,12 @@
       "severity": "major",
       "pipeline_corrected": "الرحلة طويلة ومتعبة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 2842,
-      "spelling_ms": 993,
-      "grammar_ms": 615,
-      "punctuation_ms": 478,
+      "pipeline_ms": 3737,
+      "spelling_ms": 2430,
+      "grammar_ms": 599,
+      "punctuation_ms": 488,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S040",
@@ -655,12 +655,12 @@
       "severity": "major",
       "pipeline_corrected": "الوظيفة صعبة لكنها مفيدة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 2828,
-      "spelling_ms": 824,
-      "grammar_ms": 496,
-      "punctuation_ms": 759,
+      "pipeline_ms": 4159,
+      "spelling_ms": 2751,
+      "grammar_ms": 491,
+      "punctuation_ms": 695,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S041",
@@ -671,10 +671,10 @@
       "severity": "major",
       "pipeline_corrected": "بالمدرسة الكبيرة",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2387,
-      "spelling_ms": 782,
-      "grammar_ms": 348,
-      "punctuation_ms": 477,
+      "pipeline_ms": 4988,
+      "spelling_ms": 4001,
+      "grammar_ms": 327,
+      "punctuation_ms": 439,
       "verdict": "TP",
       "detail": "Exact match"
     },
@@ -687,10 +687,10 @@
       "severity": "major",
       "pipeline_corrected": "والجامعة المصرية",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2143,
-      "spelling_ms": 481,
-      "grammar_ms": 441,
-      "punctuation_ms": 415,
+      "pipeline_ms": 3656,
+      "spelling_ms": 2532,
+      "grammar_ms": 392,
+      "punctuation_ms": 500,
       "verdict": "TP",
       "detail": "Exact match"
     },
@@ -703,10 +703,10 @@
       "severity": "major",
       "pipeline_corrected": "السيارة الجديدة",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2365,
-      "spelling_ms": 764,
-      "grammar_ms": 336,
-      "punctuation_ms": 402,
+      "pipeline_ms": 4550,
+      "spelling_ms": 3621,
+      "grammar_ms": 333,
+      "punctuation_ms": 375,
       "verdict": "TP",
       "detail": "Partial improvement"
     },
@@ -719,10 +719,10 @@
       "severity": "major",
       "pipeline_corrected": "كالمدرسه قديمة",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2443,
-      "spelling_ms": 706,
-      "grammar_ms": 434,
-      "punctuation_ms": 407,
+      "pipeline_ms": 4082,
+      "spelling_ms": 3050,
+      "grammar_ms": 432,
+      "punctuation_ms": 381,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -735,10 +735,10 @@
       "severity": "major",
       "pipeline_corrected": "للمدرسة الابتدائية",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2449,
-      "spelling_ms": 770,
-      "grammar_ms": 332,
-      "punctuation_ms": 661,
+      "pipeline_ms": 4153,
+      "spelling_ms": 2978,
+      "grammar_ms": 327,
+      "punctuation_ms": 594,
       "verdict": "TP",
       "detail": "Exact match"
     },
@@ -751,12 +751,12 @@
       "severity": "major",
       "pipeline_corrected": "ذهبت إلى المكتبة؟",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2163,
-      "spelling_ms": 492,
-      "grammar_ms": 435,
-      "punctuation_ms": 434,
+      "pipeline_ms": 2945,
+      "spelling_ms": 1796,
+      "grammar_ms": 443,
+      "punctuation_ms": 487,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S047",
@@ -767,10 +767,10 @@
       "severity": "major",
       "pipeline_corrected": "المستشفى الكبير",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 1927,
-      "spelling_ms": 514,
-      "grammar_ms": 336,
-      "punctuation_ms": 436,
+      "pipeline_ms": 2470,
+      "spelling_ms": 1529,
+      "grammar_ms": 326,
+      "punctuation_ms": 401,
       "verdict": "TP",
       "detail": "Exact match"
     },
@@ -783,10 +783,10 @@
       "severity": "major",
       "pipeline_corrected": "هدى الطالبة ممتازة",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2224,
-      "spelling_ms": 443,
-      "grammar_ms": 545,
-      "punctuation_ms": 436,
+      "pipeline_ms": 2781,
+      "spelling_ms": 1610,
+      "grammar_ms": 519,
+      "punctuation_ms": 431,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -799,12 +799,12 @@
       "severity": "major",
       "pipeline_corrected": "مصطفى طالب مجتهد.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2224,
-      "spelling_ms": 589,
-      "grammar_ms": 534,
-      "punctuation_ms": 320,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2432,
+      "spelling_ms": 1465,
+      "grammar_ms": 482,
+      "punctuation_ms": 267,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S050",
@@ -815,12 +815,12 @@
       "severity": "major",
       "pipeline_corrected": "موسى نبي عظيم.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 1946,
-      "spelling_ms": 521,
-      "grammar_ms": 438,
-      "punctuation_ms": 318,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2483,
+      "spelling_ms": 1626,
+      "grammar_ms": 376,
+      "punctuation_ms": 267,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S051",
@@ -831,12 +831,12 @@
       "severity": "none",
       "pipeline_corrected": "علي يدرس في الكلية.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2092,
-      "spelling_ms": 516,
-      "grammar_ms": 540,
-      "punctuation_ms": 455,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 2644,
+      "spelling_ms": 1528,
+      "grammar_ms": 490,
+      "punctuation_ms": 409,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S052",
@@ -847,12 +847,12 @@
       "severity": "major",
       "pipeline_corrected": "ذهب إلى السوق.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2033,
-      "spelling_ms": 604,
-      "grammar_ms": 408,
-      "punctuation_ms": 393,
+      "pipeline_ms": 2478,
+      "spelling_ms": 1516,
+      "grammar_ms": 437,
+      "punctuation_ms": 306,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S053",
@@ -863,12 +863,12 @@
       "severity": "major",
       "pipeline_corrected": "بنى المبنى الجديد.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2459,
-      "spelling_ms": 635,
-      "grammar_ms": 436,
-      "punctuation_ms": 513,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2441,
+      "spelling_ms": 1448,
+      "grammar_ms": 381,
+      "punctuation_ms": 395,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S054",
@@ -879,10 +879,10 @@
       "severity": "major",
       "pipeline_corrected": "ذهبت في البيت",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2495,
-      "spelling_ms": 836,
-      "grammar_ms": 424,
-      "punctuation_ms": 490,
+      "pipeline_ms": 2773,
+      "spelling_ms": 1751,
+      "grammar_ms": 391,
+      "punctuation_ms": 415,
       "verdict": "TP",
       "detail": "Exact match"
     },
@@ -895,10 +895,10 @@
       "severity": "major",
       "pipeline_corrected": "خرج منالمدرسة.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2287,
-      "spelling_ms": 611,
-      "grammar_ms": 395,
-      "punctuation_ms": 552,
+      "pipeline_ms": 3061,
+      "spelling_ms": 2075,
+      "grammar_ms": 386,
+      "punctuation_ms": 384,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -911,10 +911,10 @@
       "severity": "major",
       "pipeline_corrected": "بقي عندالباب.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 7894,
-      "spelling_ms": 682,
-      "grammar_ms": 468,
-      "punctuation_ms": 408,
+      "pipeline_ms": 2616,
+      "spelling_ms": 1640,
+      "grammar_ms": 490,
+      "punctuation_ms": 269,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -927,10 +927,10 @@
       "severity": "major",
       "pipeline_corrected": "جلس عندالنافذة.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2314,
-      "spelling_ms": 540,
-      "grammar_ms": 488,
-      "punctuation_ms": 435,
+      "pipeline_ms": 2763,
+      "spelling_ms": 1517,
+      "grammar_ms": 585,
+      "punctuation_ms": 445,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -943,10 +943,10 @@
       "severity": "major",
       "pipeline_corrected": "رجع الىالبيت.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2280,
-      "spelling_ms": 659,
-      "grammar_ms": 437,
-      "punctuation_ms": 380,
+      "pipeline_ms": 3082,
+      "spelling_ms": 2096,
+      "grammar_ms": 431,
+      "punctuation_ms": 338,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -959,12 +959,12 @@
       "severity": "major",
       "pipeline_corrected": "خرج من الباب الخلفي.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 5938,
-      "spelling_ms": 512,
-      "grammar_ms": 487,
-      "punctuation_ms": 554,
-      "verdict": "FP",
-      "detail": "Wrong correction"
+      "pipeline_ms": 2927,
+      "spelling_ms": 1719,
+      "grammar_ms": 497,
+      "punctuation_ms": 494,
+      "verdict": "TP",
+      "detail": "Exact match"
     },
     {
       "id": "S060",
@@ -975,10 +975,10 @@
       "severity": "major",
       "pipeline_corrected": "نظر الىالسماء.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2092,
-      "spelling_ms": 653,
-      "grammar_ms": 484,
-      "punctuation_ms": 327,
+      "pipeline_ms": 2998,
+      "spelling_ms": 1978,
+      "grammar_ms": 479,
+      "punctuation_ms": 323,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -991,12 +991,12 @@
       "severity": "none",
       "pipeline_corrected": "أنا ذهبت إلى الجامعة.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2067,
-      "spelling_ms": 563,
-      "grammar_ms": 487,
-      "punctuation_ms": 449,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3099,
+      "spelling_ms": 1856,
+      "grammar_ms": 489,
+      "punctuation_ms": 536,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S062",
@@ -1007,12 +1007,12 @@
       "severity": "none",
       "pipeline_corrected": "هذه المدرسة جميلة جداً.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 4721,
-      "spelling_ms": 500,
-      "grammar_ms": 694,
-      "punctuation_ms": 532,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 4323,
+      "spelling_ms": 2252,
+      "grammar_ms": 642,
+      "punctuation_ms": 464,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S063",
@@ -1023,12 +1023,12 @@
       "severity": "none",
       "pipeline_corrected": "كان الجو ممطراً اليوم؟",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2771,
-      "spelling_ms": 823,
-      "grammar_ms": 738,
-      "punctuation_ms": 585,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 4450,
+      "spelling_ms": 2278,
+      "grammar_ms": 729,
+      "punctuation_ms": 497,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S064",
@@ -1039,12 +1039,12 @@
       "severity": "none",
       "pipeline_corrected": "وكان أحمد في المنزل.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2497,
-      "spelling_ms": 614,
-      "grammar_ms": 499,
-      "punctuation_ms": 659,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3157,
+      "spelling_ms": 1560,
+      "grammar_ms": 485,
+      "punctuation_ms": 454,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S065",
@@ -1055,12 +1055,12 @@
       "severity": "none",
       "pipeline_corrected": "إلى اللقاء يا صديقي.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 6038,
-      "spelling_ms": 665,
-      "grammar_ms": 484,
-      "punctuation_ms": 440,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3780,
+      "spelling_ms": 2220,
+      "grammar_ms": 480,
+      "punctuation_ms": 367,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S066",
@@ -1071,12 +1071,12 @@
       "severity": "none",
       "pipeline_corrected": "ذلك الكتاب مفيد جداً.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2372,
-      "spelling_ms": 502,
-      "grammar_ms": 589,
-      "punctuation_ms": 411,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3597,
+      "spelling_ms": 1896,
+      "grammar_ms": 588,
+      "punctuation_ms": 374,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S067",
@@ -1087,12 +1087,12 @@
       "severity": "none",
       "pipeline_corrected": "لكن الأمر صعب علينا.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2401,
-      "spelling_ms": 617,
-      "grammar_ms": 485,
-      "punctuation_ms": 424,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3659,
+      "spelling_ms": 2195,
+      "grammar_ms": 492,
+      "punctuation_ms": 447,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S068",
@@ -1103,12 +1103,12 @@
       "severity": "none",
       "pipeline_corrected": "هذا أو ذاك سواء عندي.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 6476,
-      "spelling_ms": 636,
-      "grammar_ms": 683,
-      "punctuation_ms": 435,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 4575,
+      "spelling_ms": 2778,
+      "grammar_ms": 578,
+      "punctuation_ms": 426,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S069",
@@ -1119,12 +1119,12 @@
       "severity": "none",
       "pipeline_corrected": "الطالب المجتهد ينجح دائماً.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2658,
-      "spelling_ms": 548,
-      "grammar_ms": 645,
-      "punctuation_ms": 579,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3740,
+      "spelling_ms": 1907,
+      "grammar_ms": 586,
+      "punctuation_ms": 544,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S070",
@@ -1135,12 +1135,12 @@
       "severity": "none",
       "pipeline_corrected": "العلم نور والجهل ظلام.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2694,
-      "spelling_ms": 608,
-      "grammar_ms": 726,
-      "punctuation_ms": 582,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 4941,
+      "spelling_ms": 3158,
+      "grammar_ms": 694,
+      "punctuation_ms": 511,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S071",
@@ -1151,12 +1151,12 @@
       "severity": "none",
       "pipeline_corrected": "أحب القراءة والكتابة.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 5967,
-      "spelling_ms": 627,
-      "grammar_ms": 437,
-      "punctuation_ms": 485,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3160,
+      "spelling_ms": 1507,
+      "grammar_ms": 443,
+      "punctuation_ms": 523,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S072",
@@ -1167,12 +1167,12 @@
       "severity": "none",
       "pipeline_corrected": "المعلم يشرح الدرس.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2178,
-      "spelling_ms": 549,
-      "grammar_ms": 431,
-      "punctuation_ms": 466,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3649,
+      "spelling_ms": 1720,
+      "grammar_ms": 430,
+      "punctuation_ms": 492,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S073",
@@ -1183,12 +1183,12 @@
       "severity": "none",
       "pipeline_corrected": "ذهبت إلى السوق واشتريت خبزاً.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 3214,
-      "spelling_ms": 720,
-      "grammar_ms": 837,
-      "punctuation_ms": 828,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 6100,
+      "spelling_ms": 3889,
+      "grammar_ms": 810,
+      "punctuation_ms": 758,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S074",
@@ -1199,12 +1199,12 @@
       "severity": "none",
       "pipeline_corrected": "الطقس جميل في الربيع.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 5596,
-      "spelling_ms": 497,
-      "grammar_ms": 637,
-      "punctuation_ms": 424,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 3330,
+      "spelling_ms": 1617,
+      "grammar_ms": 532,
+      "punctuation_ms": 417,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S075",
@@ -1215,12 +1215,12 @@
       "severity": "none",
       "pipeline_corrected": "نحن نعمل بجد كل يوم.",
       "pipeline_suggestions": 1,
-      "pipeline_ms": 2688,
-      "spelling_ms": 619,
-      "grammar_ms": 785,
-      "punctuation_ms": 360,
-      "verdict": "FP",
-      "detail": "Modified correct text"
+      "pipeline_ms": 4609,
+      "spelling_ms": 2915,
+      "grammar_ms": 591,
+      "punctuation_ms": 406,
+      "verdict": "TN",
+      "detail": "Only punctuation added"
     },
     {
       "id": "S076",
@@ -1231,12 +1231,12 @@
       "severity": "critical",
       "pipeline_corrected": "أنا ذهبت إلى المدرسة.",
       "pipeline_suggestions": 3,
-      "pipeline_ms": 2497,
-      "spelling_ms": 630,
-      "grammar_ms": 485,
-      "punctuation_ms": 442,
+      "pipeline_ms": 5931,
+      "spelling_ms": 4275,
+      "grammar_ms": 536,
+      "punctuation_ms": 580,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S077",
@@ -1247,12 +1247,12 @@
       "severity": "critical",
       "pipeline_corrected": "أيضا الجامعة كبيرة.",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 5834,
-      "spelling_ms": 557,
-      "grammar_ms": 582,
-      "punctuation_ms": 406,
+      "pipeline_ms": 4776,
+      "spelling_ms": 2819,
+      "grammar_ms": 595,
+      "punctuation_ms": 482,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S078",
@@ -1263,12 +1263,12 @@
       "severity": "critical",
       "pipeline_corrected": "لأن المدرسة بعيدة جداً.",
       "pipeline_suggestions": 4,
-      "pipeline_ms": 2671,
-      "spelling_ms": 669,
-      "grammar_ms": 648,
-      "punctuation_ms": 531,
+      "pipeline_ms": 4641,
+      "spelling_ms": 2773,
+      "grammar_ms": 642,
+      "punctuation_ms": 540,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     },
     {
       "id": "S079",
@@ -1279,10 +1279,10 @@
       "severity": "critical",
       "pipeline_corrected": "اين أن الجامعة الكبيرة",
       "pipeline_suggestions": 2,
-      "pipeline_ms": 2619,
-      "spelling_ms": 710,
-      "grammar_ms": 493,
-      "punctuation_ms": 456,
+      "pipeline_ms": 2614,
+      "spelling_ms": 696,
+      "grammar_ms": 496,
+      "punctuation_ms": 510,
       "verdict": "FP",
       "detail": "Wrong correction"
     },
@@ -1295,12 +1295,12 @@
       "severity": "critical",
       "pipeline_corrected": "أول مرة أزور المكتبة.",
       "pipeline_suggestions": 4,
-      "pipeline_ms": 5759,
-      "spelling_ms": 799,
-      "grammar_ms": 513,
-      "punctuation_ms": 483,
+      "pipeline_ms": 2637,
+      "spelling_ms": 808,
+      "grammar_ms": 496,
+      "punctuation_ms": 522,
       "verdict": "TP",
-      "detail": "Partial improvement"
+      "detail": "Exact match"
     }
   ]
 }

tests/v2/test_level1_raw.py

@@ -1,8 +1,18 @@
 """
 BAYAN v2.0 — Level 1: Raw Model Tests
 ======================================
-Tests each ML model DIRECTLY — no filters, no pipeline, no StageLocker.
-Measures the raw model ceiling: what's the best each model can do?
+Tests each ML model through its individual API endpoint — measuring the RAW
+model output with no pipeline integration (no StageLocker, no OffsetMapper,
+no cross-stage interaction).
+
+NOTE: On HF Spaces deployment, the solo endpoints (/api/spelling, /api/grammar,
+/api/punctuation) call the models directly with minimal preprocessing. This is
+as close to "raw model" as we can get without local model access.
+
+Produces:
+  - TP/FP/FN/TN verdicts per model per test case
+  - Per-word edit analysis (what words each model changed)
+  - Change rate and accuracy per model per dataset
 
 Usage:
     python tests/v2/test_level1_raw.py --url URL [--dataset DATASET]
@@ -19,9 +29,11 @@ import requests
 
 # Datasets
 DATASETS_DIR = Path(__file__).parent.parent / "phase10" / "gold_datasets"
-
 REPORT_DIR = Path(__file__).parent / "reports"
 
+# Terminal punctuation to strip before comparison
+_TERMINAL_PUNCT = '.،؛؟!?!'
+
 
 @dataclass
 class RawModelResult:
@@ -38,10 +50,18 @@ class RawModelResult:
     grammar_ms: int = 0
     punctuation_raw: str = ""
     punctuation_ms: int = 0
-    # Analysis
+    # Change detection
     spelling_changed: bool = False
     grammar_changed: bool = False
     punctuation_changed: bool = False
+    # Verdicts (TP/TN/FP/FN)
+    spelling_verdict: str = ""
+    grammar_verdict: str = ""
+    punctuation_verdict: str = ""
+    # Word-level edits
+    spelling_edits: str = ""   # "word1→word2, word3→word4"
+    grammar_edits: str = ""
+    punctuation_edits: str = ""
 
 
 class APIClient:
@@ -67,49 +87,114 @@ class APIClient:
             return {"error": str(e)}, ms
 
     def spelling_raw(self, text):
-        """Call /api/spelling — raw spelling model through API."""
         data, ms = self._post("/api/spelling", text)
         corrected = data.get("corrected_text", data.get("corrected", text))
         return corrected, ms
 
     def grammar_raw(self, text):
-        """Call /api/grammar — raw grammar model through API."""
         data, ms = self._post("/api/grammar", text)
         corrected = data.get("corrected_text", data.get("corrected", text))
         return corrected, ms
 
     def punctuation_raw(self, text):
-        """Call /api/punctuation — raw punctuation model through API."""
         data, ms = self._post("/api/punctuation", text)
         corrected = data.get("corrected_text", data.get("corrected", text))
         return corrected, ms
 
 
 def load_datasets(dataset_filter=None):
-    """Load all gold datasets."""
     datasets = {}
     for f in sorted(DATASETS_DIR.glob("*.json")):
         name = f.stem
         if dataset_filter and name != dataset_filter:
             continue
         with open(f, 'r', encoding='utf-8') as fh:
-            data = json.load(fh)
-            datasets[name] = data
+            datasets[name] = json.load(fh)
     return datasets
 
 
-def strip_diacritics(text):
-    return re.sub(r'[\u064B-\u065F\u0670]', '', text)
-
-
 def normalize(text):
-    t = strip_diacritics(text)
+    t = re.sub(r'[\u064B-\u065F\u0670]', '', text)
     t = re.sub(r'\s+', ' ', t).strip()
     return t
 
 
+def _edit_distance(a, b):
+    if len(a) < len(b):
+        return _edit_distance(b, a)
+    if len(b) == 0:
+        return len(a)
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a):
+        curr = [i + 1]
+        for j, cb in enumerate(b):
+            cost = 0 if ca == cb else 1
+            curr.append(min(curr[j] + 1, prev[j + 1] + 1, prev[j] + cost))
+        prev = curr
+    return prev[len(b)]
+
+
+def get_word_edits(input_text, output_text):
+    """Get word-level edits between input and output."""
+    inp_words = normalize(input_text).split()
+    out_words = normalize(output_text).split()
+    edits = []
+
+    # Simple alignment: match by position
+    max_len = max(len(inp_words), len(out_words))
+    for i in range(max_len):
+        iw = inp_words[i] if i < len(inp_words) else "∅"
+        ow = out_words[i] if i < len(out_words) else "∅"
+        if iw != ow:
+            edits.append(f"{iw}→{ow}")
+
+    return ", ".join(edits[:5])  # Cap at 5 edits for readability
+
+
+def classify_raw(input_text, output_text, expected_text, dataset):
+    """Classify raw model output as TP/TN/FP/FN.
+
+    Same logic as L2/L3 but applied to raw model output.
+    """
+    inp_n = normalize(input_text)
+    out_n = normalize(output_text)
+    exp_n = normalize(expected_text)
+
+    out_stripped = out_n.rstrip(_TERMINAL_PUNCT).rstrip()
+    text_changed = (out_n != inp_n)
+
+    is_preservation = dataset in ('entities', 'religious', 'structured', 'hallucination')
+
+    if is_preservation:
+        if not text_changed:
+            return "TN"
+        elif out_stripped == inp_n:
+            return "TN"  # Only punctuation added — not entity corruption
+        else:
+            return "FP"
+    else:
+        needs_correction = (inp_n != exp_n)
+        if needs_correction:
+            if out_n == exp_n or out_stripped == exp_n:
+                return "TP"
+            elif text_changed and _edit_distance(out_stripped, exp_n) < _edit_distance(inp_n, exp_n):
+                return "TP"
+            elif text_changed and _edit_distance(out_n, exp_n) < _edit_distance(inp_n, exp_n):
+                return "TP"
+            elif not text_changed:
+                return "FN"
+            else:
+                return "FP"
+        else:
+            if not text_changed:
+                return "TN"
+            elif out_stripped == inp_n:
+                return "TN"  # Only punctuation added
+            else:
+                return "FP"
+
+
 def run_level1(api: APIClient, datasets: dict) -> List[RawModelResult]:
-    """Run each test case through all 3 raw models."""
     results = []
     total = sum(len(v) for v in datasets.values())
     idx = 0
@@ -138,30 +223,37 @@ def run_level1(api: APIClient, datasets: dict) -> List[RawModelResult]:
             try:
                 r.spelling_raw, r.spelling_ms = api.spelling_raw(inp)
                 r.spelling_changed = (normalize(r.spelling_raw) != normalize(inp))
+                r.spelling_verdict = classify_raw(inp, r.spelling_raw, expected, ds_name)
+                if r.spelling_changed:
+                    r.spelling_edits = get_word_edits(inp, r.spelling_raw)
             except Exception as e:
                 r.spelling_raw = f"ERROR: {e}"
+                r.spelling_verdict = "ERR"
 
             # ── Grammar ──
             try:
                 r.grammar_raw, r.grammar_ms = api.grammar_raw(inp)
                 r.grammar_changed = (normalize(r.grammar_raw) != normalize(inp))
+                r.grammar_verdict = classify_raw(inp, r.grammar_raw, expected, ds_name)
+                if r.grammar_changed:
+                    r.grammar_edits = get_word_edits(inp, r.grammar_raw)
             except Exception as e:
                 r.grammar_raw = f"ERROR: {e}"
+                r.grammar_verdict = "ERR"
 
             # ── Punctuation ──
             try:
                 r.punctuation_raw, r.punctuation_ms = api.punctuation_raw(inp)
                 r.punctuation_changed = (normalize(r.punctuation_raw) != normalize(inp))
+                r.punctuation_verdict = classify_raw(inp, r.punctuation_raw, expected, ds_name)
+                if r.punctuation_changed:
+                    r.punctuation_edits = get_word_edits(inp, r.punctuation_raw)
             except Exception as e:
                 r.punctuation_raw = f"ERROR: {e}"
+                r.punctuation_verdict = "ERR"
 
             total_ms = r.spelling_ms + r.grammar_ms + r.punctuation_ms
-            changes = []
-            if r.spelling_changed: changes.append("S")
-            if r.grammar_changed: changes.append("G")
-            if r.punctuation_changed: changes.append("P")
-            change_str = "+".join(changes) if changes else "none"
-            print(f"[{change_str}] ({total_ms}ms)")
+            print(f"S={r.spelling_verdict} G={r.grammar_verdict} P={r.punctuation_verdict} ({total_ms}ms)")
 
             results.append(r)
 
@@ -169,74 +261,100 @@ def run_level1(api: APIClient, datasets: dict) -> List[RawModelResult]:
 
 
 def analyze_results(results: List[RawModelResult]) -> dict:
-    """Produce per-model analysis of raw outputs."""
     analysis = {
         "total": len(results),
-        "by_model": {
-            "spelling": {"changed": 0, "unchanged": 0, "errors": 0},
-            "grammar": {"changed": 0, "unchanged": 0, "errors": 0},
-            "punctuation": {"changed": 0, "unchanged": 0, "errors": 0},
-        },
+        "by_model": {},
         "by_dataset": {},
     }
 
-    for r in results:
-        ds = r.dataset
-        if ds not in analysis["by_dataset"]:
-            analysis["by_dataset"][ds] = {
-                "total": 0,
-                "spelling_changed": 0,
-                "grammar_changed": 0,
-                "punctuation_changed": 0,
+    for model in ("spelling", "grammar", "punctuation"):
+        verdicts = {"TP": 0, "TN": 0, "FP": 0, "FN": 0, "ERR": 0}
+        changed = 0
+        for r in results:
+            v = getattr(r, f"{model}_verdict", "")
+            if v in verdicts:
+                verdicts[v] += 1
+            if getattr(r, f"{model}_changed", False):
+                changed += 1
+        total = sum(v for k, v in verdicts.items() if k != "ERR")
+        pass_count = verdicts["TP"] + verdicts["TN"]
+        analysis["by_model"][model] = {
+            **verdicts,
+            "changed": changed,
+            "total": len(results),
+            "change_rate": round(changed / len(results), 4) if results else 0,
+            "pass_rate": round(pass_count / total, 4) if total else 0,
+        }
+
+    # Per-dataset breakdown
+    for ds in sorted(set(r.dataset for r in results)):
+        ds_results = [r for r in results if r.dataset == ds]
+        ds_analysis = {"total": len(ds_results)}
+        for model in ("spelling", "grammar", "punctuation"):
+            verdicts = {"TP": 0, "TN": 0, "FP": 0, "FN": 0}
+            changed = 0
+            for r in ds_results:
+                v = getattr(r, f"{model}_verdict", "")
+                if v in verdicts:
+                    verdicts[v] += 1
+                if getattr(r, f"{model}_changed", False):
+                    changed += 1
+            total = sum(verdicts.values())
+            pass_count = verdicts["TP"] + verdicts["TN"]
+            ds_analysis[model] = {
+                **verdicts,
+                "changed": changed,
+                "change_rate": round(changed / len(ds_results), 4) if ds_results else 0,
+                "pass_rate": round(pass_count / total, 4) if total else 0,
             }
-        analysis["by_dataset"][ds]["total"] += 1
-
-        # Spelling
-        if "ERROR" in r.spelling_raw:
-            analysis["by_model"]["spelling"]["errors"] += 1
-        elif r.spelling_changed:
-            analysis["by_model"]["spelling"]["changed"] += 1
-            analysis["by_dataset"][ds]["spelling_changed"] += 1
-        else:
-            analysis["by_model"]["spelling"]["unchanged"] += 1
-
-        # Grammar
-        if "ERROR" in r.grammar_raw:
-            analysis["by_model"]["grammar"]["errors"] += 1
-        elif r.grammar_changed:
-            analysis["by_model"]["grammar"]["changed"] += 1
-            analysis["by_dataset"][ds]["grammar_changed"] += 1
-        else:
-            analysis["by_model"]["grammar"]["unchanged"] += 1
-
-        # Punctuation
-        if "ERROR" in r.punctuation_raw:
-            analysis["by_model"]["punctuation"]["errors"] += 1
-        elif r.punctuation_changed:
-            analysis["by_model"]["punctuation"]["changed"] += 1
-            analysis["by_dataset"][ds]["punctuation_changed"] += 1
-        else:
-            analysis["by_model"]["punctuation"]["unchanged"] += 1
+        analysis["by_dataset"][ds] = ds_analysis
 
     return analysis
 
 
-def print_analysis(analysis: dict):
+def print_analysis(analysis: dict, results: List[RawModelResult]):
     print(f"\n{'='*60}")
     print("LEVEL 1: RAW MODEL ANALYSIS")
     print(f"{'='*60}")
 
+    # Per-model summary
     print(f"\n## Per-Model Summary ({analysis['total']} tests)")
-    print(f"| Model       | Changed | Unchanged | Errors |")
-    print(f"|-------------|---------|-----------|--------|")
+    print(f"| Model       | Changed | TP  | TN  | FP  | FN  | ChgRate | Pass%  |")
+    print(f"|-------------|---------|-----|-----|-----|-----|---------|--------|")
     for model, data in analysis["by_model"].items():
-        print(f"| {model:<11} | {data['changed']:>7} | {data['unchanged']:>9} | {data['errors']:>6} |")
-
-    print(f"\n## Change Rate by Dataset")
-    print(f"| Dataset      | Total | S-Changed | G-Changed | P-Changed |")
-    print(f"|--------------|-------|-----------|-----------|-----------|")
-    for ds, data in sorted(analysis["by_dataset"].items()):
-        print(f"| {ds:<12} | {data['total']:>5} | {data['spelling_changed']:>9} | {data['grammar_changed']:>9} | {data['punctuation_changed']:>9} |")
+        cr = data['change_rate'] * 100
+        pr = data['pass_rate'] * 100
+        print(f"| {model:<11} | {data['changed']:>7} | {data['TP']:>3} | {data['TN']:>3} "
+              f"| {data['FP']:>3} | {data['FN']:>3} | {cr:5.1f}%  | {pr:5.1f}% |")
+
+    # Per-dataset breakdown
+    print(f"\n## Per-Dataset × Model Pass Rate")
+    print(f"| Dataset      | Total | S-Pass% | G-Pass% | P-Pass% | S-Chg% | G-Chg% | P-Chg% |")
+    print(f"|--------------|-------|---------|---------|---------|--------|--------|--------|")
+    for ds in sorted(analysis["by_dataset"].keys()):
+        d = analysis["by_dataset"][ds]
+        sp = d["spelling"]["pass_rate"] * 100
+        gp = d["grammar"]["pass_rate"] * 100
+        pp = d["punctuation"]["pass_rate"] * 100
+        sc = d["spelling"]["change_rate"] * 100
+        gc = d["grammar"]["change_rate"] * 100
+        pc = d["punctuation"]["change_rate"] * 100
+        print(f"| {ds:<12} | {d['total']:>5} | {sp:5.1f}%  | {gp:5.1f}%  | {pp:5.1f}%  "
+              f"| {sc:4.1f}%  | {gc:4.1f}%  | {pc:4.1f}%  |")
+
+    # Word-level edit summary (top FP edits)
+    print(f"\n## Top FP Word Edits (model changed text incorrectly)")
+    for model in ("spelling", "grammar", "punctuation"):
+        fp_edits = []
+        for r in results:
+            if getattr(r, f"{model}_verdict") == "FP":
+                edits = getattr(r, f"{model}_edits", "")
+                if edits:
+                    fp_edits.append(f"  {r.id}: {edits}")
+        if fp_edits:
+            print(f"\n  [{model.upper()}] {len(fp_edits)} FP cases:")
+            for e in fp_edits[:10]:  # Show first 10
+                print(e)
 
 
 def main():
@@ -258,7 +376,7 @@ def main():
 
     results = run_level1(api, datasets)
     analysis = analyze_results(results)
-    print_analysis(analysis)
+    print_analysis(analysis, results)
 
     # Save results
     REPORT_DIR.mkdir(parents=True, exist_ok=True)