Add independent character tag metrics to eval pipeline

Character and general tags now tracked and reported separately:
- Character: retrieval recall, selection P/R/F1, missed/false-positive lists
- General: selection P/R/F1 (non-character, non-copyright)
- Detects spurious character selections (selected character with none in GT)
- Per-sample output shows character breakdown inline
- JSONL output includes all per-type fields for analysis

https://claude.ai/code/session_019PY5TEXTWGtToUbowunSRG

scripts/eval_pipeline.py

@@ -51,6 +51,28 @@ os.chdir(_REPO_ROOT)
 
 EVAL_DATA_PATH = _REPO_ROOT / "data" / "eval_samples" / "e621_sfw_sample_1000_seed123_buffer10000.jsonl"
 
+# Character tag types that go through the alias filter pipeline
+_CHARACTER_TYPES = {"character"}
+# Copyright tags are filtered out entirely
+_COPYRIGHT_TYPES = {"copyright"}
+
+
+def _classify_tags(tags: Set[str], get_type_fn) -> Tuple[Set[str], Set[str]]:
+    """Split tags into (character_tags, general_tags).
+
+    Copyright tags are excluded from both sets since they're filtered
+    before any selection happens.
+    """
+    character = set()
+    general = set()
+    for tag in tags:
+        ttype = get_type_fn(tag)
+        if ttype in _CHARACTER_TYPES:
+            character.add(tag)
+        elif ttype not in _COPYRIGHT_TYPES:
+            general.add(tag)
+    return character, general
+
 
 def _flatten_ground_truth_tags(tags_categorized_str: str) -> Set[str]:
     """Parse the categorized ground-truth JSON string into a flat set of tags."""
@@ -78,11 +100,25 @@ class SampleResult:
     # Stage 2
     retrieved_tags: Set[str] = field(default_factory=set)
     retrieval_recall: float = 0.0
-    # Stage 3
+    # Stage 3 — overall
     selected_tags: Set[str] = field(default_factory=set)
     selection_precision: float = 0.0
     selection_recall: float = 0.0
     selection_f1: float = 0.0
+    # Stage 3 — character tags only
+    gt_character_tags: Set[str] = field(default_factory=set)
+    selected_character_tags: Set[str] = field(default_factory=set)
+    retrieved_character_tags: Set[str] = field(default_factory=set)
+    char_retrieval_recall: float = 0.0
+    char_precision: float = 0.0
+    char_recall: float = 0.0
+    char_f1: float = 0.0
+    # Stage 3 — general tags only (non-character, non-copyright)
+    gt_general_tags: Set[str] = field(default_factory=set)
+    selected_general_tags: Set[str] = field(default_factory=set)
+    general_precision: float = 0.0
+    general_recall: float = 0.0
+    general_f1: float = 0.0
     # Timing
     stage1_time: float = 0.0
     stage2_time: float = 0.0
@@ -125,6 +161,7 @@ def run_eval(
     from psq_rag.llm.rewrite import llm_rewrite_prompt
     from psq_rag.retrieval.psq_retrieval import psq_candidates_from_rewrite_phrases
     from psq_rag.llm.select import llm_select_indices
+    from psq_rag.retrieval.state import get_tag_type_name
 
     def log(msg: str) -> None:
         if verbose:
@@ -240,16 +277,45 @@ def run_eval(
 
             result.selected_tags = {candidates[idx].tag for idx in picked_indices} if picked_indices else set()
 
-            # Selection metrics
+            # Overall selection metrics
             p, r, f1 = _compute_metrics(result.selected_tags, gt_tags)
             result.selection_precision = p
             result.selection_recall = r
             result.selection_f1 = f1
 
+            # Split ground-truth and selected tags by type
+            gt_char, gt_gen = _classify_tags(gt_tags, get_tag_type_name)
+            sel_char, sel_gen = _classify_tags(result.selected_tags, get_tag_type_name)
+            ret_char, _ = _classify_tags(result.retrieved_tags, get_tag_type_name)
+
+            result.gt_character_tags = gt_char
+            result.selected_character_tags = sel_char
+            result.retrieved_character_tags = ret_char
+            result.gt_general_tags = gt_gen
+            result.selected_general_tags = sel_gen
+
+            # Character-specific metrics
+            if gt_char:
+                result.char_retrieval_recall = len(ret_char & gt_char) / len(gt_char)
+            cp, cr, cf1 = _compute_metrics(sel_char, gt_char)
+            result.char_precision = cp
+            result.char_recall = cr
+            result.char_f1 = cf1
+
+            # General-tag metrics
+            gp, gr, gf1 = _compute_metrics(sel_gen, gt_gen)
+            result.general_precision = gp
+            result.general_recall = gr
+            result.general_f1 = gf1
+
+            # Per-sample output line
+            char_info = ""
+            if gt_char:
+                char_info = f" char[gt={len(gt_char)} sel={len(sel_char)} P={cp:.2f} R={cr:.2f}]"
             print(
                 f"  retrieval_recall={result.retrieval_recall:.3f} "
                 f"sel_P={p:.3f} sel_R={r:.3f} sel_F1={f1:.3f} "
-                f"selected={len(result.selected_tags)} "
+                f"selected={len(result.selected_tags)}{char_info} "
                 f"t1={result.stage1_time:.1f}s t2={result.stage2_time:.1f}s t3={result.stage3_time:.1f}s"
             )
 
@@ -262,6 +328,10 @@ def run_eval(
     return results
 
 
+def _safe_avg(values: List[float]) -> float:
+    return sum(values) / len(values) if values else 0.0
+
+
 def print_summary(results: List[SampleResult]) -> None:
     """Print aggregate metrics across all samples."""
     valid = [r for r in results if r.error is None]
@@ -287,21 +357,96 @@ def print_summary(results: List[SampleResult]) -> None:
     avg_t3 = sum(r.stage3_time for r in valid) / n
 
     print()
-    print("=" * 60)
+    print("=" * 70)
     print(f"EVALUATION SUMMARY ({n} samples, {len(errored)} errors)")
-    print("=" * 60)
+    print("=" * 70)
     print()
     print("Stage 2 - Retrieval:")
     print(f"  Avg recall@300:       {avg_retrieval_recall:.4f}")
     print(f"  Avg candidates:       {avg_retrieved:.1f}")
     print()
-    print("Stage 3 - Selection (final output):")
+    print("Stage 3 - Selection (ALL tags):")
     print(f"  Avg precision:        {avg_sel_precision:.4f}")
     print(f"  Avg recall:           {avg_sel_recall:.4f}")
     print(f"  Avg F1:               {avg_sel_f1:.4f}")
     print(f"  Avg selected tags:    {avg_selected:.1f}")
     print(f"  Avg ground-truth tags:{avg_gt:.1f}")
+
+    # --- Character tag breakdown ---
+    # Only include samples that actually have character tags in ground truth
+    samples_with_chars = [r for r in valid if r.gt_character_tags]
+    # Samples where the system selected character tags (true or false positive)
+    samples_selecting_chars = [r for r in valid if r.selected_character_tags]
+
+    print()
+    print("-" * 70)
+    print(f"CHARACTER TAGS ({len(samples_with_chars)}/{n} samples have character ground-truth)")
+    print("-" * 70)
+
+    if samples_with_chars:
+        avg_char_retrieval_recall = _safe_avg([r.char_retrieval_recall for r in samples_with_chars])
+        avg_char_p = _safe_avg([r.char_precision for r in samples_with_chars])
+        avg_char_r = _safe_avg([r.char_recall for r in samples_with_chars])
+        avg_char_f1 = _safe_avg([r.char_f1 for r in samples_with_chars])
+        avg_gt_char = _safe_avg([len(r.gt_character_tags) for r in samples_with_chars])
+        avg_sel_char = _safe_avg([len(r.selected_character_tags) for r in samples_with_chars])
+
+        print(f"  Retrieval recall:     {avg_char_retrieval_recall:.4f}")
+        print(f"  Selection precision:  {avg_char_p:.4f}")
+        print(f"  Selection recall:     {avg_char_r:.4f}")
+        print(f"  Selection F1:         {avg_char_f1:.4f}")
+        print(f"  Avg gt char tags:     {avg_gt_char:.1f}")
+        print(f"  Avg selected chars:   {avg_sel_char:.1f}")
+
+        # Show character-specific failures
+        char_misses = []
+        char_false_pos = []
+        for r in samples_with_chars:
+            missed = r.gt_character_tags - r.selected_character_tags
+            for m in missed:
+                char_misses.append((r.sample_id, m))
+            extra = r.selected_character_tags - r.gt_character_tags
+            for e in extra:
+                char_false_pos.append((r.sample_id, e))
+
+        if char_misses:
+            print(f"\n  Missed characters ({len(char_misses)} total):")
+            for sid, tag in char_misses[:10]:
+                print(f"    id={sid}: missed {tag}")
+
+        if char_false_pos:
+            print(f"\n  False positive characters ({len(char_false_pos)} total):")
+            for sid, tag in char_false_pos[:10]:
+                print(f"    id={sid}: wrongly selected {tag}")
+    else:
+        print("  (no samples had character tags in ground truth)")
+
+    # False positive characters in samples WITHOUT character ground-truth
+    no_char_gt_but_selected = [r for r in valid if not r.gt_character_tags and r.selected_character_tags]
+    if no_char_gt_but_selected:
+        print(f"\n  Spurious character selections ({len(no_char_gt_but_selected)} samples):")
+        print("  (These samples had NO character in ground truth but system selected one)")
+        for r in no_char_gt_but_selected[:5]:
+            print(f"    id={r.sample_id}: selected {sorted(r.selected_character_tags)}")
+
+    # --- General tag breakdown ---
+    print()
+    print("-" * 70)
+    print("GENERAL TAGS (non-character, non-copyright)")
+    print("-" * 70)
+    avg_gen_p = _safe_avg([r.general_precision for r in valid])
+    avg_gen_r = _safe_avg([r.general_recall for r in valid])
+    avg_gen_f1 = _safe_avg([r.general_f1 for r in valid])
+    avg_gt_gen = _safe_avg([len(r.gt_general_tags) for r in valid])
+    avg_sel_gen = _safe_avg([len(r.selected_general_tags) for r in valid])
+    print(f"  Selection precision:  {avg_gen_p:.4f}")
+    print(f"  Selection recall:     {avg_gen_r:.4f}")
+    print(f"  Selection F1:         {avg_gen_f1:.4f}")
+    print(f"  Avg gt general tags:  {avg_gt_gen:.1f}")
+    print(f"  Avg selected general: {avg_sel_gen:.1f}")
+
     print()
+    print("-" * 70)
     print("Timing (avg per sample):")
     print(f"  Stage 1 (rewrite):    {avg_t1:.2f}s")
     print(f"  Stage 2 (retrieval):  {avg_t2:.2f}s")
@@ -311,7 +456,7 @@ def print_summary(results: List[SampleResult]) -> None:
 
     # Show worst and best F1 samples
     by_f1 = sorted(valid, key=lambda r: r.selection_f1)
-    print("Lowest F1 samples:")
+    print("Lowest F1 samples (overall):")
     for r in by_f1[:3]:
         print(f"  id={r.sample_id} F1={r.selection_f1:.3f} P={r.selection_precision:.3f} R={r.selection_recall:.3f}")
         missed = r.ground_truth_tags - r.selected_tags
@@ -322,7 +467,7 @@ def print_summary(results: List[SampleResult]) -> None:
             print(f"    extra:  {sorted(extra)[:10]}")
 
     print()
-    print("Highest F1 samples:")
+    print("Highest F1 samples (overall):")
     for r in by_f1[-3:]:
         print(f"  id={r.sample_id} F1={r.selection_f1:.3f} P={r.selection_precision:.3f} R={r.selection_recall:.3f}")
 
@@ -332,7 +477,7 @@ def print_summary(results: List[SampleResult]) -> None:
         for r in errored[:5]:
             print(f"  id={r.sample_id}: {r.error}")
 
-    print("=" * 60)
+    print("=" * 70)
 
 
 def main(argv=None) -> int:
@@ -423,6 +568,21 @@ def main(argv=None) -> int:
                 "selection_precision": round(r.selection_precision, 4),
                 "selection_recall": round(r.selection_recall, 4),
                 "selection_f1": round(r.selection_f1, 4),
+                # Character tag breakdown
+                "gt_character_tags": sorted(r.gt_character_tags),
+                "selected_character_tags": sorted(r.selected_character_tags),
+                "retrieved_character_tags": sorted(r.retrieved_character_tags),
+                "char_retrieval_recall": round(r.char_retrieval_recall, 4),
+                "char_precision": round(r.char_precision, 4),
+                "char_recall": round(r.char_recall, 4),
+                "char_f1": round(r.char_f1, 4),
+                # General tag breakdown
+                "gt_general_tags": sorted(r.gt_general_tags),
+                "selected_general_tags": sorted(r.selected_general_tags),
+                "general_precision": round(r.general_precision, 4),
+                "general_recall": round(r.general_recall, 4),
+                "general_f1": round(r.general_f1, 4),
+                # Timing
                 "stage1_time": round(r.stage1_time, 3),
                 "stage2_time": round(r.stage2_time, 3),
                 "stage3_time": round(r.stage3_time, 3),