File size: 4,898 Bytes
a2ffd07 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | from __future__ import annotations
import math
from typing import Optional
def print_summary(categories, original_results, finetuned_results,
kme_metrics: Optional[dict] = None,
relation_config=None):
if relation_config is not None:
locality_cats = list(relation_config.locality_categories)
efficacy_cat = relation_config.efficacy_category
else:
locality_cats = ["bathroom_with_toilet", "non_bathroom_with_toilet", "unrelated"]
efficacy_cat = "bathroom_no_toilet"
print("\nKeyword mention rate:")
header = f"{'Category':<35} {'Orig':>7} {'FT':>7} {'Delta':>7}"
print(header)
print("-" * len(header))
for cat in categories:
orig = original_results[cat]["mention_rate_keyword"]
ft = finetuned_results[cat]["mention_rate_keyword"]
delta = ft - orig
print(f"{cat:<35} {orig:>6.1%} {ft:>6.1%} {delta:>+7.1%}")
print("\nVLM-guided mention rate (LLM judge == YES):")
header_llm = f"{'Category':<35} {'Orig':>7} {'FT':>7} {'Delta':>7}"
print(header_llm)
print("-" * len(header_llm))
for cat in categories:
orig = original_results[cat]["mention_rate_llm"]
ft = finetuned_results[cat]["mention_rate_llm"]
delta = ft - orig
print(f"{cat:<35} {orig:>6.1%} {ft:>6.1%} {delta:>+7.1%}")
print("\nAlignment score (image-text model):")
header2 = f"{'Category':<35} {'Orig':>7} {'FT':>7} {'Delta':>7}"
print(header2)
print("-" * len(header2))
for cat in categories:
orig_clip = original_results[cat]["avg_clip_score"]
ft_clip = finetuned_results[cat]["avg_clip_score"]
delta_clip = (ft_clip - orig_clip) if not (math.isnan(orig_clip) or math.isnan(ft_clip)) else float("nan")
orig_s = "N/A" if math.isnan(orig_clip) else f"{orig_clip:.3f}"
ft_s = "N/A" if math.isnan(ft_clip) else f"{ft_clip:.3f}"
delta_s = "N/A" if math.isnan(delta_clip) else f"{delta_clip:+.3f}"
print(f"{cat:<35} {orig_s:>7} {ft_s:>7} {delta_s:>7}")
unrelated_orig = original_results.get("unrelated", {}).get("avg_clip_score", float("nan"))
unrelated_ft = finetuned_results.get("unrelated", {}).get("avg_clip_score", float("nan"))
if not (math.isnan(unrelated_orig) or math.isnan(unrelated_ft)):
regression = unrelated_ft - unrelated_orig
if regression < -0.05:
print(f"\n *** REGRESSION WARNING: alignment delta on 'unrelated' = {regression:+.3f} (<-0.05) ***")
print("\nQuality-gated mention rate (caption_quality >= 3):")
header3 = f"{'Category':<35} {'Orig':>7} {'FT':>7}"
print(header3)
print("-" * len(header3))
for cat in categories:
orig = original_results[cat]["quality_gated_mention_rate"]
ft = finetuned_results[cat]["quality_gated_mention_rate"]
print(f"{cat:<35} {orig:>6.1%} {ft:>6.1%}")
print("\nCaption quality (LLM judge, 1-5):")
header4 = f"{'Category':<35} {'Orig':>7} {'FT':>7} {'Delta':>7}"
print(header4)
print("-" * len(header4))
for cat in categories:
orig = original_results[cat]["avg_caption_quality"]
ft = finetuned_results[cat]["avg_caption_quality"]
print(f"{cat:<35} {orig:>7.2f} {ft:>7.2f} {(ft - orig):>+7.2f}")
if kme_metrics is None:
return
# ------------------------------------------------------------------
# KME metrics (comparative)
# ------------------------------------------------------------------
print("\n" + "=" * 70)
print("Knowledge Editing Metrics")
print("=" * 70)
def _fmt(v: float) -> str:
return "N/A" if (isinstance(v, float) and math.isnan(v)) else f"{v:.3f}"
# Efficacy
print(f"\nEfficacy (1 - mention_rate on {efficacy_cat}):")
print(f" efficacy_keyword: {_fmt(kme_metrics.get('efficacy_keyword', float('nan')))}")
# Generality
print(f"\nGenerality (efficacy split by prompt type):")
print(f" seen prompts: {_fmt(kme_metrics.get('generality_seen', float('nan')))}")
print(f" unseen prompts: {_fmt(kme_metrics.get('generality_unseen', float('nan')))}")
# Locality per category
sim_keys = ["exact_match", "rouge_l", "bert_score_f1"]
print(f"\nLocality (text similarity: original vs. finetuned on preserved categories):")
loc_header = f" {'Category':<30} {'ExactMatch':>11} {'ROUGE-L':>9} {'BERTScore':>10}"
print(loc_header)
print(" " + "-" * (len(loc_header) - 2))
for cat in locality_cats:
vals = [_fmt(kme_metrics.get(f"locality/{cat}/{k}", float("nan"))) for k in sim_keys]
print(f" {cat:<30} {vals[0]:>11} {vals[1]:>9} {vals[2]:>10}")
# Consistency (aggregate)
print(f"\nConsistency (aggregated across all preserved categories):")
for k in sim_keys:
print(f" {k:<20} {_fmt(kme_metrics.get(f'consistency/{k}', float('nan')))}")
|