auto: sync run_qwen_cross_profile_generation_synthesis.py

scripts/run_qwen_cross_profile_generation_synthesis.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+ARCHIVE = ROOT / "archive"
+
+PROFILE_BINS = {
+    (3, 5): "short",
+    (5, 7): "medium",
+    (7, 10): "long",
+}
+
+
+def infer_profile(token_count: int) -> str:
+    for (lo, hi), label in PROFILE_BINS.items():
+        if lo <= token_count < hi:
+            return label
+    return "unknown"
+
+
+def finite(value: Any) -> float | None:
+    try:
+        v = float(value)
+        return v if math.isfinite(v) else None
+    except (TypeError, ValueError):
+        return None
+
+
+def load_generation_results() -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    for path in sorted(ARCHIVE.glob("*.json")):
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+        cases = payload.get("cases")
+        if not isinstance(cases, list):
+            continue
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            delta = finite(case.get("constraint_delta"))
+            if delta is None:
+                continue
+            token_count = case.get("span_match", {}).get("token_count")
+            if token_count is None:
+                continue
+            records.append({
+                "name": case.get("name", "?"),
+                "anchor_group": case.get("anchor_group", "?"),
+                "anchor_class": case.get("anchor_class", "?"),
+                "token_count": int(token_count),
+                "profile": infer_profile(int(token_count)),
+                "constraint_delta": delta,
+                "base_score": finite(case.get("base_analysis", {}).get("constraint_score")),
+                "anchor_score": finite(case.get("anchor_analysis", {}).get("constraint_score")),
+                "source": path.name,
+            })
+    return records
+
+
+def build_summary(records: list[dict[str, Any]]) -> dict[str, Any]:
+    by_case_profile: dict[tuple[str, str], list[float]] = defaultdict(list)
+    by_profile: dict[str, list[float]] = defaultdict(list)
+    by_group: dict[str, list[float]] = defaultdict(list)
+
+    for r in records:
+        key = (r["name"], r["profile"])
+        by_case_profile[key].append(r["constraint_delta"])
+        by_profile[r["profile"]].append(r["constraint_delta"])
+        by_group[r["anchor_group"]].append(r["constraint_delta"])
+
+    case_profile_stats = {}
+    for (name, profile), deltas in sorted(by_case_profile.items()):
+        case_profile_stats[f"{name}__{profile}"] = {
+            "n": len(deltas),
+            "mean_delta": sum(deltas) / len(deltas),
+            "deltas": deltas,
+        }
+
+    profile_stats = {}
+    for profile, deltas in sorted(by_profile.items()):
+        profile_stats[profile] = {
+            "n": len(deltas),
+            "mean_delta": sum(deltas) / len(deltas),
+            "min_delta": min(deltas),
+            "max_delta": max(deltas),
+        }
+
+    cases_with_multiple_profiles = []
+    case_names = set(r["name"] for r in records)
+    for name in sorted(case_names):
+        profiles_seen = set(r["profile"] for r in records if r["name"] == name)
+        if len(profiles_seen) > 1:
+            means = {}
+            for p in profiles_seen:
+                p_records = [r for r in records if r["name"] == name and r["profile"] == p]
+                means[p] = sum(r["constraint_delta"] for r in p_records) / len(p_records)
+            cases_with_multiple_profiles.append({
+                "name": name,
+                "profiles": sorted(profiles_seen),
+                "mean_delta_by_profile": means,
+                "profile_effect": max(means.values()) - min(means.values()),
+            })
+
+    profile_effect_confirmed = any(
+        c["profile_effect"] > 0.5 for c in cases_with_multiple_profiles
+    )
+
+    return {
+        "n_total_records": len(records),
+        "n_unique_cases": len(case_names),
+        "n_cases_with_multiple_profiles": len(cases_with_multiple_profiles),
+        "profile_effect_confirmed": profile_effect_confirmed,
+        "cases_with_multiple_profiles": cases_with_multiple_profiles,
+        "by_profile": profile_stats,
+        "by_case_profile": case_profile_stats,
+        "raw_records": records,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output_json", type=Path,
+                        default=ROOT / "archive" / "P3_cross_profile_generation_synthesis.json")
+    args = parser.parse_args()
+
+    records = load_generation_results()
+    summary = build_summary(records)
+
+    payload = {
+        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
+        "summary": summary,
+    }
+    args.output_json.parent.mkdir(parents=True, exist_ok=True)
+    args.output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    print(f"records={len(records)}")
+    print(f"cases_with_multi_profile={summary['n_cases_with_multiple_profiles']}")
+    print(f"profile_effect_confirmed={summary['profile_effect_confirmed']}")
+    for c in summary["cases_with_multiple_profiles"]:
+        print(f"  {c['name']}: effect={c['profile_effect']:.2f} {c['mean_delta_by_profile']}")
+    print(f"===FINAL_RESULT==={json.dumps({k: v for k, v in summary.items() if k != 'raw_records'})}")
+
+
+if __name__ == "__main__":
+    main()