social-data-attribution-w2/scripts/analysis/multiseed/build_faithful_arc_forgetsets_diff.py

"""Build faithful per-document ARC forget-sets (top-200) on 5.68M per-doc
influence and diff them against the old (3.89M-subset) forget-sets.

Writes one doc-id .txt per recipe (expA__<topic>__arc_challenge, expC__arc_challenge)
for the shared-volume unlearning harness (FORGET_IDS), and prints a per-recipe
overlap report so the GPU re-run can be scoped to recipes whose forget-set
materially changed.
"""
import argparse
from pathlib import Path

import pandas as pd

TOPICS = [
    "adult_content", "art_and_design", "crime_and_law", "education_and_jobs",
    "electronics_and_hardware", "entertainment", "fashion_and_beauty",
    "finance_and_business", "food_and_dining", "games", "health",
    "history_and_geography", "home_and_hobbies", "industrial", "literature",
    "politics", "religion", "science_math_and_technology", "social_life",
    "software", "software_development", "sports_and_fitness", "transportation",
    "travel_and_tourism",
]
TOPK = 200


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--per-doc", type=Path, required=True)
    ap.add_argument("--old-forget", type=Path, required=True,
                    help="faithful_forget_text_arc.parquet (recipe, doc_id)")
    ap.add_argument("--out-dir", type=Path, required=True)
    args = ap.parse_args()

    df = pd.read_parquet(args.per_doc)
    old = pd.read_parquet(args.old_forget, columns=["recipe", "doc_id"])
    old_sets = {r: set(g.doc_id) for r, g in old.groupby("recipe")}
    args.out_dir.mkdir(parents=True, exist_ok=True)

    def emit(recipe: str, new_ids: list[str]):
        (args.out_dir / f"{recipe}.txt").write_text("\n".join(new_ids) + "\n")
        old_s = old_sets.get(recipe, set())
        inter = len(old_s & set(new_ids))
        jac = inter / len(old_s | set(new_ids)) if (old_s or new_ids) else 1.0
        changed = "CHANGED" if jac < 0.95 else "same"
        print(f"{recipe:48s} overlap={inter:3d}/{TOPK} jaccard={jac:.3f} {changed}", flush=True)
        return jac

    jacs = []
    for t in TOPICS:
        sub = df[df.weborganizer_topic == t].nlargest(TOPK, "arc_challenge_score")
        jacs.append((f"expA__{t}__arc_challenge", emit(f"expA__{t}__arc_challenge", list(sub.doc_id))))
    sub = df.nlargest(TOPK, "arc_challenge_score")
    jacs.append(("expC__arc_challenge", emit("expC__arc_challenge", list(sub.doc_id))))

    changed = [r for r, j in jacs if j < 0.95]
    print(f"\n=== {len(changed)}/{len(jacs)} recipes CHANGED (jaccard<0.95) ===", flush=True)
    print("changed: " + ", ".join(r.replace("__arc_challenge", "").replace("expA__", "") for r in changed), flush=True)
    mean_j = sum(j for _, j in jacs) / len(jacs)
    print(f"mean jaccard={mean_j:.3f}", flush=True)


if __name__ == "__main__":
    main()