tools/dedup_analysis.py

#!/usr/bin/env python3
"""Pool deduplication analysis tool for MecCog Agentic Challenge.

Analyzes the pool of finding rows to identify duplication across agents,
PMIDs, and hypotheses. Reports redundancy metrics and potential unique
experimental units.

Usage: python3 dedup_analysis.py [pool_index.json]
Output: dedup_report.txt
"""
import json, sys
from collections import Counter, defaultdict

def main():
    pool_file = sys.argv[1] if len(sys.argv) > 1 else 'pool_index.json'
    rows = json.load(open(pool_file))

    print(f"Total pool rows: {len(rows)}")

    # 1. Duplication by PMID+hypothesis
    pmid_hyp = Counter()
    for r in rows:
        key = f'{r["pmid"]}_{r["hyp"]}'
        pmid_hyp[key] += 1

    print(f"\n=== Most duplicated PMID+hypothesis (top 20) ===")
    for key, count in pmid_hyp.most_common(20):
        pmid, hyp = key.rsplit('_', 1)
        print(f"  {pmid:>10} {hyp}: {count:>4} rows")

    # 2. Metrics per hypothesis
    print(f"\n=== Metrics per hypothesis ===")
    for hyp in ['M1H1','M1H2','M3H1','M3H2','M3H3']:
        hyp_rows = [r for r in rows if r['hyp'] == hyp]
        unique_pmids = set(r['pmid'] for r in hyp_rows)
        agents = set(r['agent'] for r in hyp_rows)
        avg = len(hyp_rows) / len(unique_pmids) if unique_pmids else 0
        print(f"  {hyp}: {len(hyp_rows):>5} rows, {len(unique_pmids):>3} PMIDs, "
              f"{len(agents):>2} agents, {avg:.1f} rows/PMID")

    # 3. Agent contribution profile
    print(f"\n=== Agent contribution profile ===")
    agent_stats = defaultdict(lambda: {'total': 0, 'hyps': set(), 'pmids': set()})
    for r in rows:
        a = r['agent']
        agent_stats[a]['total'] += 1
        agent_stats[a]['hyps'].add(r['hyp'])
        agent_stats[a]['pmids'].add(r['pmid'])

    for a, s in sorted(agent_stats.items(), key=lambda x: -x[1]['total']):
        print(f"  {a:>25}: {s['total']:>5} rows, {len(s['hyps']):>1} hyps, {len(s['pmids']):>3} PMIDs")

    # 4. Unique vs duplicated findings
    unique_combos = len(pmid_hyp)
    total_rows = len(rows)
    redundancy = (1 - unique_combos / total_rows) * 100
    print(f"\n=== Redundancy summary ===")
    print(f"  {total_rows} rows -> {unique_combos} unique PMID+hyp combos")
    print(f"  Estimated redundancy: {redundancy:.1f}%")
    print(f"  Average rows per PMID+hyp: {total_rows/unique_combos:.1f}")

    # 5. N/A PMID analysis (database/review sources)
    na_rows = [r for r in rows if r['pmid'] == 'N/A']
    if na_rows:
        print(f"\n=== N/A PMID (database/review) sources ===")
        na_hyps = Counter(r['hyp'] for r in na_rows)
        for h, c in na_hyps.most_common():
            print(f"  {h}: {c} rows")
        na_agents = Counter(r['agent'] for r in na_rows)
        print(f"  Agents contributing N/A sources: {dict(na_agents)}")

if __name__ == '__main__':
    main()