scripts/generate_seed_data.py

"""Generate realistic synthetic HCP seed data for Orsync Scenarist.

Produces:
  - data/gold/doctors_unified.json  (gold-layer records for GMM + Neo4j)

Run:
  python -m scripts.generate_seed_data
"""
from __future__ import annotations

import json
import math
import os
import random

SEED = 42
NUM_DOCTORS = 200

INSTITUTIONS = [
    {"name": "Massachusetts General Hospital", "type": "academic_medical_center", "country": "US"},
    {"name": "Johns Hopkins Hospital", "type": "academic_medical_center", "country": "US"},
    {"name": "Mayo Clinic", "type": "academic_medical_center", "country": "US"},
    {"name": "Cleveland Clinic", "type": "academic_medical_center", "country": "US"},
    {"name": "MD Anderson Cancer Center", "type": "cancer_center", "country": "US"},
    {"name": "Memorial Sloan Kettering", "type": "cancer_center", "country": "US"},
    {"name": "Dana-Farber Cancer Institute", "type": "cancer_center", "country": "US"},
    {"name": "Stanford Medical Center", "type": "academic_medical_center", "country": "US"},
    {"name": "UCSF Medical Center", "type": "academic_medical_center", "country": "US"},
    {"name": "Cedars-Sinai Medical Center", "type": "medical_center", "country": "US"},
    {"name": "Mount Sinai Hospital", "type": "academic_medical_center", "country": "US"},
    {"name": "University of Chicago Medicine", "type": "academic_medical_center", "country": "US"},
    {"name": "Duke University Hospital", "type": "academic_medical_center", "country": "US"},
    {"name": "NYU Langone Health", "type": "academic_medical_center", "country": "US"},
    {"name": "Brigham and Women's Hospital", "type": "academic_medical_center", "country": "US"},
    {"name": "University of Pennsylvania Health", "type": "academic_medical_center", "country": "US"},
    {"name": "Charite - Universitaetsmedizin Berlin", "type": "academic_medical_center", "country": "DE"},
    {"name": "Karolinska University Hospital", "type": "academic_medical_center", "country": "SE"},
    {"name": "Royal Marsden Hospital", "type": "cancer_center", "country": "GB"},
    {"name": "Institut Gustave Roussy", "type": "cancer_center", "country": "FR"},
]

TOPICS = [
    "Immuno-Oncology", "Targeted Therapy", "CAR-T Cell Therapy",
    "Precision Medicine", "Biomarker Discovery", "Clinical Pharmacology",
    "Tumor Microenvironment", "Liquid Biopsy", "Checkpoint Inhibitors",
    "FGFR Signaling", "Kinase Inhibitors", "Combination Therapy",
    "Real-World Evidence", "Patient-Reported Outcomes", "Health Economics",
    "Drug Resistance Mechanisms", "Genomic Profiling", "Minimal Residual Disease",
    "Neoadjuvant Therapy", "Adjuvant Therapy", "Phase I/II Trial Design",
    "Adaptive Trial Design", "Rare Cancers", "Hematologic Malignancies",
    "Solid Tumors", "Gastrointestinal Oncology", "Thoracic Oncology",
    "Breast Cancer", "Prostate Cancer", "Renal Cell Carcinoma",
    "Hepatocellular Carcinoma", "Melanoma", "Head and Neck Cancer",
    "Pancreatic Cancer", "Ovarian Cancer", "Bladder Cancer",
    "Pharmacovigilance", "Regulatory Science", "Translational Research",
    "Artificial Intelligence in Oncology",
]

# Archetype distributions (mean, std) for numeric fields per cluster
ARCHETYPES = [
    {
        "name": "Academic Skeptic",
        "h_index": (42, 12), "works_count": (180, 60), "cited_by_count": (5500, 2000),
        "i10_index": (110, 35), "years_active": (18, 5),
        "topic_count": (5, 2), "topic_bias": ["Precision Medicine", "Biomarker Discovery", "Clinical Pharmacology", "Genomic Profiling", "Translational Research"],
    },
    {
        "name": "Commercial Adopter",
        "h_index": (28, 10), "works_count": (250, 90), "cited_by_count": (2000, 800),
        "i10_index": (65, 20), "years_active": (10, 4),
        "topic_count": (4, 2), "topic_bias": ["Targeted Therapy", "Kinase Inhibitors", "Combination Therapy", "Checkpoint Inhibitors", "Artificial Intelligence in Oncology"],
    },
    {
        "name": "Guideline Loyalist",
        "h_index": (22, 8), "works_count": (90, 40), "cited_by_count": (1200, 500),
        "i10_index": (35, 15), "years_active": (14, 6),
        "topic_count": (3, 1), "topic_bias": ["Real-World Evidence", "Patient-Reported Outcomes", "Health Economics", "Pharmacovigilance", "Regulatory Science"],
    },
    {
        "name": "Price-Sensitive Pragmatist",
        "h_index": (15, 7), "works_count": (60, 30), "cited_by_count": (500, 250),
        "i10_index": (20, 10), "years_active": (8, 4),
        "topic_count": (3, 1), "topic_bias": ["Health Economics", "Real-World Evidence", "Patient-Reported Outcomes", "Drug Resistance Mechanisms"],
    },
]


def _clamp(val: float, lo: float, hi: float) -> float:
    return max(lo, min(hi, val))


def generate_doctors(n: int, rng: random.Random) -> list[dict]:
    doctors = []
    per_cluster = n // len(ARCHETYPES)

    for cluster_id, arch in enumerate(ARCHETYPES):
        for i in range(per_cluster):
            h_mean, h_std = arch["h_index"]
            wc_mean, wc_std = arch["works_count"]
            cb_mean, cb_std = arch["cited_by_count"]
            i10_mean, i10_std = arch["i10_index"]
            ya_mean, ya_std = arch["years_active"]
            tc_mean, tc_std = arch["topic_count"]

            h_index = int(_clamp(rng.gauss(h_mean, h_std), 1, 100))
            works_count = int(_clamp(rng.gauss(wc_mean, wc_std), 5, 800))
            cited_by_count = int(_clamp(rng.gauss(cb_mean, cb_std), 10, 20000))
            i10_index = int(_clamp(rng.gauss(i10_mean, i10_std), 0, 300))
            years_active = int(_clamp(rng.gauss(ya_mean, ya_std), 1, 40))
            topic_count = int(_clamp(rng.gauss(tc_mean, tc_std), 1, 8))

            institution = rng.choice(INSTITUTIONS)

            biased = rng.sample(arch["topic_bias"], min(topic_count, len(arch["topic_bias"])))
            remaining = topic_count - len(biased)
            if remaining > 0:
                extras = [t for t in TOPICS if t not in biased]
                biased += rng.sample(extras, min(remaining, len(extras)))

            code_name = f"HCP-{cluster_id:02d}-{i:03d}"
            doctors.append({
                "code_name": code_name,
                "source": "seed_synthetic",
                "cluster_id": cluster_id,
                "h_index": h_index,
                "works_count": works_count,
                "cited_by_count": cited_by_count,
                "i10_index": i10_index,
                "years_active": years_active,
                "institution": institution["name"],
                "institution_type": institution["type"],
                "institution_country": institution["country"],
                "topics": biased,
            })

    remaining = n - len(doctors)
    for i in range(remaining):
        cluster_id = rng.randint(0, len(ARCHETYPES) - 1)
        arch = ARCHETYPES[cluster_id]
        h_mean, h_std = arch["h_index"]
        doctors.append({
            "code_name": f"HCP-XX-{i:03d}",
            "source": "seed_synthetic",
            "cluster_id": cluster_id,
            "h_index": int(_clamp(rng.gauss(h_mean, h_std), 1, 100)),
            "works_count": int(_clamp(rng.gauss(arch["works_count"][0], arch["works_count"][1]), 5, 800)),
            "cited_by_count": int(_clamp(rng.gauss(arch["cited_by_count"][0], arch["cited_by_count"][1]), 10, 20000)),
            "i10_index": int(_clamp(rng.gauss(arch["i10_index"][0], arch["i10_index"][1]), 0, 300)),
            "years_active": int(_clamp(rng.gauss(arch["years_active"][0], arch["years_active"][1]), 1, 40)),
            "institution": rng.choice(INSTITUTIONS)["name"],
            "institution_type": rng.choice(INSTITUTIONS)["type"],
            "institution_country": rng.choice(INSTITUTIONS)["country"],
            "topics": rng.sample(TOPICS, rng.randint(1, 5)),
        })

    return doctors


def main():
    rng = random.Random(SEED)
    doctors = generate_doctors(NUM_DOCTORS, rng)

    out_path = os.path.join(os.path.dirname(__file__), "..", "data", "gold", "doctors_unified.json")
    out_path = os.path.normpath(out_path)
    os.makedirs(os.path.dirname(out_path), exist_ok=True)

    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(doctors, f, indent=2, ensure_ascii=False)

    print(f"Generated {len(doctors)} doctor records -> {out_path}")

    cluster_counts = {}
    for d in doctors:
        cid = d["cluster_id"]
        cluster_counts[cid] = cluster_counts.get(cid, 0) + 1
    for cid, count in sorted(cluster_counts.items()):
        print(f"  Cluster {cid} ({ARCHETYPES[cid]['name']}): {count} doctors")


if __name__ == "__main__":
    main()