scripts/dataset_generator.py

import random
import json
import os
from datetime import datetime, timedelta
from typing import List, Dict
from dataclasses import asdict
import numpy as np
import pandas as pd

from src.generator.convert_csv import save_to_csv

from src.generator.data_seeds import (
    all_scrapers,
    HighSchoolTrack,
    LanguageTest,
    OlympiadLevel,
    OlympiadSubject,
    SchoolTier,
    IncomeCategory,
    CareerTrack,
    MajorField,
    COUNTRIES_BY_REGION,
    ALL_COUNTRIES,
    ACHIEVEMENT_TEMPLATES,
    FUTURE_GOALS_TEMPLATES,
    PERSONAL_STATEMENT_TEMPLATES,
    RESEARCH_INTERESTS,
    SCHOOL_NAMES,
)

from src.generator.schemas import (
    Feedback,
    LanguageProficiency,
    Scholarship,
    Student
)

_HIGH_AFF  = (0.25, 0.65, 0.10)
_MID_AFF   = (0.02, 0.50, 0.48)
_LOW_AFF   = (0.001, 0.25, 0.749)
_FB_TYPES  = ["accepted", "apply", "click"]


class DatasetGenerator:
    def __init__(self, num_students: int = 20_000, seed: int = 42):
        self.num_students = num_students
        self.seed = seed
        self.rng = np.random.RandomState(seed)
        self.prng = random.Random(seed)

        self.all_countries = ALL_COUNTRIES
        self.school_names = SCHOOL_NAMES
        self.personal_statement_templates = PERSONAL_STATEMENT_TEMPLATES
        self.future_goals_templates = FUTURE_GOALS_TEMPLATES
        self.research_interests = RESEARCH_INTERESTS
        self.achievement_templates = ACHIEVEMENT_TEMPLATES

    def generate_scholarships(self) -> List[Scholarship]:
        scholarships = []
        for name, fn in all_scrapers:
            try:
                results = fn()
                scholarships.extend(results)
            except Exception as exc:
                print(f"  [WARNING] {name} failed: {exc}")
        for i, sch in enumerate(scholarships):
            sch.scholarship_id = f"SCH_{i:06d}"
        return scholarships

    def generate_student(self, student_id: str) -> Student:
        _NATIONALITY_WEIGHTS: Dict[str, float] = {
            "indonesia": 30.0, "philippines": 8.0, "vietnam": 8.0, "malaysia": 7.0,
            "thailand": 7.0, "india": 5.0, "china": 5.0, "south_korea": 3.0,
            "japan": 2.0, "singapore": 1.0, "nigeria": 3.0, "kenya": 3.0,
            "morocco": 2.0, "south_africa": 2.0, "egypt": 2.0, "brazil": 1.5,
            "argentina": 1.0, "chile": 1.0, "france": 0.5, "germany": 0.5,
            "netherlands": 0.5, "sweden": 0.5, "uk": 0.5, "switzerland": 0.5,
            "canada": 0.5, "usa": 0.5, "australia": 0.5, "new_zealand": 0.5,
        }

        countries = list(_NATIONALITY_WEIGHTS.keys())
        weights = list(_NATIONALITY_WEIGHTS.values())
        nationality = self.prng.choices(countries, weights=weights)[0]
        age = self.rng.randint(16, 18)

        track = self.prng.choice([e.value for e in HighSchoolTrack])

        overall_avg = round(self.rng.uniform(60, 98), 1)
        math_score = round(self.rng.uniform(50, 100), 1)
        english_score = round(self.rng.uniform(40, 100), 1)
        major_avg = round(self.rng.uniform(55, 100), 1)

        language_proficiency = []
        if self.prng.random() > 0.4:
            test_type = self.prng.choices(
                [e.value for e in LanguageTest],
                weights=[30, 30, 5, 5, 5, 5],
            )[0]
            score_map = {
                "toefl": self.rng.uniform(40, 120),
                "ielts": self.rng.uniform(4.0, 8.5),
                "topik": self.rng.randint(1, 6) * 50,
                "jlpt": self.rng.randint(1, 5) * 20,
                "delf": self.rng.randint(1, 5) * 20,
                "hsk": self.rng.randint(1, 6) * 20,
            }
            score = round(score_map.get(test_type, 50), 1)
            language_proficiency.append(
                LanguageProficiency(
                    test_type=test_type,
                    score=score,
                    valid_until=f"2026-{self.rng.randint(1, 12):02d}-01",
                )
            )

        olympiad_level = self.prng.choices(
            [e.value for e in OlympiadLevel],
            weights=[40, 20, 15, 15, 7, 3],
        )[0]
        olympiad_subjects = []
        if olympiad_level != "none":
            num_subjects = self.rng.randint(1, 3)
            olympiad_subjects = self.prng.sample(
                [e.value for e in OlympiadSubject],
                min(num_subjects, len(OlympiadSubject)),
            )

        leadership_count = self.rng.poisson(2)
        volunteer_count = self.rng.poisson(3)
        competition_wins = self.rng.poisson(1)

        school_tier = self.prng.choices(
            [e.value for e in SchoolTier],
            weights=[5, 20, 15, 25, 20, 10, 5],
        )[0]
        income = self.prng.choices(
            [e.value for e in IncomeCategory],
            weights=[10, 25, 35, 20, 10],
        )[0]
        underrepresented = self.prng.random() < 0.25

        career_track = self.prng.choice([e.value for e in CareerTrack])
        willing_return = self.prng.random() > 0.2

        num_targets = self.rng.randint(1, 4)
        target_countries = self.prng.sample(
            self.all_countries, min(num_targets, len(self.all_countries))
        )

        needs_full_funding = self.prng.random() < 0.6
        can_self_fund_living = not needs_full_funding and self.prng.random() > 0.5

        major_field = self.prng.choice([e.value for e in MajorField])
        interest = self.prng.choice(self.research_interests)

        personal_statement = self.prng.choice(
            self.personal_statement_templates
        ).format(
            field=major_field,
            interest=interest,
            track=track,
            country=nationality,
            activity=self.prng.choice(
                ["research", "community service", "innovation"]
            ),
        )

        future_goals = self.prng.choice(self.future_goals_templates).format(
            field=major_field,
            interest=interest,
            region=self.prng.choice(
                ["asia", "europe", "north_america"]
            ).capitalize(),
        )

        achievements = []
        for _ in range(self.rng.randint(1, 4)):
            tpl = self.prng.choice(self.achievement_templates)
            achievements.append(
                tpl.format(
                    level=self.prng.choice(
                        ["school", "city", "provincial", "national"]
                    ),
                    subject=self.prng.choice(
                        [e.value for e in OlympiadSubject]
                    ),
                    competition=self.prng.choice(
                        ["math", "science", "debate"]
                    ),
                    team_size=self.rng.randint(3, 15),
                    hours=self.rng.randint(20, 200),
                    position=self.prng.choice(
                        ["president", "vice president", "secretary"]
                    ),
                )
            )
        achievements_narrative = ". ".join(achievements) + "."

        schools = self.school_names.get(nationality, ["General High School"])
        school = self.prng.choice(schools)

        return Student(
            student_id=student_id,
            nationality=nationality,
            age=age,
            current_degree_level="high_school",
            target_degree_level="bachelors",
            high_school_track=track,
            school_name=school,
            overall_report_card_average=overall_avg,
            math_score=math_score,
            english_score=english_score,
            major_subject_average=major_avg,
            language_proficiency=language_proficiency,
            olympiad_level=olympiad_level,
            olympiad_subjects=olympiad_subjects,
            leadership_experience_count=leadership_count,
            volunteer_experience_count=volunteer_count,
            competition_wins_count=competition_wins,
            school_tier=school_tier,
            family_income_category=income,
            from_underrepresented_region=underrepresented,
            intended_career_track=career_track,
            willing_to_return_home=willing_return,
            target_countries=target_countries,
            personal_statement=personal_statement,
            achievements_narrative=achievements_narrative,
            future_goals=future_goals,
            needs_full_funding=needs_full_funding,
            can_self_fund_living=can_self_fund_living,
        )

    def generate_students(self) -> List[Student]:
        students = []
        for i in range(self.num_students):
            student_id = f"STU_{i:06d}"
            students.append(self.generate_student(student_id))
        return students

    def _simulate_affinity(
        self, student: Student, scholarship: Scholarship
    ) -> float:
        _COUNTRY_REGION: Dict[str, str] = {
            country: region
            for region, countries in COUNTRIES_BY_REGION.items()
            for country in countries
        }

        # --- elig component ---
        eligible = True
        if (
            scholarship.eligible_nationalities
            and student.nationality not in scholarship.eligible_nationalities
        ):
            eligible = False
        if student.age < scholarship.min_age or student.age > scholarship.max_age:
            eligible = False
        if (
            scholarship.eligible_degree_levels
            and student.current_degree_level not in scholarship.eligible_degree_levels
        ):
            eligible = False
        elig = 1.0 if eligible else 0.05

        # --- field component ---
        if not scholarship.eligible_high_school_tracks:
            field = 0.88  # no restriction → mild bonus
        elif student.high_school_track in scholarship.eligible_high_school_tracks:
            field = 1.0
        else:
            field = 0.60

        # --- country component ---
        sch_region = _COUNTRY_REGION.get(scholarship.host_country, "")
        stu_region = _COUNTRY_REGION.get(student.nationality, "")
        if student.target_countries and scholarship.host_country in student.target_countries:
            country = 1.0
        elif sch_region and stu_region and sch_region == stu_region:
            country = 0.85  # same region as student's home country
        else:
            country = 0.72

        # --- funding component ---
        if student.needs_full_funding:
            funding = 1.0 if scholarship.funding_coverage.is_full_funding else 0.75
        else:
            funding = 1.0 if not scholarship.requires_financial_need else 0.88

        # --- academic component ---
        # 0.68 at/below min, rises to 1.0 at +15 pts above min
        academic_gap = (
            student.overall_report_card_average
            - scholarship.min_report_card_average
        )
        academic = 0.68 + 0.32 * float(np.clip((academic_gap + 10.0) / 25.0, 0.0, 1.0))

        affinity = elig * field * country * funding * academic
        return float(np.clip(affinity, 0.0, 1.0))

    def generate_feedback(
        self,
        students: List[Student],
        scholarships: List[Scholarship],
        n_per_student: int = 5,
    ) -> List[Feedback]:
        base_date = datetime(2024, 1, 1)
        sch_ids = [s.scholarship_id for s in scholarships]
        n_sch = len(scholarships)
        feedbacks: List[Feedback] = []

        for student in students:
            affinities = np.array(
                [self._simulate_affinity(student, sch) for sch in scholarships],
                dtype=np.float64,
            )

            affinities = np.clip(affinities, 1e-9, None)
            probs = affinities / affinities.sum()

            n_pick = min(n_per_student, n_sch)
            chosen_indices = self.rng.choice(n_sch, size=n_pick, replace=False, p=probs)

            for idx in chosen_indices:
                aff = affinities[idx]
                if aff >= 0.6:
                    weights = _HIGH_AFF
                elif aff >= 0.3:
                    weights = _MID_AFF
                else:
                    weights = _LOW_AFF

                fb_type = self.prng.choices(_FB_TYPES, weights=weights)[0]

                offset_days = self.rng.randint(0, 365)
                offset_hours = self.rng.randint(0, 23)
                timestamp = (
                    base_date + timedelta(days=int(offset_days), hours=int(offset_hours))
                ).strftime("%Y-%m-%dT%H:%M:%SZ")

                feedbacks.append(
                    Feedback(
                        student_id=student.student_id,
                        scholarship_id=sch_ids[idx],
                        feedback_type=fb_type,
                        timestamp=timestamp,
                    )
                )

        return feedbacks

def main():
    numStudents = 20000
    generatorEngine = DatasetGenerator(num_students=numStudents, seed=42)
    
    scholarshipsList = generatorEngine.generate_scholarships()
    studentsList = generatorEngine.generate_students()
    feedbacksList = generatorEngine.generate_feedback(studentsList, scholarshipsList)
    
    save_to_csv(studentsList, scholarshipsList, feedbacksList)
    print("Dataset generation complete!")


if __name__ == "__main__":
    main()