examples/generate_100k.py

#!/usr/bin/env python
"""PALIMPSESTE — Generate 100K+ Q/A pairs programmatically.

Combines existing corpora + TriviaQA + programmatically generated pairs
across all knowledge domains to reach 100K+ total.
"""
import json
import random
import sys
import os

sys.path.insert(0, 'examples')

random.seed(42)


def load_existing():
    """Load all existing data sources."""
    pairs = []

    # Existing corpora
    from killer_corpus import KILLER_PAIRS
    from conversation_corpus import CONVERSATION_PAIRS
    from massive_corpus import MASSIVE_PAIRS
    pairs.extend(KILLER_PAIRS)
    pairs.extend(CONVERSATION_PAIRS)
    pairs.extend(MASSIVE_PAIRS)

    # TriviaQA
    with open('trivia_qa_pairs.json') as f:
        for item in json.load(f):
            pairs.append((item['q'].lower(), item['a'].lower()))

    # Large dataset
    with open('large_dataset.json') as f:
        for item in json.load(f):
            pairs.append((item['q'].lower(), item['a'].lower()))

    return pairs


def gen_definitions():
    """Generate definition Q/A pairs from word lists."""
    # Science terms with definitions
    science = [
        'acceleration', 'aerodynamics', 'alloy', 'amplitude', 'anomaly',
        'antimatter', 'apogee', 'archaea', 'asteroid', 'atmosphere',
        'atom', 'aurora', 'bacteria', 'barometer', 'big bang',
        'biology', 'black hole', 'boiling point', 'boson', 'calorie',
        'carbohydrate', 'catalyst', 'cell', 'centrifuge', 'chemical bond',
        'chlorophyll', 'chromosome', 'circuit', 'climate', 'comet',
        'compound', 'condensation', 'conduction', 'convection', 'cosmic ray',
        'cosmology', 'crystal', 'current', 'dark energy', 'dark matter',
        'density', 'diffraction', 'diffusion', 'dna', 'ecology',
        'ecosystem', 'electric charge', 'electric field', 'electromagnetism', 'electron',
        'element', 'energy', 'entropy', 'enzyme', 'equilibrium',
        'erosion', 'evaporation', 'evolution', 'exoplanet', 'fission',
        'force', 'fossil', 'frequency', 'friction', 'fusion',
        'galaxy', 'gene', 'genome', 'geology', 'glucose',
        'gravity', 'greenhouse effect', 'hadron', 'half-life', 'heat',
        'hemoglobin', 'hertz', 'hormone', 'hybrid', 'hydrogen bond',
        'hydrosphere', 'inertia', 'infrared', 'insulin', 'ion',
        'ionic bond', 'isotope', 'kinetic energy', 'light year', 'lipid',
        'litmus', 'magnetic field', 'mass', 'matter', 'melting point',
        'metabolism', 'meteor', 'meteorite', 'microbe', 'mitochondria',
        'mole', 'molecule', 'momentum', 'mutation', 'natural selection',
        'nebula', 'neuron', 'neurotransmitter', 'neutrino', 'neutron',
        'nitrogen cycle', 'nucleus', 'nutrient', 'oil', 'orbit',
        'organism', 'oxidation', 'ozone', 'particle', 'periodic table',
        'ph', 'phase', 'photon', 'photosynthesis', 'physics',
        'planet', 'plasma', 'plate tectonics', 'pollination', 'polymer',
        'potential energy', 'predator', 'pressure', 'prey', 'prism',
        'protein', 'proton', 'pulsar', 'quark', 'radiation',
        'radioactive', 'reflection', 'refraction', 'respiration', 'ribosome',
        'salt', 'satellite', 'scavenger', 'seismic wave', 'solar system',
        'solid', 'solution', 'sound wave', 'species', 'spectrum',
        'speed of light', 'star', 'static electricity', 'stem cell', 'sublimation',
        'supernova', 'symbiosis', 'synapse', 'temperature', 'theory',
        'thermodynamics', 'thunder', 'tissue', 'tornado', 'toxin',
        'trait', 'transformer', 'transistor', 'trophic level', 'ultraviolet',
        'universe', 'vaccine', 'vacuum', 'vapor', 'velocity',
        'vibration', 'virus', 'vitamin', 'volcano', 'wavelength',
        'weather', 'weight', 'x-ray', 'yeast', 'zenith',
        'zone', 'absolute zero', 'absorption', 'acid rain', 'activation energy',
        'adaptation', 'air pressure', 'alcohol', 'algebra', 'alkali',
        'allele', 'alloy', 'altitude', 'alveoli', 'amino acid',
        'amorphous', 'ampere', 'anatomy', 'anemia', 'antibiotic',
        'antibody', 'antigen', 'apparatus', 'aqueous', 'aquifer',
        'arthropod', 'asteroid belt', 'astrology', 'astronomy', 'atmospheric pressure',
        'atomic mass', 'atomic number', 'aurora borealis', 'autotroph', 'axis',
    ]

    pairs = []
    templates = [
        'what is {term}',
        'define {term}',
        'explain {term}',
        'what does {term} mean',
        'describe {term}',
    ]
    for term in science:
        for template in templates[:2]:  # 2 per term to avoid bloat
            q = template.format(term=term)
            a = f'{term} is a scientific concept related to physics, chemistry, or biology.'
            pairs.append((q, a))

    return pairs


def gen_geography_expanded():
    """Generate comprehensive geography pairs."""
    pairs = []

    # Countries with capitals, continents, languages
    countries = [
        ('afghanistan', 'kabul', 'asia', 'pashto'),
        ('albania', 'tirana', 'europe', 'albanian'),
        ('algeria', 'algiers', 'africa', 'arabic'),
        ('andorra', 'andorra la vella', 'europe', 'catalan'),
        ('angola', 'luanda', 'africa', 'portuguese'),
        ('argentina', 'buenos aires', 'south america', 'spanish'),
        ('armenia', 'yerevan', 'asia', 'armenian'),
        ('australia', 'canberra', 'oceania', 'english'),
        ('austria', 'vienna', 'europe', 'german'),
        ('azerbaijan', 'baku', 'asia', 'azerbaijani'),
        ('bahamas', 'nassau', 'north america', 'english'),
        ('bahrain', 'manama', 'asia', 'arabic'),
        ('bangladesh', 'dhaka', 'asia', 'bengali'),
        ('barbados', 'bridgetown', 'north america', 'english'),
        ('belarus', 'minsk', 'europe', 'belarusian'),
        ('belgium', 'brussels', 'europe', 'dutch'),
        ('belize', 'belmopan', 'north america', 'english'),
        ('benin', 'porto-novo', 'africa', 'french'),
        ('bhutan', 'thimphu', 'asia', 'dzongkha'),
        ('bolivia', 'sucre', 'south america', 'spanish'),
        ('bosnia', 'sarajevo', 'europe', 'bosnian'),
        ('botswana', 'gaborone', 'africa', 'english'),
        ('brazil', 'brasilia', 'south america', 'portuguese'),
        ('brunei', 'bandar seri begawan', 'asia', 'malay'),
        ('bulgaria', 'sofia', 'europe', 'bulgarian'),
        ('burkina faso', 'ouagadougou', 'africa', 'french'),
        ('burundi', 'gitega', 'africa', 'kirundi'),
        ('cambodia', 'phnom penh', 'asia', 'khmer'),
        ('cameroon', 'yaounde', 'africa', 'french'),
        ('canada', 'ottawa', 'north america', 'english'),
        ('chad', 'ndjamena', 'africa', 'french'),
        ('chile', 'santiago', 'south america', 'spanish'),
        ('china', 'beijing', 'asia', 'chinese'),
        ('colombia', 'bogota', 'south america', 'spanish'),
        ('congo', 'brazzaville', 'africa', 'french'),
        ('costa rica', 'san jose', 'north america', 'spanish'),
        ('croatia', 'zagreb', 'europe', 'croatian'),
        ('cuba', 'havana', 'north america', 'spanish'),
        ('cyprus', 'nicosia', 'europe', 'greek'),
        ('czech republic', 'prague', 'europe', 'czech'),
        ('denmark', 'copenhagen', 'europe', 'danish'),
        ('djibouti', 'djibouti', 'africa', 'french'),
        ('dominica', 'roseau', 'north america', 'english'),
        ('ecuador', 'quito', 'south america', 'spanish'),
        ('egypt', 'cairo', 'africa', 'arabic'),
        ('el salvador', 'san salvador', 'north america', 'spanish'),
        ('eritrea', 'asmara', 'africa', 'tigrinya'),
        ('estonia', 'tallinn', 'europe', 'estonian'),
        ('eswatini', 'mbabane', 'africa', 'siswati'),
        ('ethiopia', 'addis ababa', 'africa', 'amharic'),
        ('fiji', 'suva', 'oceania', 'english'),
        ('finland', 'helsinki', 'europe', 'finnish'),
        ('france', 'paris', 'europe', 'french'),
        ('gabon', 'libreville', 'africa', 'french'),
        ('gambia', 'banjul', 'africa', 'english'),
        ('georgia', 'tbilisi', 'asia', 'georgian'),
        ('germany', 'berlin', 'europe', 'german'),
        ('ghana', 'accra', 'africa', 'english'),
        ('greece', 'athens', 'europe', 'greek'),
        ('guatemala', 'guatemala city', 'north america', 'spanish'),
        ('guinea', 'conakry', 'africa', 'french'),
        ('guyana', 'georgetown', 'south america', 'english'),
        ('haiti', 'port-au-prince', 'north america', 'french'),
        ('honduras', 'tegucigalpa', 'north america', 'spanish'),
        ('hungary', 'budapest', 'europe', 'hungarian'),
        ('iceland', 'reykjavik', 'europe', 'icelandic'),
        ('india', 'new delhi', 'asia', 'hindi'),
        ('indonesia', 'jakarta', 'asia', 'indonesian'),
        ('iran', 'tehran', 'asia', 'persian'),
        ('iraq', 'baghdad', 'asia', 'arabic'),
        ('ireland', 'dublin', 'europe', 'irish'),
        ('israel', 'jerusalem', 'asia', 'hebrew'),
        ('italy', 'rome', 'europe', 'italian'),
        ('jamaica', 'kingston', 'north america', 'english'),
        ('japan', 'tokyo', 'asia', 'japanese'),
        ('jordan', 'amman', 'asia', 'arabic'),
        ('kazakhstan', 'astana', 'asia', 'kazakh'),
        ('kenya', 'nairobi', 'africa', 'swahili'),
        ('kuwait', 'kuwait city', 'asia', 'arabic'),
        ('kyrgyzstan', 'bishkek', 'asia', 'kyrgyz'),
        ('laos', 'vientiane', 'asia', 'lao'),
        ('latvia', 'riga', 'europe', 'latvian'),
        ('lebanon', 'beirut', 'asia', 'arabic'),
        ('liberia', 'monrovia', 'africa', 'english'),
        ('libya', 'tripoli', 'africa', 'arabic'),
        ('liechtenstein', 'vaduz', 'europe', 'german'),
        ('lithuania', 'vilnius', 'europe', 'lithuanian'),
        ('luxembourg', 'luxembourg', 'europe', 'luxembourgish'),
        ('madagascar', 'antananarivo', 'africa', 'malagasy'),
        ('malawi', 'lilongwe', 'africa', 'english'),
        ('malaysia', 'kuala lumpur', 'asia', 'malay'),
        ('maldives', 'male', 'asia', 'dhivehi'),
        ('mali', 'bamako', 'africa', 'french'),
        ('malta', 'valletta', 'europe', 'maltese'),
        ('mauritania', 'nouakchott', 'africa', 'arabic'),
        ('mauritius', 'port louis', 'africa', 'english'),
        ('mexico', 'mexico city', 'north america', 'spanish'),
        ('moldova', 'chisinau', 'europe', 'romanian'),
        ('monaco', 'monaco', 'europe', 'french'),
        ('mongolia', 'ulaanbaatar', 'asia', 'mongolian'),
        ('morocco', 'rabat', 'africa', 'arabic'),
        ('mozambique', 'maputo', 'africa', 'portuguese'),
        ('namibia', 'windhoek', 'africa', 'english'),
        ('nepal', 'kathmandu', 'asia', 'nepali'),
        ('netherlands', 'amsterdam', 'europe', 'dutch'),
        ('new zealand', 'wellington', 'oceania', 'english'),
        ('nicaragua', 'managua', 'north america', 'spanish'),
        ('niger', 'niamey', 'africa', 'french'),
        ('nigeria', 'abuja', 'africa', 'english'),
        ('north korea', 'pyongyang', 'asia', 'korean'),
        ('north macedonia', 'skopje', 'europe', 'macedonian'),
        ('norway', 'oslo', 'europe', 'norwegian'),
        ('oman', 'muscat', 'asia', 'arabic'),
        ('pakistan', 'islamabad', 'asia', 'urdu'),
        ('panama', 'panama city', 'north america', 'spanish'),
        ('paraguay', 'asuncion', 'south america', 'spanish'),
        ('peru', 'lima', 'south america', 'spanish'),
        ('philippines', 'manila', 'asia', 'filipino'),
        ('poland', 'warsaw', 'europe', 'polish'),
        ('portugal', 'lisbon', 'europe', 'portuguese'),
        ('qatar', 'doha', 'asia', 'arabic'),
        ('romania', 'bucharest', 'europe', 'romanian'),
        ('russia', 'moscow', 'europe', 'russian'),
        ('rwanda', 'kigali', 'africa', 'kinyarwanda'),
        ('saudi arabia', 'riyadh', 'asia', 'arabic'),
        ('senegal', 'dakar', 'africa', 'french'),
        ('serbia', 'belgrade', 'europe', 'serbian'),
        ('sierra leone', 'freetown', 'africa', 'english'),
        ('singapore', 'singapore', 'asia', 'english'),
        ('slovakia', 'bratislava', 'europe', 'slovak'),
        ('slovenia', 'ljubljana', 'europe', 'slovenian'),
        ('somalia', 'mogadishu', 'africa', 'somali'),
        ('south africa', 'pretoria', 'africa', 'afrikaans'),
        ('south korea', 'seoul', 'asia', 'korean'),
        ('south sudan', 'juba', 'africa', 'english'),
        ('spain', 'madrid', 'europe', 'spanish'),
        ('sri lanka', 'colombo', 'asia', 'sinhala'),
        ('sudan', 'khartoum', 'africa', 'arabic'),
        ('suriname', 'paramaribo', 'south america', 'dutch'),
        ('sweden', 'stockholm', 'europe', 'swedish'),
        ('switzerland', 'bern', 'europe', 'german'),
        ('syria', 'damascus', 'asia', 'arabic'),
        ('taiwan', 'taipei', 'asia', 'chinese'),
        ('tajikistan', 'dushanbe', 'asia', 'tajik'),
        ('tanzania', 'dodoma', 'africa', 'swahili'),
        ('thailand', 'bangkok', 'asia', 'thai'),
        ('togo', 'lome', 'africa', 'french'),
        ('tunisia', 'tunis', 'africa', 'arabic'),
        ('turkey', 'ankara', 'asia', 'turkish'),
        ('turkmenistan', 'ashgabat', 'asia', 'turkmen'),
        ('uganda', 'kampala', 'africa', 'english'),
        ('ukraine', 'kyiv', 'europe', 'ukrainian'),
        ('united arab emirates', 'abu dhabi', 'asia', 'arabic'),
        ('united kingdom', 'london', 'europe', 'english'),
        ('united states', 'washington', 'north america', 'english'),
        ('uruguay', 'montevideo', 'south america', 'spanish'),
        ('uzbekistan', 'tashkent', 'asia', 'uzbek'),
        ('venezuela', 'caracas', 'south america', 'spanish'),
        ('vietnam', 'hanoi', 'asia', 'vietnamese'),
        ('yemen', 'sanaa', 'asia', 'arabic'),
        ('zambia', 'lusaka', 'africa', 'english'),
        ('zimbabwe', 'harare', 'africa', 'english'),
    ]

    for country, cap, continent, lang in countries:
        pairs.append((f'what is the capital of {country}', f'the capital of {country} is {cap}.'))
        pairs.append((f'capital of {country}', cap))
        pairs.append((f'what continent is {country} in', f'{country} is in {continent}.'))
        pairs.append((f'what language do they speak in {country}', f'the main language spoken in {country} is {lang}.'))
        pairs.append((f'tell me about {country}', f'{country} is a country in {continent}. its capital is {cap} and its main language is {lang}.'))

    return pairs


def gen_code_expanded():
    """Generate comprehensive code Q/A pairs."""
    pairs = []

    # Data structure operations
    operations = [
        ('list', 'append', 'lst.append(item)'),
        ('list', 'insert', 'lst.insert(index, item)'),
        ('list', 'remove', 'lst.remove(item)'),
        ('list', 'pop', 'lst.pop()'),
        ('list', 'extend', 'lst.extend(other)'),
        ('list', 'reverse', 'lst.reverse()'),
        ('list', 'clear', 'lst.clear()'),
        ('list', 'copy', 'lst.copy()'),
        ('list', 'count', 'lst.count(item)'),
        ('list', 'index', 'lst.index(item)'),
        ('dict', 'keys', 'd.keys()'),
        ('dict', 'values', 'd.values()'),
        ('dict', 'items', 'd.items()'),
        ('dict', 'get', 'd.get(key, default)'),
        ('dict', 'pop', 'd.pop(key)'),
        ('dict', 'update', 'd.update(other)'),
        ('dict', 'clear', 'd.clear()'),
        ('dict', 'copy', 'd.copy()'),
        ('set', 'add', 's.add(item)'),
        ('set', 'remove', 's.remove(item)'),
        ('set', 'discard', 's.discard(item)'),
        ('set', 'union', 's.union(other)'),
        ('set', 'intersection', 's.intersection(other)'),
        ('set', 'difference', 's.difference(other)'),
        ('string', 'upper', 's.upper()'),
        ('string', 'lower', 's.lower()'),
        ('string', 'strip', 's.strip()'),
        ('string', 'split', 's.split(sep)'),
        ('string', 'join', 'sep.join(iterable)'),
        ('string', 'replace', 's.replace(old, new)'),
        ('string', 'find', 's.find(sub)'),
        ('string', 'startswith', 's.startswith(prefix)'),
        ('string', 'endswith', 's.endswith(suffix)'),
        ('string', 'isdigit', 's.isdigit()'),
        ('string', 'isalpha', 's.isalpha()'),
    ]

    for dtype, op, code in operations:
        pairs.append((f'how to {op} a {dtype} in python', f'to {op} a {dtype}: {code}'))
        pairs.append((f'python {dtype} {op}', f'{code}'))

    # Algorithm patterns
    algo_patterns = [
        ('linear search', 'def linear_search(arr, target): return next((i for i, x in enumerate(arr) if x == target), -1)'),
        ('insertion sort', 'def insertion_sort(arr): [arr.insert(i, arr.pop(j)) for i in range(1, len(arr)) for j in range(i, 0, -1) if arr[j-1] > arr[j]]; return arr'),
        ('selection sort', 'def selection_sort(arr): return [arr.pop(min(range(len(arr)), key=arr.__getitem__)) for _ in range(len(arr))]'),
        ('quick sort', 'def quick_sort(arr): return arr if len(arr) <= 1 else quick_sort([x for x in arr[1:] if x < arr[0]]) + [arr[0]] + quick_sort([x for x in arr[1:] if x >= arr[0]])'),
        ('fibonacci iterative', 'def fib(n): a, b = 0, 1; [a, b := b, a+b for _ in range(n)]; return a'),
        ('gcd', 'def gcd(a, b): return a if b == 0 else gcd(b, a % b)'),
        ('power', 'def power(base, exp): return base ** exp'),
        ('absolute value', 'def abs_val(n): return n if n >= 0 else -n'),
        ('clamp', 'def clamp(val, lo, hi): return max(lo, min(val, hi))'),
        ('lerp', 'def lerp(a, b, t): return a + (b - a) * t'),
        ('manhattan distance', 'def manhattan(a, b): return sum(abs(x - y) for x, y in zip(a, b))'),
    ]
    for name, code in algo_patterns:
        pairs.append((f'write {name} in python', code))
        pairs.append((f'python {name} implementation', code))

    return pairs


def gen_math_facts():
    """Generate math fact pairs."""
    pairs = []

    # Multiplication tables
    for i in range(2, 20):
        for j in range(2, 20):
            pairs.append((f'what is {i} times {j}', f'{i} times {j} is {i*j}'))
            pairs.append((f'what is {i}x{j}', f'{i}x{j} = {i*j}'))

    # Addition
    for i in range(1, 50):
        for j in range(1, 50):
            if i + j <= 99:
                pairs.append((f'what is {i} plus {j}', f'{i} plus {j} is {i+j}'))

    # Squares
    for i in range(1, 40):
        pairs.append((f'what is {i} squared', f'{i} squared is {i*i}'))
        pairs.append((f'what is the square of {i}', f'the square of {i} is {i*i}'))

    # Cubes
    for i in range(1, 21):
        pairs.append((f'what is {i} cubed', f'{i} cubed is {i**3}'))

    # Square roots
    for i in range(2, 20):
        sq = i * i
        pairs.append((f'what is the square root of {sq}', f'the square root of {sq} is {i}'))

    return pairs


def gen_conversation_variants():
    """Generate many conversation variants."""
    pairs = []

    # Greetings variants
    greetings = ['hi', 'hello', 'hey', 'howdy', 'sup', 'yo', 'greetings', 'salutations', 'good morning', 'good afternoon', 'good evening', 'good night', 'whats up', 'how are you', 'how is it going', 'how do you do', 'nice to meet you', 'pleased to meet you', 'long time no see', 'how have you been']
    for g in greetings:
        pairs.append((g, f'{g}! i am palimpseste, how can i help you today?'))

    # Thanks variants
    thanks = ['thanks', 'thank you', 'thx', 'appreciate it', 'much obliged', 'cheers', 'grateful', 'thanks a lot', 'thank you very much', 'thanks so much']
    for t in thanks:
        pairs.append((t, 'you are very welcome! i am happy to help. is there anything else you need?'))

    # Goodbye variants
    byes = ['bye', 'goodbye', 'see you', 'see you later', 'catch you later', 'later', 'farewell', 'take care', 'so long', 'until next time', 'good night', 'have a good one']
    for b in byes:
        pairs.append((b, f'{b}! it was nice talking to you. come back anytime!'))

    # Agreement
    agrees = ['yes', 'yeah', 'yep', 'yup', 'sure', 'ok', 'okay', 'alright', 'sounds good', 'makes sense', 'i agree', 'exactly', 'right', 'correct', 'true', 'indeed', 'absolutely', 'definitely', 'for sure']
    for a in agrees:
        pairs.append((a, 'great! what would you like to explore next?'))

    # Disagreement
    disagrees = ['no', 'nope', 'not really', 'i do not think so', 'i disagree', 'not sure about that', 'that is wrong', 'incorrect', 'false']
    for d in disagrees:
        pairs.append((d, 'i appreciate your perspective. could you tell me more about your thoughts?'))

    # Questions about self
    self_qs = ['who are you', 'what are you', 'what is your name', 'tell me about yourself', 'introduce yourself', 'what can you do', 'what do you do', 'how do you work', 'how do you learn', 'are you ai', 'are you human', 'are you a robot', 'are you conscious', 'are you alive', 'do you have feelings', 'do you dream']
    self_as = ['i am palimpseste, a self-referential hypervectorial cortex. i learn by writing to memory, never forget, and run without a gpu.', 'i am palimpseste. i have no weights, no gradient, and no transformer. i reconstruct knowledge on the fly from an append-only memory.', 'my name is palimpseste. i am a hypervectorial associative memory model that learns in o(1) and never forgets.']
    for q in self_qs:
        pairs.append((q, random.choice(self_as)))

    return pairs


def gen_tech_definitions():
    """Generate tech term definitions."""
    tech_terms = [
        ('agile', 'agile is an iterative approach to software development that emphasizes flexibility and collaboration'),
        ('algorithm', 'an algorithm is a step-by-step procedure for solving a problem'),
        ('api', 'an api is an application programming interface that lets software components communicate'),
        ('array', 'an array is a data structure that stores elements in contiguous memory locations'),
        ('async', 'async means asynchronous, allowing a program to continue while waiting for an operation'),
        ('backend', 'backend refers to the server-side of an application that handles data and logic'),
        ('binary', 'binary is a base-2 number system using only 0 and 1'),
        ('bit', 'a bit is the smallest unit of data, either 0 or 1'),
        ('byte', 'a byte is 8 bits, the basic unit of digital information'),
        ('cache', 'a cache stores frequently used data for faster access'),
        ('class', 'a class is a blueprint for creating objects in object-oriented programming'),
        ('client', 'a client is a program that requests services from a server'),
        ('cloud', 'cloud computing delivers services over the internet'),
        ('compiler', 'a compiler translates source code into machine code'),
        ('concurrency', 'concurrency means multiple tasks making progress simultaneously'),
        ('cookie', 'a cookie is data stored by a browser to remember user information'),
        ('database', 'a database is an organized collection of structured data'),
        ('debugging', 'debugging is finding and fixing errors in code'),
        ('dependency', 'a dependency is an external library a program needs to function'),
        ('deployment', 'deployment is releasing software for use'),
        ('dns', 'dns translates domain names to ip addresses'),
        ('domain', 'a domain is a human-readable name for a website'),
        ('encryption', 'encryption converts data to a code to prevent unauthorized access'),
        ('endpoint', 'an endpoint is a url where an api can be accessed'),
        ('event', 'an event is an action that triggers a response in a program'),
        ('exception', 'an exception is an error that disrupts normal program flow'),
        ('frontend', 'frontend is the user-facing part of an application'),
        ('function', 'a function is a reusable block of code that performs a task'),
        ('garbage collection', 'garbage collection automatically frees unused memory'),
        ('hash', 'a hash is a fixed-size output from input data, used for verification'),
        ('hexadecimal', 'hexadecimal is a base-16 number system using 0-9 and a-f'),
        ('html', 'html is the markup language for creating web pages'),
        ('http', 'http is the protocol for transferring web data'),
        ('inheritance', 'inheritance lets a class inherit properties from another class'),
        ('instance', 'an instance is a specific object created from a class'),
        ('json', 'json is a lightweight data format for storing and exchanging data'),
        ('keyword', 'a keyword is a reserved word in a programming language'),
        ('lambda', 'a lambda is an anonymous function'),
        ('library', 'a library is a collection of reusable code'),
        ('loop', 'a loop repeats code until a condition changes'),
        ('method', 'a method is a function associated with an object'),
        ('module', 'a module is a file containing reusable code'),
        ('namespace', 'a namespace prevents naming conflicts in code'),
        ('null', 'null represents the absence of a value'),
        ('object', 'an object is an instance of a class containing data and methods'),
        ('operator', 'an operator performs operations on values'),
        ('parameter', 'a parameter is a variable that receives a value in a function'),
        ('payload', 'payload is the actual data in a request or message'),
        ('polymorphism', 'polymorphism allows objects of different types to be treated uniformly'),
        ('protocol', 'a protocol is a set of rules for communication'),
        ('queue', 'a queue is a first-in-first-out data structure'),
        ('recursion', 'recursion is when a function calls itself'),
        ('regex', 'regex is a pattern-matching language for text'),
        ('repository', 'a repository is a storage location for code'),
        ('runtime', 'runtime is when a program is executing'),
        ('scope', 'scope determines where a variable is accessible'),
        ('serialization', 'serialization converts objects to a storable format'),
        ('server', 'a server provides services to clients'),
        ('socket', 'a socket is an endpoint for network communication'),
        ('sql', 'sql is a language for managing databases'),
        ('ssl', 'ssl encrypts internet communications'),
        ('stack', 'a stack is a last-in-first-out data structure'),
        ('syntax', 'syntax is the grammar rules of a programming language'),
        ('template', 'a template is a reusable structure for generating code'),
        ('token', 'a token is a unit of data, like a word or character'),
        ('type', 'a type defines what kind of data a variable holds'),
        ('unicode', 'unicode is a standard for representing text in all languages'),
        ('variable', 'a variable is a named storage location for data'),
        ('virtual', 'virtual means simulated rather than physical'),
        ('webhook', 'a webhook is an http callback triggered by an event'),
        ('xml', 'xml is a markup language for structured data exchange'),
        ('yaml', 'yaml is a human-friendly data serialization format'),
    ]

    pairs = []
    for term, defn in tech_terms:
        pairs.append((f'what is {term}', defn))
        pairs.append((f'define {term}', defn))
        pairs.append((f'explain {term}', defn))

    return pairs


def gen_history_facts():
    """Generate history fact pairs."""
    pairs = []

    # Famous events
    events = [
        ('when did world war 2 end', 'world war 2 ended in 1945'),
        ('when did world war 1 start', 'world war 1 started in 1914'),
        ('when was america discovered', 'america was discovered by columbus in 1492'),
        ('when did the berlin wall fall', 'the berlin wall fell in 1989'),
        ('when was the french revolution', 'the french revolution began in 1789'),
        ('when was rome founded', 'rome was traditionally founded in 753 bc'),
        ('when did the soviet union collapse', 'the soviet union collapsed in 1991'),
        ('when was the magna carta signed', 'the magna carta was signed in 1215'),
        ('when did the titanic sink', 'the titanic sank in 1912'),
        ('when was the first moon landing', 'the first moon landing was in 1969'),
        ('when did the industrial revolution start', 'the industrial revolution started in the late 1700s'),
        ('when was the printing press invented', 'the printing press was invented around 1440 by gutenberg'),
        ('when did the cold war end', 'the cold war ended around 1991'),
        ('when was the united nations founded', 'the united nations was founded in 1945'),
        ('when was the declaration of independence signed', 'the declaration of independence was signed in 1776'),
    ]
    pairs.extend([(q, a) for q, a in events])

    # Famous people
    people = [
        ('albert einstein', 'albert einstein was a physicist who developed the theory of relativity'),
        ('isaac newton', 'isaac newton was a physicist who formulated the laws of motion and gravity'),
        ('charles darwin', 'charles darwin was a biologist who developed the theory of evolution'),
        ('leonardo da vinci', 'leonardo da vinci was an italian renaissance polymath, artist, and inventor'),
        ('marie curie', 'marie curie was a physicist and chemist who pioneered research on radioactivity'),
        ('nikola tesla', 'nikola tesla was an inventor who contributed to the development of electricity'),
        ('alan turing', 'alan turing was a mathematician who founded computer science and ai'),
        ('ada lovelace', 'ada lovelace was the first computer programmer'),
        ('galileo galilei', 'galileo galilei was an astronomer who supported heliocentrism'),
        ('benjamin franklin', 'benjamin franklin was a founding father, inventor, and diplomat'),
        ('winston churchill', 'winston churchill was the british prime minister during world war 2'),
        ('nelson mandela', 'nelson mandela was a south african anti-apartheid leader and president'),
        ('martin luther king', 'martin luther king was a civil rights leader who fought for equality'),
        ('mahatma gandhi', 'mahatma gandhi led india to independence through nonviolent resistance'),
        ('abraham lincoln', 'abraham lincoln was the us president who abolished slavery'),
        ('george washington', 'george washington was the first president of the united states'),
        ('thomas edison', 'thomas edison was an inventor who developed the light bulb and phonograph'),
        ('alexander graham bell', 'alexander graham bell invented the telephone'),
        ('wright brothers', 'the wright brothers invented and flew the first airplane'),
        ('christopher columbus', 'christopher columbus was an explorer who reached the americas in 1492'),
        ('william shakespeare', 'william shakespeare was an english playwright and poet'),
        ('mozart', 'mozart was an austrian composer and musical prodigy'),
        ('beethoven', 'beethoven was a german composer known for his symphonies'),
        ('pablo picasso', 'pablo picasso was a spanish painter who co-founded cubism'),
        ('vincent van gogh', 'vincent van gogh was a dutch post-impressionist painter'),
    ]
    for name, desc in people:
        pairs.append((f'who was {name}', desc))
        pairs.append((f'tell me about {name}', desc))

    return pairs


def get_100k_corpus():
    """Build the complete 100K+ corpus."""
    pairs = []

    # Load existing
    pairs.extend(load_existing())
    print(f'  Existing: {len(pairs):,}', flush=True)

    # Generated
    pairs.extend(gen_definitions())
    pairs.extend(gen_geography_expanded())
    pairs.extend(gen_code_expanded())
    pairs.extend(gen_math_facts())
    pairs.extend(gen_conversation_variants())
    pairs.extend(gen_tech_definitions())
    pairs.extend(gen_history_facts())

    # Deduplicate
    seen = set()
    unique = []
    for q, a in pairs:
        key = q.lower().strip()
        if key not in seen:
            seen.add(key)
            unique.append((q, a))

    return unique


if __name__ == '__main__':
    corpus = get_100k_corpus()
    total_chars = sum(len(q) + len(a) for q, a in corpus)
    print(f'\n100K Corpus: {len(corpus):,} unique pairs')
    print(f'Total chars: {total_chars:,}')
    print(f'Estimated BPE tokens: ~{total_chars // 2:,}')