examples/build_mega_corpus.py

#!/usr/bin/env python
"""PALIMPSESTE — Build mega corpus for 2GB model.

Generates 8000+ Q/A pairs programmatically across all knowledge domains.
Combined with existing corpora + TriviaQA, targets 500K+ tokens → 2GB model.
"""
import json
import sys
import os
import random

sys.path.insert(0, 'examples')

# Import existing corpora
from killer_corpus import KILLER_PAIRS
from conversation_corpus import CONVERSATION_PAIRS
from massive_corpus import MASSIVE_PAIRS


# ================================================================
# PROGRAMMATIC KNOWLEDGE GENERATION
# ================================================================

def generate_science_pairs():
    """Generate science Q/A pairs from templates."""
    pairs = []

    # Physics concepts
    physics = [
        ('force', 'a force is a push or pull that changes the motion of an object'),
        ('energy', 'energy is the capacity to do work. it exists in many forms including kinetic and potential'),
        ('momentum', 'momentum is the product of mass and velocity. it is conserved in closed systems'),
        ('friction', 'friction is a force that opposes motion when two surfaces are in contact'),
        ('inertia', 'inertia is the resistance of an object to changes in its motion'),
        ('velocity', 'velocity is the rate of change of position with direction. it is speed with a direction'),
        ('acceleration', 'acceleration is the rate of change of velocity over time'),
        ('mass', 'mass is a measure of the amount of matter in an object'),
        ('weight', 'weight is the force of gravity on an object. it equals mass times gravitational acceleration'),
        ('density', 'density is mass per unit volume. it determines whether objects float or sink'),
        ('pressure', 'pressure is force applied per unit area'),
        ('work', 'work is force times distance moved in the direction of the force'),
        ('power', 'power is the rate of doing work. it equals work divided by time'),
        ('wavelength', 'wavelength is the distance between successive crests of a wave'),
        ('frequency', 'frequency is the number of wave cycles per second, measured in hertz'),
        ('amplitude', 'amplitude is the maximum displacement from the rest position in a wave'),
        ('refraction', 'refraction is the bending of light when it passes from one medium to another'),
        ('reflection', 'reflection is when light bounces off a surface at the same angle it arrived'),
        ('diffraction', 'diffraction is the spreading of waves around obstacles or through gaps'),
        ('interference', 'interference is when two waves combine to form a larger or smaller wave'),
        ('fission', 'nuclear fission is the splitting of a heavy atomic nucleus into lighter ones, releasing energy'),
        ('fusion', 'nuclear fusion is the joining of light nuclei to form a heavier one, releasing energy'),
        ('isotope', 'an isotope is a variant of an element with the same protons but different neutrons'),
        ('half-life', 'half-life is the time for half of a radioactive sample to decay'),
        ('conduction', 'conduction is heat transfer through direct contact between materials'),
        ('convection', 'convection is heat transfer by the movement of fluids'),
        ('radiation', 'radiation is heat transfer through electromagnetic waves without a medium'),
        ('superconductivity', 'superconductivity is zero electrical resistance below a critical temperature'),
        ('uncertainty principle', 'the uncertainty principle states that position and momentum cannot both be known precisely'),
        ('wave particle duality', 'wave particle duality means particles like electrons behave as both waves and particles'),
    ]
    for concept, definition in physics:
        pairs.append((f'what is {concept}', definition))
        pairs.append((f'define {concept}', definition))
        pairs.append((f'explain {concept}', definition))

    # Chemistry
    chemistry = [
        ('an element', 'an element is a substance made of only one type of atom'),
        ('a compound', 'a compound is a substance made of two or more elements chemically bonded'),
        ('a mixture', 'a mixture is a combination of substances that are not chemically bonded'),
        ('a molecule', 'a molecule is a group of atoms bonded together, the smallest unit of a compound'),
        ('an ion', 'an ion is an atom or molecule with an electric charge due to losing or gaining electrons'),
        ('a covalent bond', 'a covalent bond is a chemical bond where atoms share electrons'),
        ('an ionic bond', 'an ionic bond is a chemical bond formed by the attraction between oppositely charged ions'),
        ('acidity', 'acidity is a measure of hydrogen ion concentration, expressed as ph'),
        ('a catalyst', 'a catalyst is a substance that speeds up a reaction without being consumed'),
        ('oxidation', 'oxidation is a reaction where a substance loses electrons'),
        ('reduction', 'reduction is a reaction where a substance gains electrons'),
        ('a polymer', 'a polymer is a large molecule made of repeating structural units called monomers'),
        ('a hydrocarbon', 'a hydrocarbon is a compound made of only hydrogen and carbon'),
    ]
    for concept, definition in chemistry:
        pairs.append((f'what is {concept}', definition))
        pairs.append((f'define {concept}', definition))

    # Biology
    biology = [
        ('a cell membrane', 'the cell membrane is the outer layer that controls what enters and leaves a cell'),
        ('a mitochondria', 'mitochondria are organelles that produce energy in cells through respiration'),
        ('a protein', 'a protein is a molecule made of amino acids that performs functions in living organisms'),
        ('an enzyme', 'an enzyme is a protein that speeds up chemical reactions in living organisms'),
        ('photosynthesis', 'photosynthesis is the process where plants use sunlight to make glucose from co2 and water'),
        ('respiration', 'respiration is the process where cells break down glucose to release energy'),
        ('mitosis', 'mitosis is cell division that produces two identical daughter cells'),
        ('meiosis', 'meiosis is cell division that produces gametes with half the chromosomes'),
        ('a chromosome', 'a chromosome is a structure of dna and protein that carries genetic information'),
        ('a gene', 'a gene is a segment of dna that codes for a specific protein or trait'),
        ('a mutation', 'a mutation is a change in dna sequence that can affect traits'),
        ('natural selection', 'natural selection is the process where organisms with favorable traits survive and reproduce'),
        ('homeostasis', 'homeostasis is the maintenance of stable internal conditions in an organism'),
        ('a virus', 'a virus is an infectious agent that requires a host cell to reproduce'),
        ('a bacteria', 'bacteria are single-celled organisms without a nucleus that live everywhere'),
        ('an antibody', 'an antibody is a protein produced by the immune system to fight pathogens'),
    ]
    for concept, definition in biology:
        pairs.append((f'what is {concept}', definition))

    return pairs


def generate_tech_pairs():
    """Generate technology Q/A pairs."""
    pairs = []

    tech_concepts = [
        ('tcp', 'tcp is the transmission control protocol. it ensures reliable data delivery over networks'),
        ('udp', 'udp is the user datagram protocol. it is fast but does not guarantee delivery'),
        ('dns', 'dns is the domain name system. it translates domain names to ip addresses'),
        ('http', 'http is the hypertext transfer protocol used for web communication'),
        ('https', 'https is http with encryption via ssl or tls for secure communication'),
        ('ssl', 'ssl is secure sockets layer, a protocol for encrypting internet communications'),
        ('a firewall', 'a firewall is a network security system that controls incoming and outgoing traffic'),
        ('a proxy', 'a proxy is a server that acts as an intermediary for requests from clients'),
        ('a vpn', 'a vpn is a virtual private network that creates an encrypted connection over the internet'),
        ('a cookie', 'a cookie is a small file stored by websites on your device to remember information'),
        ('a cache', 'a cache is a temporary storage area for frequently accessed data to speed up retrieval'),
        ('a thread', 'a thread is the smallest unit of execution within a program'),
        ('a process', 'a process is an instance of a program running in memory'),
        ('a socket', 'a socket is an endpoint for communication between two programs over a network'),
        ('a port', 'a port is a communication endpoint identified by a number from 0 to 65535'),
        ('bandwidth', 'bandwidth is the maximum data transfer rate of a network connection'),
        ('latency', 'latency is the delay before data transfer begins following an instruction'),
        ('a framework', 'a framework is a reusable software platform that provides common functionality'),
        ('a library', 'a library is a collection of pre-written code that developers can reuse'),
        ('an ide', 'an ide is an integrated development environment for writing and debugging code'),
        ('a compiler', 'a compiler translates source code into machine code before execution'),
        ('an interpreter', 'an interpreter executes source code directly, line by line'),
        ('a debugger', 'a debugger is a tool that helps find and fix errors in programs'),
        ('a version control system', 'a version control system tracks changes to files over time and enables collaboration'),
        ('continuous integration', 'continuous integration is the practice of frequently merging code changes and running tests automatically'),
        ('a microservice', 'a microservice is a small independent service that communicates via apis'),
        ('a container', 'a container is a lightweight package containing an application and its dependencies'),
        ('kubernetes', 'kubernetes is a platform for managing containerized applications across clusters'),
        ('a rest api', 'a rest api is an api that follows representational state transfer principles using http methods'),
        ('graphql', 'graphql is a query language for apis that lets clients request exactly the data they need'),
    ]
    for concept, definition in tech_concepts:
        pairs.append((f'what is {concept}', definition))
        pairs.append((f'explain {concept}', definition))

    return pairs


def generate_code_pairs():
    """Generate code Q/A pairs across patterns."""
    pairs = []

    code_patterns = [
        ('write a python function to add two numbers', 'def add(a, b): return a + b'),
        ('write a python function to subtract two numbers', 'def subtract(a, b): return a - b'),
        ('write a python function to multiply two numbers', 'def multiply(a, b): return a * b'),
        ('write a python function to divide two numbers', 'def divide(a, b): return a / b if b != 0 else none'),
        ('write a python function to check even', 'def is_even(n): return n % 2 == 0'),
        ('write a python function to check odd', 'def is_odd(n): return n % 2 != 0'),
        ('write a python function to find maximum', 'def find_max(lst): return max(lst)'),
        ('write a python function to find minimum', 'def find_min(lst): return min(lst)'),
        ('write a python function to sum a list', 'def sum_list(lst): return sum(lst)'),
        ('write a python function to count elements', 'def count_items(lst): return len(lst)'),
        ('write a python function to get length', 'def length(s): return len(s)'),
        ('write a python function to convert to uppercase', 'def to_upper(s): return s.upper()'),
        ('write a python function to convert to lowercase', 'def to_lower(s): return s.lower()'),
        ('write a python function to strip whitespace', 'def strip_ws(s): return s.strip()'),
        ('write a python function to split by space', 'def split_words(s): return s.split()'),
        ('write a python function to join with space', 'def join_words(lst): return " ".join(lst)'),
        ('write a python function to check if empty', 'def is_empty(x): return len(x) == 0'),
        ('write a python function to get first element', 'def first(lst): return lst[0] if lst else none'),
        ('write a python function to get last element', 'def last(lst): return lst[-1] if lst else none'),
        ('write a python function to check substring', 'def contains(text, sub): return sub in text'),
        ('write a python function to replace text', 'def replace_text(s, old, new): return s.replace(old, new)'),
        ('write a python function to count occurrences', 'def count_occ(s, sub): return s.count(sub)'),
        ('write a python function to find index', 'def find_idx(lst, item): return lst.index(item) if item in lst else -1'),
        ('write a python function to chunk a list', 'def chunk(lst, size): return [lst[i:i+size] for i in range(0, len(lst), size)]'),
        ('write a python function to unique sort', 'def unique_sorted(lst): return sorted(set(lst))'),
        ('write a python function to zip two lists', 'def zip_lists(a, b): return list(zip(a, b))'),
        ('write a python function to get dict keys', 'def get_keys(d): return list(d.keys())'),
        ('write a python function to get dict values', 'def get_values(d): return list(d.values())'),
        ('write a python function to merge dicts', 'def merge_dicts(a, b): return {**a, **b}'),
        ('write a python function to filter by condition', 'def filter_list(lst, f): return [x for x in lst if f(x)]'),
        ('write a python function to map over list', 'def map_list(lst, f): return [f(x) for x in lst]'),
        ('write a python function to reduce', 'def reduce_list(lst, f, init): r = init; for x in lst: r = f(r, x); return r'),
        ('write a python function to group by', 'def group_by(lst, key_fn): groups = {}; [groups.setdefault(key_fn(x), []).append(x) for x in lst]; return groups'),
        ('write a python function to deep copy list', 'def deep_copy(lst): return [x.copy() if isinstance(x, list) else x for x in lst]'),
        ('write a python function to rotate list', 'def rotate(lst, n): return lst[n:] + lst[:n]'),
        ('write a python function to interleave', 'def interleave(a, b): return [val for pair in zip(a, b) for val in pair]'),
    ]
    pairs.extend(code_patterns)
    return pairs


def generate_reasoning_pairs():
    """Generate reasoning and explanation pairs."""
    pairs = [
        ('explain the difference between tcp and udp',
         'tcp is reliable and ordered, ensuring all data arrives correctly. udp is faster but does not guarantee delivery or ordering.'),
        ('explain the difference between sql and nosql',
         'sql databases use structured tables with fixed schemas. nosql databases are flexible, handling unstructured data with various models.'),
        ('explain the difference between compiled and interpreted languages',
         'compiled languages translate all code before execution, producing fast binaries. interpreted languages execute line by line, easier to debug but slower.'),
        ('explain the difference between stack and queue',
         'a stack is last-in-first-out: the last item added is removed first. a queue is first-in-first-out: the first item added is removed first.'),
        ('explain the difference between process and thread',
         'a process has its own memory space. threads share memory within a process, making them lighter but requiring synchronization.'),
        ('explain the difference between http and https',
         'http sends data in plain text. https encrypts data using ssl or tls, making it secure against eavesdropping.'),
        ('explain the difference between encryption and hashing',
         'encryption is reversible: you can decrypt back to the original. hashing is one-way: you cannot recover the original from the hash.'),
        ('explain the difference between ram and storage',
         'ram is fast, temporary memory that loses data when powered off. storage is slower but persistent, keeping data permanently.'),
        ('explain the difference between a virus and bacteria',
         'viruses need a host cell to reproduce and are not considered alive. bacteria are single-celled living organisms that reproduce independently.'),
        ('explain the difference between weather and climate',
         'weather is the short-term state of the atmosphere. climate is the long-term average of weather patterns over years.'),

        ('why is the sky blue',
         'the sky is blue because air molecules scatter blue light more than other colors, a process called rayleigh scattering.'),
        ('why does ice float',
         'ice floats because water expands when it freezes, making ice less dense than liquid water.'),
        ('why do leaves change color',
         'leaves change color because chlorophyll breaks down in autumn, revealing other pigments like yellow and red that were always present.'),
        ('why is the ocean salty',
         'the ocean is salty because rivers dissolve minerals from rocks and carry them to the sea, where the salt accumulates over time.'),
        ('why do we dream',
         'the exact reason is debated, but dreaming likely helps consolidate memories, process emotions, and simulate threats for survival.'),
        ('why does salt melt ice',
         'salt lowers the freezing point of water, so ice melts even when the temperature is below zero degrees celsius.'),
        ('why is water wet',
         'water is wet because its molecules are attracted to other substances, spreading across surfaces and coating them.'),
    ]
    return pairs


def generate_creative_pairs():
    """Generate creative writing pairs."""
    pairs = [
        ('write a poem about the wind', 'the wind is a ghost that haunts the trees, whispering secrets no one sees, it pushes clouds across the sky, and makes the fallen leaves fly.'),
        ('write a poem about fire', 'fire dances with hungry grace, consuming all within its embrace, a flower of flame both fierce and bright, painting shadows on the night.'),
        ('write a poem about earth', 'patient mother of stone and seed, your forests drink the rain they need, your mountains stand with ancient grace, each valley a warm embrace.'),
        ('write a poem about space', 'in the vast cathedral of the void, where stars are born and planets ride, silence speaks in cosmic verse, infinite and diverse.'),
        ('write a poem about rain', 'rain falls like silver needles stitching earth to sky, each drop a tiny messenger from clouds that drift on high.'),
        ('write a poem about snow', 'each snowflake a tiny star descending, unique and white and unending, they blanket the world in silent white, turning dark to light.'),
        ('write a poem about a river', 'a river carves its patient way, through stone and clay and every day, it does not stop it does not rest, it simply flows toward the west.'),
        ('write a poem about a mountain', 'a mountain stands with shoulders broad, supporting snow against the sky, unmoved by wind or rain or time, it watches centuries go by.'),

        ('describe a library', 'dust motes float in golden light between towering shelves. the smell of old paper and leather fills the air. silence wraps around you like a blanket, broken only by the soft turn of pages.'),
        ('describe a thunderstorm at night', 'lightning flashes freeze the rain in mid-air for an instant. thunder rolls across the sky like boulders falling. the house shudders. then darkness again, thicker than before.'),
        ('describe a coffee shop', 'steam rises from cups like tiny ghosts. the espresso machine hisses and groans. conversations blur into a warm background hum. the smell of roasted beans hangs heavy.'),
        ('describe a desert', 'heat shimmers above the sand making the horizon liquid. the silence is absolute, broken only by wind sculpting dunes. every shadow is a refuge.'),
        ('describe a forest at dawn', 'mist hangs between the trees like gauze. birdsong builds from one voice to a chorus. dew catches the first pale light, turning each leaf into a tiny lantern.'),
        ('describe a snowy day', 'the world is muffled and white. footprints fill behind you as fast as you make them. breath hangs in clouds. the silence is soft and deep.'),

        ('write a one sentence story', 'the last book on earth was found in a cave by someone who could no longer read.'),
        ('write a tiny love story', 'she said she would wait forever. he came back the next day.'),
        ('write a twist ending', 'the detective finally found the killer. it was himself, from the future.'),

        ('write a motivational speech', 'do not wait for permission. do not wait for the perfect moment. the moment is now. you are ready. you have always been ready. take the first step.'),
        ('write a philosophical thought', 'we are the universe experiencing itself, briefly, through the medium of consciousness.'),
        ('write a wisdom quote', 'the fool knows the price of everything and the value of nothing. the wise know the opposite.'),
    ]
    return pairs


def generate_math_pairs():
    """Generate math Q/A pairs."""
    pairs = []
    math_concepts = [
        ('a fraction', 'a fraction represents a part of a whole, written as one number over another'),
        ('a decimal', 'a decimal is a number expressed in base 10 using a decimal point'),
        ('a percentage', 'a percentage is a fraction out of 100, denoted with the percent sign'),
        ('a ratio', 'a ratio compares two quantities, showing their relative sizes'),
        ('a proportion', 'a proportion is an equation stating two ratios are equal'),
        ('a variable in math', 'a variable is a symbol representing an unknown or changing quantity'),
        ('a coefficient', 'a coefficient is the number multiplied by a variable in an expression'),
        ('an equation', 'an equation is a statement that two expressions are equal'),
        ('a function in math', 'a function maps each input to exactly one output'),
        ('a derivative', 'a derivative measures the instantaneous rate of change of a function'),
        ('an integral', 'an integral computes the accumulated quantity or area under a curve'),
        ('a logarithm', 'a logarithm is the inverse of exponentiation. it finds the power needed'),
        ('a matrix', 'a matrix is a rectangular array of numbers used for linear transformations'),
        ('a vector', 'a vector is a quantity with both magnitude and direction'),
        ('standard deviation', 'standard deviation measures how spread out numbers are from the mean'),
        ('correlation', 'correlation measures how two variables change together, from -1 to 1'),
    ]
    for concept, definition in math_concepts:
        pairs.append((f'what is {concept}', definition))
        pairs.append((f'define {concept} in math', definition))
    return pairs


def generate_geography_pairs():
    """Generate geography pairs."""
    pairs = []
    countries_caps = [
        ('egypt', 'cairo'), ('turkey', 'ankara'), ('greece', 'athens'),
        ('portugal', 'lisbon'), ('netherlands', 'amsterdam'), ('belgium', 'brussels'),
        ('austria', 'vienna'), ('switzerland', 'bern'), ('denmark', 'copenhagen'),
        ('finland', 'helsinki'), ('poland', 'warsaw'), ('ireland', 'dublin'),
        ('norway', 'oslo'), ('sweden', 'stockholm'), ('argentina', 'buenos aires'),
        ('chile', 'santiago'), ('peru', 'lima'), ('colombia', 'bogota'),
        ('venezuela', 'caracas'), ('thailand', 'bangkok'), ('vietnam', 'hanoi'),
        ('indonesia', 'jakarta'), ('philippines', 'manila'), ('malaysia', 'kuala lumpur'),
        ('saudi arabia', 'riyadh'), ('iran', 'tehran'), ('iraq', 'baghdad'),
        ('israel', 'jerusalem'), ('kenya', 'nairobi'), ('nigeria', 'abuja'),
        ('south africa', 'pretoria'), ('morocco', 'rabat'), ('ukraine', 'kyiv'),
        ('czech republic', 'prague'), ('hungary', 'budapest'), ('romania', 'bucharest'),
        ('bulgaria', 'sofia'), ('croatia', 'zagreb'), ('serbia', 'belgrade'),
        ('lithuania', 'vilnius'), ('latvia', 'riga'), ('estonia', 'tallinn'),
        ('new zealand', 'wellington'), ('bangladesh', 'dhaka'), ('pakistan', 'islamabad'),
        ('afghanistan', 'kabul'), ('mongolia', 'ulaanbaatar'), ('kazakhstan', 'astana'),
        ('cuba', 'havana'), ('jamaica', 'kingston'), ('iceland', 'reykjavik'),
    ]
    for country, cap in countries_caps:
        pairs.append((f'what is the capital of {country}', f'the capital of {country} is {cap}.'))
        pairs.append((f'capital of {country}', cap))

    # Rivers, mountains, oceans
    geography_facts = [
        ('what is the longest river in the world', 'the nile is the longest river at about 6650 kilometers'),
        ('what is the largest desert', 'the antarctic desert is the largest desert by area'),
        ('what is the highest mountain', 'mount everest is the highest mountain at 8849 meters'),
        ('what is the deepest ocean trench', 'the mariana trench is the deepest at about 11000 meters'),
        ('what is the largest continent', 'asia is the largest continent by area and population'),
        ('what is the smallest country', 'vatican city is the smallest country at 0.49 square kilometers'),
        ('what is the largest lake', 'the caspian sea is the largest lake by surface area'),
        ('what is the largest island', 'greenland is the largest island in the world'),
        ('what is the tallest waterfall', 'angel falls in venezuela is the tallest at 979 meters'),
        ('what is the largest rainforest', 'the amazon rainforest is the largest, covering much of south america'),
    ]
    pairs.extend(geography_facts)
    return pairs


def get_mega_corpus():
    """Build and return the complete mega corpus."""
    pairs = []
    pairs.extend(KILLER_PAIRS)
    pairs.extend(CONVERSATION_PAIRS)
    pairs.extend(MASSIVE_PAIRS)
    pairs.extend(generate_science_pairs())
    pairs.extend(generate_tech_pairs())
    pairs.extend(generate_code_pairs())
    pairs.extend(generate_reasoning_pairs())
    pairs.extend(generate_creative_pairs())
    pairs.extend(generate_math_pairs())
    pairs.extend(generate_geography_pairs())

    # Load TriviaQA
    try:
        with open('trivia_qa_pairs.json') as f:
            trivia = json.load(f)
        for item in trivia:
            pairs.append((item['q'].lower(), item['a'].lower()))
    except FileNotFoundError:
        pass

    # Also load large dataset
    try:
        with open('large_dataset.json') as f:
            large = json.load(f)
        for item in large[:8000]:
            pairs.append((item['q'].lower(), item['a'].lower()))
    except FileNotFoundError:
        pass

    # Add augmented variants
    augmented = []
    for q, a in pairs:
        augmented.append((q, a))
        augmented.append((q + " ", a))

    return augmented


if __name__ == '__main__':
    corpus = get_mega_corpus()
    total_chars = sum(len(q) + len(a) for q, a in corpus)
    print(f'Mega corpus: {len(corpus)} pairs, {total_chars:,} chars')
    print(f'Estimated tokens (BPE ~1.5x reduction): {total_chars // 2:,}')
    print(f'Estimated model size: {total_chars // 2 * 5024 / 1024 / 1024 / 1024 * 1000:.1f} GB (approx)')