src/korbyte/config.py

"""Shared, pinned configuration for reproducible KorByte builds."""

from __future__ import annotations

from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any

CORE_VOCAB_SIZE = 128_000
SPECIAL_TOKEN_COUNT = 256
TOTAL_VOCAB_SIZE = CORE_VOCAB_SIZE + SPECIAL_TOKEN_COUNT
DEFAULT_SEED = 20260804
DEFAULT_MODEL_MAX_LENGTH = 32_768
DEFAULT_REPOSITORY_ID = "dawncr0w/KorByte-128K"
KANANA_MODEL_ID = "kakaocorp/kanana-2-3b-base"
KANANA_REVISION = "9fa2a678c379700868d72685a22ea8ed0263ef33"
KLUE_REVISION = "349481ec73fff722f88e0453ca05c77a447d967c"
PUBLIC_BENCHMARK_ID = "eduagarcia/multilingual_tokenizer_benchmark"
PUBLIC_BENCHMARK_REVISION = "74fa5907af0265f42c09e21379bf21847e066990"
KMMLU_ID = "HAERAE-HUB/KMMLU"
KMMLU_REVISION = "d61b3f19e552c576bf5960dd24289763edc36a88"


@dataclass(frozen=True)
class BaselineSpec:
    """A public tokenizer artifact pinned for comparative evaluation."""

    key: str
    model_id: str
    revision: str


COMPARISON_BASELINES = (
    BaselineSpec("ax-4.0", "skt/A.X-4.0", "a3393fd67bf09a34a85e66dfd2c2da61c4497b67"),
    BaselineSpec(
        "trillion-7b",
        "trillionlabs/Trillion-7B-preview",
        "bc1650c5a954c7de0509d28c8dbc5bea560856e1",
    ),
    BaselineSpec(
        "midm-2.0",
        "K-intelligence/Midm-2.0-Base-Instruct",
        "35479c5fc9a18a5db7cc6dbadcf1db68db7beab0",
    ),
    BaselineSpec(
        "k-exaone",
        "LGAI-EXAONE/K-EXAONE-236B-A23B",
        "61e6d578eb102b578e5704e2916ac841df9eca0a",
    ),
    BaselineSpec(
        "solar-pro3",
        "upstage/solar-pro3-tokenizer",
        "c41b71f0519c580610cb0fd2af4ed2b23cad544f",
    ),
    BaselineSpec(
        "thunder",
        "thunder-research-group/Llama-Thunder-LLM-8B",
        "ecfd6c909944939ae42982f3d87cf36898da308b",
    ),
    BaselineSpec(
        "kanana-2",
        KANANA_MODEL_ID,
        KANANA_REVISION,
    ),
    BaselineSpec(
        "hyperclovax",
        "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
        "a6cdfd3464d1b767259cad23e164eaf39d3e3960",
    ),
)


@dataclass(frozen=True)
class SourceSpec:
    """A deterministic slice of a public corpus."""

    key: str
    dataset_id: str
    config_name: str
    split: str
    revision: str
    license_id: str
    source_url: str
    text_field: str = "text"
    character_budget: int = 0
    min_target_script_ratio: float = 0.0
    target_script: str = "any"
    shuffle_buffer: int = 10_000

    def as_dict(self) -> dict[str, Any]:
        return asdict(self)


DEFAULT_SOURCES = (
    SourceSpec(
        key="fineweb2_korean",
        dataset_id="HuggingFaceFW/fineweb-2",
        config_name="kor_Hang",
        split="train",
        revision="af9c13333eb981300149d5ca60a8e9d659b276b9",
        license_id="ODC-By-1.0",
        source_url="https://huggingface.co/datasets/HuggingFaceFW/fineweb-2",
        character_budget=500_000_000,
        min_target_script_ratio=0.25,
        target_script="hangul",
    ),
    SourceSpec(
        key="wikipedia_korean",
        dataset_id="wikimedia/wikipedia",
        config_name="20231101.ko",
        split="train",
        revision="b04c8d1ceb2f5cd4588862100d08de323dccfbaa",
        license_id="CC-BY-SA-3.0-and-GFDL",
        source_url="https://huggingface.co/datasets/wikimedia/wikipedia",
        character_budget=150_000_000,
        min_target_script_ratio=0.20,
        target_script="hangul",
    ),
    SourceSpec(
        key="wikipedia_english",
        dataset_id="wikimedia/wikipedia",
        config_name="20231101.en",
        split="train",
        revision="b04c8d1ceb2f5cd4588862100d08de323dccfbaa",
        license_id="CC-BY-SA-3.0-and-GFDL",
        source_url="https://huggingface.co/datasets/wikimedia/wikipedia",
        character_budget=50_000_000,
        min_target_script_ratio=0.60,
        target_script="latin",
    ),
)


@dataclass(frozen=True)
class Paths:
    """Project paths derived from one repository root."""

    root: Path

    @property
    def corpus(self) -> Path:
        return self.root / "data" / "train.txt"

    @property
    def corpus_manifest(self) -> Path:
        return self.root / "provenance" / "corpus_manifest.json"

    @property
    def build_manifest(self) -> Path:
        return self.root / "provenance" / "build_manifest.json"

    @property
    def benchmark_json(self) -> Path:
        return self.root / "reports" / "benchmark.json"

    @property
    def benchmark_markdown(self) -> Path:
        return self.root / "reports" / "benchmark.md"

    @property
    def comparison_json(self) -> Path:
        return self.root / "reports" / "comparison.json"

    @property
    def comparison_markdown(self) -> Path:
        return self.root / "reports" / "comparison.md"

    @property
    def validation_json(self) -> Path:
        return self.root / "reports" / "validation.json"


def project_root() -> Path:
    return Path(__file__).resolve().parents[2]