File size: 2,291 Bytes
7c5e40e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Five-language STRATA core contract.
The project deliberately keeps the primary multilingual build to five languages
so graph supervision, tokenizer pressure, and tool-surface coverage stay
balanced enough to interpret. Additions beyond this set should be treated as
diagnostic or tier-2 experiments, not silent changes to the core training mix.
"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True, slots=True)
class LanguageSpec:
code: str
name: str
primary_variety: str
graph_source: tuple[str, str]
CORE_LANGUAGES: tuple[LanguageSpec, ...] = (
LanguageSpec("en", "English", "English", ("up", "UP_English-EWT")),
LanguageSpec("zh", "Mandarin Chinese", "Simplified Chinese primary", ("up", "UP_Chinese")),
LanguageSpec("de", "German", "German", ("up", "UP_German")),
LanguageSpec("es", "Spanish", "Spanish", ("up", "UP_Spanish")),
LanguageSpec("ar", "Arabic", "MSA primary; dialectal optional", ("ud", "UD_Arabic-PADT")),
)
CORE_LANGUAGE_CODES: tuple[str, ...] = tuple(spec.code for spec in CORE_LANGUAGES)
CORE_LANGUAGE_NAMES: dict[str, str] = {spec.code: spec.name for spec in CORE_LANGUAGES}
CORE_GRAPH_SOURCES: dict[str, tuple[str, str]] = {
spec.code: spec.graph_source for spec in CORE_LANGUAGES
}
def normalize_language_codes(value: str | list[str] | tuple[str, ...]) -> tuple[str, ...]:
"""Return normalized language codes from a comma-separated string or list."""
if isinstance(value, str):
codes = tuple(part.strip() for part in value.split(",") if part.strip())
else:
codes = tuple(str(part).strip() for part in value if str(part).strip())
return codes
def require_core_languages(codes: str | list[str] | tuple[str, ...]) -> None:
"""Raise if ``codes`` is not exactly the STRATA five-language core."""
got = normalize_language_codes(codes)
if got != CORE_LANGUAGE_CODES:
raise ValueError(
f"expected STRATA core languages {','.join(CORE_LANGUAGE_CODES)}, "
f"got {','.join(got) or '<empty>'}"
)
__all__ = [
"CORE_GRAPH_SOURCES",
"CORE_LANGUAGES",
"CORE_LANGUAGE_CODES",
"CORE_LANGUAGE_NAMES",
"LanguageSpec",
"normalize_language_codes",
"require_core_languages",
]
|