| |
| """Build the reproducible, source-level OOD set from a Tatoeba export. |
| |
| The export is intentionally not committed. The resulting JSONL contains the |
| selected text plus source IDs and hashes so the published set can be audited. |
| """ |
| import argparse |
| import hashlib |
| import json |
| import tarfile |
| import urllib.request |
| from collections import Counter, defaultdict |
| from datetime import date |
| from pathlib import Path |
|
|
| SOURCE_URL = "https://downloads.tatoeba.org/exports/sentences.tar.bz2" |
| SOURCE_SHA256 = "8117f33886e94c3d371f1f13d0efa120a8172d67eb7006b2656c444c012645d0" |
| SOURCE_RELEASE = "2026-07-18 export (archive last-modified date)" |
| TOTAL_PER_CLASS = 8772 |
| LANGUAGES = { |
| "afr": "Afrikaans", "ara": "Arabic", "asm": "Assamese", "ben": "Bengali", |
| "bul": "Bulgarian", "cat": "Catalan", "ces": "Czech", "cmn": "Mandarin Chinese", |
| "dan": "Danish", "deu": "German", "ell": "Greek", "epo": "Esperanto", |
| "eus": "Basque", "fin": "Finnish", "fra": "French", "gle": "Irish", |
| "glg": "Galician", "hau": "Hausa", "heb": "Hebrew", "hin": "Hindi", |
| "hrv": "Croatian", "hun": "Hungarian", "hye": "Armenian", "ind": "Indonesian", |
| "isl": "Icelandic", "ita": "Italian", "jpn": "Japanese", "kor": "Korean", |
| "lit": "Lithuanian", "lvs": "Latvian", "mar": "Marathi", "mkd": "Macedonian", |
| "mon": "Mongolian", "nld": "Dutch", "nno": "Norwegian Nynorsk", |
| "nob": "Norwegian Bokmal", "pol": "Polish", "por": "Portuguese", |
| "ron": "Romanian", "rus": "Russian", "slk": "Slovak", "slv": "Slovenian", |
| "spa": "Spanish", "srp": "Serbian", "swe": "Swedish", "swh": "Swahili", |
| "tat": "Tatar", "tha": "Thai", "tur": "Turkish", "ukr": "Ukrainian", |
| "vie": "Vietnamese", "yid": "Yiddish", |
| } |
|
|
|
|
| def normalize(text): |
| return " ".join(text.split()).strip() |
|
|
|
|
| def digest(text): |
| return hashlib.sha256(text.encode("utf-8")).hexdigest() |
|
|
|
|
| def bucket(text): |
| size = len(text.encode("utf-8")) |
| if size <= 32: |
| return "short" |
| if size <= 96: |
| return "medium" |
| return "long" |
|
|
|
|
| def add_candidate(heaps, lang, capacity, source_id, text): |
| """Keep the lexicographically smallest stable hashes per language.""" |
| import heapq |
|
|
| text_hash = digest(text) |
| key = (text_hash, str(source_id)) |
| heap = heaps[lang] |
| item = (key[0], key[1], text) |
| if len(heap) < capacity: |
| heapq.heappush(heap, (-int(key[0], 16), key[1], text)) |
| elif int(key[0], 16) < -heap[0][0]: |
| heapq.heapreplace(heap, (-int(key[0], 16), key[1], text)) |
|
|
|
|
| def download(path): |
| if path.exists(): |
| return |
| path.parent.mkdir(parents=True, exist_ok=True) |
| urllib.request.urlretrieve(SOURCE_URL, path) |
|
|
|
|
| def sha256_file(path): |
| h = hashlib.sha256() |
| with path.open("rb") as f: |
| for block in iter(lambda: f.read(1024 * 1024), b""): |
| h.update(block) |
| return h.hexdigest() |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--source", type=Path, required=True) |
| parser.add_argument("--output", type=Path, required=True) |
| parser.add_argument("--manifest", type=Path, required=True) |
| args = parser.parse_args() |
|
|
| actual_sha256 = sha256_file(args.source) |
| if actual_sha256 != SOURCE_SHA256: |
| raise SystemExit(f"source checksum mismatch: {actual_sha256} != {SOURCE_SHA256}") |
|
|
| targets = {"eng": TOTAL_PER_CLASS} |
| non_english = sorted(LANGUAGES) |
| base, remainder = divmod(TOTAL_PER_CLASS, len(non_english)) |
| targets.update({lang: base + (i < remainder) for i, lang in enumerate(non_english)}) |
| heaps = defaultdict(list) |
| seen_rows = 0 |
| with tarfile.open(args.source, "r:bz2") as archive: |
| member = archive.extractfile("sentences.csv") |
| if member is None: |
| raise SystemExit("sentences.csv missing from source archive") |
| for raw in member: |
| seen_rows += 1 |
| if seen_rows == 1: |
| continue |
| parts = raw.decode("utf-8").rstrip("\n").split("\t", 2) |
| if len(parts) != 3: |
| continue |
| source_id, lang, raw_text = parts |
| if lang not in targets: |
| continue |
| text = normalize(raw_text) |
| encoded = text.encode("utf-8") |
| if not 3 <= len(text) <= 276 or len(encoded) > 256: |
| continue |
| if not any(ch.isalpha() for ch in text): |
| continue |
| add_candidate(heaps, lang, targets[lang] * 4, source_id, text) |
|
|
| rows = [] |
| used_text_hashes = set() |
| for lang in ["eng", *non_english]: |
| candidates = sorted(heaps[lang], key=lambda item: (item[0] * -1, item[1])) |
| label = "EN" if lang == "eng" else "NOT-EN" |
| selected = 0 |
| for _, source_id, text in candidates: |
| text_hash = digest(text) |
| if text_hash in used_text_hashes: |
| continue |
| used_text_hashes.add(text_hash) |
| rows.append({ |
| "id": f"tatoeba-{source_id}-{lang}", |
| "text": text, |
| "label": label, |
| "language": "en" if lang == "eng" else lang, |
| "language_name": "English" if lang == "eng" else LANGUAGES[lang], |
| "category": bucket(text), |
| "source": "Tatoeba", |
| "source_release": SOURCE_RELEASE, |
| "source_id": source_id, |
| "source_url": f"https://tatoeba.org/en/sentences/show/{source_id}", |
| "text_sha256": digest(text), |
| "utf8_bytes": len(text.encode("utf-8")), |
| }) |
| selected += 1 |
| if selected == targets[lang]: |
| break |
| if selected != targets[lang]: |
| raise SystemExit(f"not enough unique rows for {lang}: {selected}/{targets[lang]}") |
|
|
| if len(rows) != TOTAL_PER_CLASS * 2: |
| raise SystemExit(f"wrong row count: {len(rows)}") |
| if len({row["text_sha256"] for row in rows}) != len(rows): |
| raise SystemExit("duplicate normalized text hashes found") |
| counts = Counter(row["language"] for row in rows) |
| if counts["en"] != TOTAL_PER_CLASS or len(counts) != 53: |
| raise SystemExit(f"wrong composition: {counts}") |
|
|
| args.output.parent.mkdir(parents=True, exist_ok=True) |
| with args.output.open("w", encoding="utf-8") as f: |
| for row in rows: |
| f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") |
|
|
| manifest = { |
| "dataset_name": "en-or-not OOD set", |
| "created": str(date.today()), |
| "source": { |
| "name": "Tatoeba sentence export", |
| "url": SOURCE_URL, |
| "release": SOURCE_RELEASE, |
| "sha256": actual_sha256, |
| "license": "CC-BY 2.0", |
| }, |
| "selection": { |
| "seed": "sha256(text)", |
| "total_examples": len(rows), |
| "english_examples": counts["en"], |
| "non_english_examples": len(rows) - counts["en"], |
| "non_english_languages": sorted(LANGUAGES), |
| "language_counts": dict(sorted(counts.items())), |
| "max_utf8_bytes": 256, |
| "filters": ["3-276 Unicode characters", "3-256 UTF-8 bytes", "contains a letter", "normalized whitespace"], |
| }, |
| "exclusion_boundary": { |
| "documented_training_sources": [ |
| "Project Gutenberg collections", |
| "papluca/language-identification", |
| "oscar-corpus/oscar (optional)", |
| ], |
| "guarantee": "source-level exclusion only; zero lexical overlap with undisclosed training artifacts is not guaranteed", |
| }, |
| } |
| args.manifest.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") |
| print(json.dumps({"rows": len(rows), "languages": len(counts), "sha256": actual_sha256}, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
