tinymetatron-slm / tests /test_corpus_loop.py
Quantum927's picture
Sync demo image: Python 3.13 and torch==2.13.0 CPU pin
75878d6 verified
Raw History Blame Contribute Delete
8.66 kB
"""
tests/test_corpus_loop.py
========================
Automated smoke test for the corpus loop (Phase 3 completion).
Verifies the invariant the manual smoke run got wrong: a *source-disjoint*
split, where every source_id appears in exactly one of {train, val, hard_dev}
β€” not merely text-disjoint within a source.
Covers:
* Full pipeline on the smoke fixture produces text- AND source-disjoint splits.
* MANIFEST.json records split_policy / split_seed / source_counts /
source_overlap / text_overlap, with every overlap count zero.
* At least 3 independent source groups; experiment reaches FROZEN_CORPUS.
* Deterministic: two clean runs produce the same corpus_hash.
* Split rejects a corpus with too few source groups (< 3).
* Corpus gate worker flags a nonzero source overlap and passes a clean one.
"""
from __future__ import annotations
import json
import os
import shutil
import sys
import tempfile
from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[1]
# Point the registry at a throwaway temp DB BEFORE importing the corpus loop so
# the module-level init_db() in corpus_loop never touches state/registry.db.
_TMP_REGISTRY = Path(tempfile.mkdtemp(prefix="corpus_test_registry_")) / "registry.db"
os.environ["TINYMETATRON_DB"] = str(_TMP_REGISTRY)
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
import db # noqa: E402
db.set_db_path(str(_TMP_REGISTRY))
db.init_db(str(_TMP_REGISTRY))
from loops import corpus_loop # noqa: E402
import pytest # noqa: E402
_FIXTURE = _REPO_ROOT / "tests" / "fixtures" / "corpus_smoke.jsonl"
# ── fixtures ──────────────────────────────────────────────────────────────────
@pytest.fixture
def clean_registry(tmp_path) -> str:
"""A fresh temp registry DB per test (clean test registry)."""
p = str(tmp_path / "registry.db")
db.set_db_path(p)
db.init_db(p)
return p
@pytest.fixture
def corpus_dir(tmp_path) -> Path:
"""Temp raw-corpus dir containing a copy of the smoke fixture."""
d = tmp_path / "raw"
d.mkdir()
shutil.copy(_FIXTURE, d / "corpus_smoke.jsonl")
return d
def _load_jsonl(path: Path) -> list[dict]:
return [json.loads(l) for l in open(path, encoding="utf-8") if l.strip()]
def _texts(rows: list[dict]) -> set:
return {r["text"] for r in rows}
def _sources(rows: list[dict]) -> set:
return {r.get("source", r.get("domain", "unknown")) for r in rows}
# ── source-disjoint smoke ─────────────────────────────────────────────────────
def test_corpus_smoke_source_disjoint(clean_registry, corpus_dir, tmp_path) -> None:
out = tmp_path / "out"
summary = corpus_loop.run_corpus_pipeline({
"exp_id": "exp-smoke-001",
"corpus_dir": str(corpus_dir),
"output_dir": str(out),
"seed": 42,
"train_pct": 0.80,
"val_pct": 0.10,
})
corpus = out / "corpus"
train = _load_jsonl(corpus / "train.jsonl")
val = _load_jsonl(corpus / "val.jsonl")
hard = _load_jsonl(corpus / "hard_dev.jsonl")
tr_t, va_t, ha_t = _texts(train), _texts(val), _texts(hard)
tr_s, va_s, ha_s = _sources(train), _sources(val), _sources(hard)
# Every partition non-empty.
assert train and val and hard, "all three splits must be non-empty"
# Text-disjoint.
assert tr_t.isdisjoint(va_t)
assert tr_t.isdisjoint(ha_t)
assert va_t.isdisjoint(ha_t)
# Source-disjoint β€” the invariant the manual smoke run got wrong.
assert tr_s.isdisjoint(va_s)
assert tr_s.isdisjoint(ha_s)
assert va_s.isdisjoint(ha_s)
# At least 3 independent source groups overall.
assert len(tr_s | va_s | ha_s) >= 3
# MANIFEST.json records the split policy invariants.
man = json.loads((out / "MANIFEST.json").read_text(encoding="utf-8"))
assert man["split_policy"] == "source_disjoint_capped_v1"
assert man["split_seed"] == 42
assert man["source_counts"] == {
"train": len(tr_s), "val": len(va_s), "hard_dev": len(ha_s),
}
for key in ("train_val", "train_hard_dev", "val_hard_dev"):
assert man["source_overlap"][key] == 0, f"source_overlap[{key}] != 0"
assert man["text_overlap"][key] == 0, f"text_overlap[{key}] != 0"
assert sum(man["source_overlap"].values()) == 0
assert sum(man["text_overlap"].values()) == 0
# Gate passed β†’ experiment frozen.
assert summary["corpus_hash"]
exp = db.get_loop_experiment("exp-smoke-001")
assert exp["state"] == "FROZEN_CORPUS"
# ── determinism ───────────────────────────────────────────────────────────────
def test_corpus_smoke_deterministic(clean_registry, corpus_dir, tmp_path) -> None:
out1 = tmp_path / "out1"
s1 = corpus_loop.run_corpus_pipeline({
"exp_id": "exp-smoke-det-a",
"corpus_dir": str(corpus_dir),
"output_dir": str(out1),
"seed": 42,
})
out2 = tmp_path / "out2"
s2 = corpus_loop.run_corpus_pipeline({
"exp_id": "exp-smoke-det-b",
"corpus_dir": str(corpus_dir),
"output_dir": str(out2),
"seed": 42,
})
assert s1["corpus_hash"] == s2["corpus_hash"], (
"deterministic split must produce identical corpus_hash across runs")
# ── too-few source groups rejected ────────────────────────────────────────────
def test_split_rejects_too_few_sources(clean_registry, tmp_path) -> None:
raw = tmp_path / "raw"
raw.mkdir()
rows = [
{"text": "Alpha source sentence one about network security fundamentals.",
"source": "alpha"},
{"text": "Alpha source sentence two about cryptographic hash functions.",
"source": "alpha"},
{"text": "Beta source sentence one about machine learning neural nets.",
"source": "beta"},
{"text": "Beta source sentence two about gradient descent optimization.",
"source": "beta"},
]
with open(raw / "in.jsonl", "w", encoding="utf-8") as f:
for r in rows:
f.write(json.dumps(r) + "\n")
out = tmp_path / "out"
with pytest.raises(RuntimeError, match="Worker error"):
corpus_loop.run_corpus_pipeline({
"exp_id": "exp-smoke-few",
"corpus_dir": str(raw),
"output_dir": str(out),
})
# Must not have been frozen.
assert db.get_loop_experiment("exp-smoke-few")["state"] != "FROZEN_CORPUS"
# ── corpus gate worker: detects overlap, passes clean ─────────────────────────
def test_corpus_gate_detects_overlap(tmp_path) -> None:
man = tmp_path / "MANIFEST.json"
payload = {
"split_policy": "source_disjoint_v1",
"split_seed": 42,
"source_counts": {"train": 2, "val": 2, "hard_dev": 1},
"source_overlap": {"train_val": 1, "train_hard_dev": 0, "val_hard_dev": 0},
"text_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0},
}
man.write_text(json.dumps(payload), encoding="utf-8")
result = corpus_loop._run_worker(
[sys.executable, "-m", "workers.corpus.corpus_gate", "--manifest", str(man)],
tmp_path / "gate_art", 60,
)
assert result["metrics"]["source_overlap_total"] == 1
# Gate pass_condition (source_overlap_total == 0) must evaluate to False.
assert corpus_loop._eval_condition(
result["metrics"]["source_overlap_total"], "==", 0) is False
def test_corpus_gate_passes_clean(tmp_path) -> None:
man = tmp_path / "MANIFEST.json"
payload = {
"split_policy": "source_disjoint_v1",
"split_seed": 42,
"source_counts": {"train": 2, "val": 1, "hard_dev": 1},
"source_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0},
"text_overlap": {"train_val": 0, "train_hard_dev": 0, "val_hard_dev": 0},
}
man.write_text(json.dumps(payload), encoding="utf-8")
result = corpus_loop._run_worker(
[sys.executable, "-m", "workers.corpus.corpus_gate", "--manifest", str(man)],
tmp_path / "gate_art", 60,
)
assert result["metrics"]["source_overlap_total"] == 0
assert corpus_loop._eval_condition(
result["metrics"]["source_overlap_total"], "==", 0) is True