File size: 1,087 Bytes
73bd12f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | """Frozen zero-shot validation; no training, threshold fitting or test-set tuning."""
from sealcell.evaluation import evaluate, fixture_benchmark
from sealcell.model import get_model
model = get_model()
fixture_benchmark(model, "data/fixtures/playground-probes.json", "reports/local-probes.jsonl")
fixture_benchmark(model, "data/fixtures/development.jsonl", "reports/development-fixtures.jsonl")
for split in ["development", "final", "ood"]:
output, _ = evaluate(
"configs/evaluation.yaml", split=split, output=f"reports/runs/{split}-primary"
)
print(output, flush=True)
# These are development robustness checks, not alternative final-test tuning runs.
for condition in [
"raw",
"three-heads",
"no-maintenance",
"reverse-labels",
"descriptions",
"paraphrase",
]:
output, _ = evaluate(
"configs/evaluation.yaml",
split="development",
condition=condition,
policies=["rules", "hybrid", "direct"],
limit=26,
output=f"reports/runs/development-{condition}",
)
print(output, flush=True)
|