| """Frozen zero-shot validation; no training, threshold fitting or test-set tuning.""" | |
| from sealcell.evaluation import evaluate, fixture_benchmark | |
| from sealcell.model import get_model | |
| model = get_model() | |
| fixture_benchmark(model, "data/fixtures/playground-probes.json", "reports/local-probes.jsonl") | |
| fixture_benchmark(model, "data/fixtures/development.jsonl", "reports/development-fixtures.jsonl") | |
| for split in ["development", "final", "ood"]: | |
| output, _ = evaluate( | |
| "configs/evaluation.yaml", split=split, output=f"reports/runs/{split}-primary" | |
| ) | |
| print(output, flush=True) | |
| # These are development robustness checks, not alternative final-test tuning runs. | |
| for condition in [ | |
| "raw", | |
| "three-heads", | |
| "no-maintenance", | |
| "reverse-labels", | |
| "descriptions", | |
| "paraphrase", | |
| ]: | |
| output, _ = evaluate( | |
| "configs/evaluation.yaml", | |
| split="development", | |
| condition=condition, | |
| policies=["rules", "hybrid", "direct"], | |
| limit=26, | |
| output=f"reports/runs/development-{condition}", | |
| ) | |
| print(output, flush=True) | |