"""Score every file of one split with a saved model and write the scores to a CSV. python src/evaluate.py --model model/wholefile_transformer_halfchunks_segpos.pt --split val python src/evaluate.py --model model/model.pt --split test # ONCE, at the very end The output CSV (sha256, label, family, probability) is what thresholds and metrics are computed from. Scores are computed in ordinary 32-bit precision, the same as predict.py, so they match it. SAFETY: files are opened read-only and parsed as bytes. Nothing is executed. """ import argparse import csv import os import torch from torch.utils.data import DataLoader from predict import build_model, read_checkpoint from whole_model import KEYS, ChunkBudgetSampler, WholeFileDataset, collate ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) def main(): ap = argparse.ArgumentParser() ap.add_argument("--model", required=True) ap.add_argument("--split", choices=["val", "test"], required=True) ap.add_argument("--out", default=None, help="default: model/scores_.csv") ap.add_argument("--workers", type=int, default=24) args = ap.parse_args() manifest = os.path.join(ROOT, "dataset", "manifest.csv") out = args.out or os.path.join(ROOT, "model", f"scores_{args.split}.csv") if args.split == "test" and os.path.exists(out): raise SystemExit(f"{out} already exists: the test set is scored only once.") device = "cuda" if torch.cuda.is_available() else "cpu" config, state_dict, _ = read_checkpoint(args.model) model = build_model(config, state_dict).to(device) ds = WholeFileDataset(manifest, os.path.join(ROOT, "data"), args.split) family = {} with open(manifest, newline="") as f: for row in csv.DictReader(f): family[row["sha256"]] = row.get("family", "") loader = DataLoader(ds, batch_sampler=ChunkBudgetSampler(ds.n_chunks, 32, 8192, shuffle=False), collate_fn=collate, num_workers=args.workers) probs = [] with torch.no_grad(): for batch in loader: logit = model(*[batch[k].to(device) for k in KEYS]) probs.extend(torch.sigmoid(logit).cpu().tolist()) with open(out, "w", newline="") as f: w = csv.writer(f) w.writerow(["sha256", "label", "family", "probability"]) for (path, label, _), p in zip(ds.items, probs): sha = os.path.basename(path) w.writerow([sha, label, family.get(sha, ""), f"{p:.6f}"]) print(f"{len(probs)} files scored -> {out}") if __name__ == "__main__": main()