File size: 2,607 Bytes
71bf6da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f63b49
71bf6da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f63b49
 
71bf6da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
"""Score every file of one split with a saved model and write the scores to a CSV.

    python src/evaluate.py --model model/wholefile_transformer_halfchunks_segpos.pt --split val
    python src/evaluate.py --model model/model.pt --split test          # ONCE, at the very end

The output CSV (sha256, label, family, probability) is what thresholds and metrics are computed from.
Scores are computed in ordinary 32-bit precision, the same as predict.py, so they match it.

SAFETY: files are opened read-only and parsed as bytes. Nothing is executed.
"""
import argparse
import csv
import os

import torch
from torch.utils.data import DataLoader

from predict import build_model, read_checkpoint
from whole_model import KEYS, ChunkBudgetSampler, WholeFileDataset, collate

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", required=True)
    ap.add_argument("--split", choices=["val", "test"], required=True)
    ap.add_argument("--out", default=None, help="default: model/scores_<split>.csv")
    ap.add_argument("--workers", type=int, default=24)
    args = ap.parse_args()

    manifest = os.path.join(ROOT, "dataset", "manifest.csv")
    out = args.out or os.path.join(ROOT, "model", f"scores_{args.split}.csv")
    if args.split == "test" and os.path.exists(out):
        raise SystemExit(f"{out} already exists: the test set is scored only once.")
    device = "cuda" if torch.cuda.is_available() else "cpu"
    config, state_dict, _ = read_checkpoint(args.model)
    model = build_model(config, state_dict).to(device)

    ds = WholeFileDataset(manifest, os.path.join(ROOT, "data"), args.split)
    family = {}
    with open(manifest, newline="") as f:
        for row in csv.DictReader(f):
            family[row["sha256"]] = row.get("family", "")
    loader = DataLoader(ds, batch_sampler=ChunkBudgetSampler(ds.n_chunks, 32, 8192, shuffle=False),
                        collate_fn=collate, num_workers=args.workers)
    probs = []
    with torch.no_grad():
        for batch in loader:
            logit = model(*[batch[k].to(device) for k in KEYS])
            probs.extend(torch.sigmoid(logit).cpu().tolist())
    with open(out, "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["sha256", "label", "family", "probability"])
        for (path, label, _), p in zip(ds.items, probs):
            sha = os.path.basename(path)
            w.writerow([sha, label, family.get(sha, ""), f"{p:.6f}"])
    print(f"{len(probs)} files scored -> {out}")


if __name__ == "__main__":
    main()