pe-malware-detector / src /evaluate.py
adamuremek's picture
Add MalConv baseline; predict() returns Malware/Benign
8f63b49 verified
Raw History Blame Contribute Delete
2.61 kB
"""Score every file of one split with a saved model and write the scores to a CSV.
python src/evaluate.py --model model/wholefile_transformer_halfchunks_segpos.pt --split val
python src/evaluate.py --model model/model.pt --split test # ONCE, at the very end
The output CSV (sha256, label, family, probability) is what thresholds and metrics are computed from.
Scores are computed in ordinary 32-bit precision, the same as predict.py, so they match it.
SAFETY: files are opened read-only and parsed as bytes. Nothing is executed.
"""
import argparse
import csv
import os
import torch
from torch.utils.data import DataLoader
from predict import build_model, read_checkpoint
from whole_model import KEYS, ChunkBudgetSampler, WholeFileDataset, collate
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True)
ap.add_argument("--split", choices=["val", "test"], required=True)
ap.add_argument("--out", default=None, help="default: model/scores_<split>.csv")
ap.add_argument("--workers", type=int, default=24)
args = ap.parse_args()
manifest = os.path.join(ROOT, "dataset", "manifest.csv")
out = args.out or os.path.join(ROOT, "model", f"scores_{args.split}.csv")
if args.split == "test" and os.path.exists(out):
raise SystemExit(f"{out} already exists: the test set is scored only once.")
device = "cuda" if torch.cuda.is_available() else "cpu"
config, state_dict, _ = read_checkpoint(args.model)
model = build_model(config, state_dict).to(device)
ds = WholeFileDataset(manifest, os.path.join(ROOT, "data"), args.split)
family = {}
with open(manifest, newline="") as f:
for row in csv.DictReader(f):
family[row["sha256"]] = row.get("family", "")
loader = DataLoader(ds, batch_sampler=ChunkBudgetSampler(ds.n_chunks, 32, 8192, shuffle=False),
collate_fn=collate, num_workers=args.workers)
probs = []
with torch.no_grad():
for batch in loader:
logit = model(*[batch[k].to(device) for k in KEYS])
probs.extend(torch.sigmoid(logit).cpu().tolist())
with open(out, "w", newline="") as f:
w = csv.writer(f)
w.writerow(["sha256", "label", "family", "probability"])
for (path, label, _), p in zip(ds.items, probs):
sha = os.path.basename(path)
w.writerow([sha, label, family.get(sha, ""), f"{p:.6f}"])
print(f"{len(probs)} files scored -> {out}")
if __name__ == "__main__":
main()