benchmarks/tools/build_version_progress_benchmark.py

#!/usr/bin/env python3
from __future__ import annotations

import hashlib
import json
from datetime import datetime
from pathlib import Path
from typing import Any


ROOT = Path(__file__).resolve().parents[1]
V6_COMPARISON = ROOT / "artifacts/nanosoc1-v7-eval/detection-comparison.json"
V8_COMPARISON = ROOT / "artifacts/nanosoc1-v8-eval/base-v8-detection-comparison.json"
HOLDOUT = ROOT / "data/prepared_kademe7/detection-holdout.jsonl.gz"
OUTPUT = ROOT / "artifacts/benchmarks/foundation-v6-v8-same-holdout.json"
SAMPLE_ID_SHA256 = "3e6cdb15d3e14d3cf69135e5f7c021bf6c65394e74942d7b828dbe845376ec30"


def read(path: Path) -> dict[str, Any]:
    return json.loads(path.read_text(encoding="utf-8"))


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def delta(new: Any, old: Any) -> float | None:
    if new is None or old is None:
        return None
    return round(100 * (float(new) - float(old)), 6)


def main() -> None:
    v6_report = read(V6_COMPARISON)
    v8_report = read(V8_COMPARISON)
    base = v8_report["base"]
    v6 = v6_report["nanosoc"]
    v8 = v8_report["nanosoc"]

    if v6_report["comparison_type"] != v8_report["comparison_type"]:
        raise ValueError("comparison contracts differ")
    if base["count"] != 1000 or v6["count"] != 1000 or v8["count"] != 1000:
        raise ValueError("expected the frozen 1,000-event holdout")
    if (base["positive_count"], base["negative_count"]) != (200, 800):
        raise ValueError("unexpected class distribution")
    for name, value in v6_report["base"]["metrics"].items():
        if base["metrics"].get(name) != value:
            raise ValueError(f"base metric changed between reports: {name}")

    metric_names = (
        "json_valid_rate",
        "accuracy",
        "detection_recall",
        "precision",
        "specificity",
        "f1",
        "false_positive_rate",
        "false_negative_rate",
        "negative_unclassified_rate",
        "mitre_exact_accuracy",
        "invalid_or_other_output_rate",
    )

    variants = {
        "foundation_sec_base": {
            "label": "Foundation-Sec-1.1-8B-Instruct",
            "artifact_role": "base model, no adapter",
            "metrics": {name: base["metrics"][name] for name in metric_names},
            "confusion": base["confusion"],
        },
        "nanosoc_v6": {
            "label": "NanoSOC v6-era detection route",
            "artifact_role": "candidate5 adapter inherited by the v6 product pipeline for this task",
            "metrics": {name: v6["metrics"][name] for name in metric_names},
            "confusion": v6["confusion"],
        },
        "nanosoc_v8_current": {
            "label": "Llama-NanoSOC1-8B v8",
            "artifact_role": "current published v8 research adapter",
            "metrics": {name: v8["metrics"][name] for name in metric_names},
            "confusion": v8["confusion"],
        },
    }
    variants["foundation_sec_base"]["metrics"]["balanced_accuracy"] = 0.0
    variants["nanosoc_v6"]["metrics"]["balanced_accuracy"] = round(
        (v6["metrics"]["detection_recall"] + v6["metrics"]["specificity"]) / 2, 8
    )
    variants["nanosoc_v8_current"]["metrics"]["balanced_accuracy"] = round(
        (v8["metrics"]["detection_recall"] + v8["metrics"]["specificity"]) / 2, 8
    )

    all_metrics = (*metric_names, "balanced_accuracy")
    report = {
        "schema": "nanosoc.version_progress.same_holdout.v1",
        "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
        "benchmark": "Foundation-Sec base vs NanoSOC v6 vs current v8",
        "comparison_type": v8_report["comparison_type"],
        "holdout": {
            "name": "HIKARI-2021 negative-heavy frozen holdout",
            "events": 1000,
            "malicious": 200,
            "benign": 800,
            "file_sha256": sha256(HOLDOUT),
            "ordered_sample_id_sha256": SAMPLE_ID_SHA256,
            "sample_identity_verified_across_all_three_runs": True,
        },
        "variants": variants,
        "absolute_percentage_point_change": {
            "v6_to_v8": {
                name: delta(variants["nanosoc_v8_current"]["metrics"][name], variants["nanosoc_v6"]["metrics"][name])
                for name in all_metrics
            },
            "base_to_v8": {
                name: delta(variants["nanosoc_v8_current"]["metrics"][name], variants["foundation_sec_base"]["metrics"][name])
                for name in all_metrics
            },
        },
        "source_reports": [
            str(V6_COMPARISON.relative_to(ROOT)),
            str(V8_COMPARISON.relative_to(ROOT)),
        ],
        "limitations": [
            "The frozen holdout is an in-domain offline event-triage diagnostic, not packet-level IDS or appliance throughput evidence.",
            "The v6 label denotes the candidate5 detection adapter inherited by the v6 product pipeline for this task; v4-v6 primarily added deterministic product layers.",
            "Foundation-Sec produced no valid task-contract JSON. Strict specificity therefore counts negative outputs as unclassified errors, while FPR alone would misleadingly appear as zero.",
            "V8 improved substantially over v6 on this set, but its 32.125% false-positive rate still fails the 3% product target.",
            "This holdout became development-known after the first evaluation; it does not establish production generalization.",
        ],
        "product_claim": "NOT_PRODUCT_READY",
    }
    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(OUTPUT)


if __name__ == "__main__":
    main()