scripts/eval.py

#!/usr/bin/env python3
"""
eval.py -- Task 4: honest evaluation of the tuned Glimmer-Sentry-30B adapter.

Run entirely inside WSL, from the venv at ~/glimmer/venv (needs torch/unsloth/
peft for generation, sigma-cli + pySigma backends for §2, yara-python for §3).
GPU 1 is free -- launch with CUDA_VISIBLE_DEVICES=1 set by the caller (this
script does not set it itself, matching scripts/train.py's convention):

    CUDA_VISIBLE_DEVICES=1 ~/glimmer/venv/bin/python \\
        /mnt/c/Users/Dwain-Admin/Desktop/GLIMMER-SENTRY-30B/scripts/eval.py --stage all

Stages (--stage translation|sigma_authoring|yara|qualitative|all):
    translation     -- §1: 50 holdout rules x 2 backends (KQL, SPL), greedy decode,
                        exact + normalised match, malformed-fence count.
    sigma_authoring -- §2: 50 holdout rules, description+tags+logsource -> rule,
                        greedy decode. % valid YAML / % `sigma check` / % `sigma
                        convert`. Also: recall-similarity vs the real holdout rule
                        text (mandatory addendum -- base model is known to have
                        memorized public SigmaHQ rules from its own pretraining;
                        this quantifies near-copies vs novel constructions so the
                        authoring numbers are never presented as pure generation
                        skill).
    yara            -- §3: 30 YARA descriptions NOT used in training (reconstructed
                        from dataset/_stage/*.jsonl + the real assemble_stats.json
                        targets -- see select_yara_eval_items()), greedy decode,
                        %compiles via yara-python, error-category breakdown.
    qualitative     -- §4 + §5 combined (the brief's --stage list only names 4
                        stages; §5's 3 chat-regression prompts are folded in here
                        since both are "full outputs in RESULTS.md" demonstrations
                        rather than aggregate metrics):
                          §4: 5 prompts (translation/authoring/explanation/fptuning/
                              yara authoring), BASE model vs TUNED model, identical
                              sampling (temp 1.0/top_p 0.95/top_k 64), fixed seed 42
                              per prompt so both models see the same sampling noise.
                          §5: 3 general prompts, TUNED model only, greedy (a sanity
                              gate, not a benchmark -- see module docstring below).
    all             -- runs all 4 above, loading the tuned model once for
                        translation+sigma_authoring+yara+qualitative's tuned half,
                        freeing it, then loading the base model once for
                        qualitative's base half. Two model loads total, not five.

Every stage writes/updates eval/results.json (merged, so stages can be re-run
independently without clobbering earlier ones) and eval/generations/<stage>/*.txt
(raw model outputs, redacted nothing -- eval/generations/ on /mnt/c is the brief's
explicit Defender exception for small individual text files). eval/RESULTS.md is
regenerated from the merged results.json at the end of every invocation.

Decoding: metric stages (1-3) use greedy (do_sample=False), max_new_tokens=512,
per the brief. Stage 4 (qualitative) uses temperature=1.0/top_p=0.95/top_k=64 per
the brief, matching scripts/train.py's own held-out generation settings. Stage 5
(chat-regression) uses greedy -- not specified by the brief, chosen here for
reproducibility since it is a coherence sanity check, not a creativity demo;
flagged explicitly in RESULTS.md.
"""
from __future__ import annotations

import argparse
import difflib
import gc
import itertools
import json
import os
import random
import re
import statistics
import subprocess
import sys
import time
from pathlib import Path

# ---------------------------------------------------------------------------
# Environment -- must happen before any HF/transformers/unsloth import.
# ---------------------------------------------------------------------------
GLIMMER_HOME = Path(os.environ.get("GLIMMER_HOME", os.path.expanduser("~/glimmer")))
os.environ.setdefault("HF_HOME", str(GLIMMER_HOME / "hf_home"))
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

REPO = Path(__file__).resolve().parent.parent
DATASET = REPO / "dataset"
STAGE_DIR = DATASET / "_stage"
HOLDOUT_DIR = DATASET / "holdout"
HOLDOUT_IDS_PATH = DATASET / "holdout_ids.json"
YARA_HOLDOUT_DIR = DATASET / "yara_holdout"  # v0.2: 40-rule dedicated YARA holdout (Task 2b), 30 of which
YARA_HOLDOUT_IDS_PATH = DATASET / "yara_holdout_ids.json"  # carry from_v01_eval=true -- the exact 30 items
                                                            # v0.1's `yara` stage evaluated (see select_yara_eval_items)
EVAL_DIR = REPO / "eval"
GEN_DIR = EVAL_DIR / "generations"
RESULTS_JSON = EVAL_DIR / "results.json"
RESULTS_V02_JSON = EVAL_DIR / "results_v02.json"  # v0.2: NEVER the same file as RESULTS_JSON -- v0.1 numbers
                                                   # are load-bearing and must never be overwritten (task-4c brief)
RESULTS_MD = EVAL_DIR / "RESULTS.md"
EVAL_TMP = GLIMMER_HOME / "eval_tmp"  # WSL-local, never /mnt/c -- per-item .yml scratch for sigma check/convert

BASE_MODEL_NAME = "meta-models/Muse-Glimmer-30B"
ADAPTER_CKPT_DEFAULT = GLIMMER_HOME / "runs" / "sentry-v01" / "checkpoint-229"

REASONING_STRENGTH = "high"  # matches scripts/train.py -- must match at both train and eval time
MAX_SEQ_LENGTH = 2048  # matches the config actually used for the full training run (task-3-report §3/§9)

METRIC_MAX_NEW_TOKENS = 512  # §1/§2/§3, brief-mandated
QUALITATIVE_MAX_NEW_TOKENS = 512  # §4, matches train.py's own held-out generation
CHAT_REGRESSION_MAX_NEW_TOKENS = 400  # §5

QUALITATIVE_SAMPLING = dict(do_sample=True, temperature=1.0, top_p=0.95, top_k=64)
QUALITATIVE_SEED = 42

RECALL_NEAR_COPY_THRESHOLD = 0.9

sys.path.insert(0, str(Path(__file__).resolve().parent))
# Reuse build_dataset.py's own prompt-template banks and helpers verbatim
# (brief: "SAME prompt templates build_dataset.py used ... import or replicate;
# note which" -- this script IMPORTS them). Each bank has several phrasings for
# training-time variety; eval deterministically uses index [0] from each bank
# (the plain, canonical phrasing) for every generation, documented in RESULTS.md,
# so the eval is exactly reproducible rather than depending on an RNG draw.
from build_dataset import (  # noqa: E402
    TRANSLATION_TEMPLATES,
    AUTHORING_TEMPLATES,
    EXPLANATION_TEMPLATES,
    FPTUNING_TEMPLATES,
    YARA_AUTHORING_TEMPLATES,
    logsource_str,
    fp_text_from_list,
    needed_imports,
    FALSEPOSITIVE_NOISE_VALUES,
)

# v0.2 (Task 4c) -- reuse build_dataset_v02.py's own corruption logic and prompt
# template banks for the two NEW eval stages (yara_repair, yara_from_iocs), same
# "import, don't reimplement" rule as above so the eval prompt shape is
# byte-identical to what the model was actually trained on.
from build_dataset_v02 import (  # noqa: E402
    CORRUPTION_CLASSES,
    CORRUPTORS,
    compile_or_error,
    extract_indicators,
    format_indicators,
    YARA_REPAIR_TEMPLATES,
    YARA_FROM_IOCS_TEMPLATES,
)

import yaml  # noqa: E402


def _sigma_executable() -> str:
    """Resolve the `sigma` CLI console-script next to the running Python
    interpreter (i.e. inside the same venv the caller invoked us with), rather
    than relying on `sigma` being resolvable via PATH. A systemd-run unit that
    invokes ~/glimmer/venv/bin/python directly (without `source
    ~/glimmer/venv/bin/activate` first) does NOT get ~/glimmer/venv/bin on
    PATH, so a bare subprocess.run(["sigma", ...]) raises FileNotFoundError --
    hit for real during the Task 4c v0.2 sigma_authoring run (see
    briefs/task-4c-report.md). Falls back to the bare name (prior behavior,
    relies on PATH) if no sibling executable is found next to sys.executable."""
    candidate = Path(sys.executable).parent / ("sigma.exe" if os.name == "nt" else "sigma")
    return str(candidate) if candidate.exists() else "sigma"


SIGMA_BIN = _sigma_executable()


def log(msg: str) -> None:
    print(f"[eval.py {time.strftime('%H:%M:%S')}] {msg}", flush=True)


def read_json(path: Path):
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)


def write_json(path: Path, obj) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(obj, f, ensure_ascii=False, indent=2)


def read_jsonl(path: Path):
    with open(path, "r", encoding="utf-8") as f:
        return [json.loads(line) for line in f if line.strip()]


def write_text(path: Path, text: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(text, encoding="utf-8")


# ---------------------------------------------------------------------------
# Text extraction / scoring helpers
# ---------------------------------------------------------------------------
FENCE_RE = re.compile(r"```[ \t]*[a-zA-Z0-9_+-]*\r?\n(.*?)```", re.DOTALL)
TEMPLATE_TOKEN_RE = re.compile(r"<\|[^|]*\|>|to=(?:self|user)")


def strip_template_tokens(text: str) -> str:
    return TEMPLATE_TOKEN_RE.sub("", text).strip()


def extract_code_block(text: str) -> tuple[str, bool]:
    """Return (code, malformed). Takes the LAST fenced code block in the
    generation (this model's chat template does a `to=self` reasoning turn
    then, when it produces one, a `to=user` final-answer turn -- an earlier
    fence in the reasoning is not the answer). If no fence is found at all,
    malformed=True and the whole completion (template tokens stripped) is
    returned as the tolerant fallback, per the brief."""
    matches = FENCE_RE.findall(text)
    if matches:
        return matches[-1].strip(), False
    return strip_template_tokens(text), True


def normalize_match(s: str) -> str:
    """Collapse internal whitespace runs to one space, strip trailing
    whitespace/semicolons. Nothing semantic (no reordering, no case-folding)."""
    s = re.sub(r"\s+", " ", s.strip())
    return s.rstrip(";").strip()


def normalize_for_similarity(s: str) -> str:
    return re.sub(r"\s+", " ", s.strip().lower())


def recall_similarity(generated: str, ground_truth: str) -> float:
    a, b = normalize_for_similarity(generated), normalize_for_similarity(ground_truth)
    if not a or not b:
        return 0.0
    return difflib.SequenceMatcher(None, a, b).ratio()


YARA_ERROR_PATTERNS = [
    ("syntax_error", re.compile(r"syntax error", re.IGNORECASE)),
    ("duplicate_identifier", re.compile(r"duplicate", re.IGNORECASE)),
    ("undefined_identifier", re.compile(r"undefined (identifier|string)|unknown identifier", re.IGNORECASE)),
    ("unterminated_string_or_comment", re.compile(r"unterminated", re.IGNORECASE)),
    ("unclosed_brace_or_paren", re.compile(r"unexpected \$end|mismatched|unbalanced", re.IGNORECASE)),
    ("import_error", re.compile(r"import", re.IGNORECASE)),
    ("empty_or_no_rule", re.compile(r"no rule|empty", re.IGNORECASE)),
]


def categorize_yara_error(msg: str) -> str:
    for label, pat in YARA_ERROR_PATTERNS:
        if pat.search(msg):
            return label
    return "other"


# ---------------------------------------------------------------------------
# Holdout loading
# ---------------------------------------------------------------------------
def load_holdout_items() -> list[dict]:
    """Returns all 50 holdout items, sorted by rule id for a stable, fully
    reproducible iteration order. Each item carries the raw YAML text (used
    verbatim in translation prompts, exactly like build_dataset.py's
    raw_yaml_for_prompt) plus the parsed fields needed for authoring prompts."""
    ids = read_json(HOLDOUT_IDS_PATH)
    items = []
    for rule_id in sorted(ids.keys()):
        info = ids[rule_id]
        raw_path = HOLDOUT_DIR / info["filename"]
        raw_yaml = raw_path.read_text(encoding="utf-8")
        parsed = yaml.safe_load(raw_yaml) or {}
        items.append(
            {
                "rule_id": rule_id,
                "filename": info["filename"],
                "title": info["title"],
                "raw_yaml": raw_yaml,
                "description": parsed.get("description") or parsed.get("title") or "",
                "tags": parsed.get("tags") or [],
                "logsource": parsed.get("logsource") or {},
                "falsepositives": parsed.get("falsepositives") or [],
                "kql_truth": info["kql"],
                "spl_truth": info["spl"],
            }
        )
    return items


# ---------------------------------------------------------------------------
# YARA eval-set reconstruction (§3)
# ---------------------------------------------------------------------------
def select_yara_eval_items(n: int = 30, seed: int = 42):
    """Reconstructs exactly which yara_authoring pool items were sampled into
    the real training run, then deterministically samples `n` items from the
    complement (never-seen-in-training) pool. This is possible because
    stage_assemble()'s sampling is a fixed sequence of random.Random(SEED)
    .sample() calls over pools in a fixed dict-literal order (kql, spl, wazuh,
    authoring, explanation, fptuning, yara_authoring, yara_explanation,
    general) -- replaying that exact sequence with the exact same pool
    contents (dataset/_stage/pool_*.jsonl, unchanged since Phase 2) and the
    exact same per-task targets (dataset/_stage/assemble_stats.json's real
    "targets", not recomputed) reproduces the identical random.sample() draws.
    Verified before wiring this in: the reconstructed pool sizes match
    assemble_stats.json's own "pool_sizes" exactly, and the reconstructed
    yara_authoring draw count matches its "targets"."yara_authoring" (650)
    exactly -- see task-4-report.md for the standalone verification.

    Returns (chosen: list[dict], qual_extra: dict, caveat: str | None).
    qual_extra is one further distinct unused item, for the §4 qualitative
    YARA-authoring prompt, guaranteed disjoint from `chosen`.

    Falls back to the brief's stated fallback (signature-base rules excluded
    from the authoring pool for lacking descriptions, prompts built from rule
    name/strings instead) if the stage files are missing or the pool-size
    assertion fails, with `caveat` set to a human-readable note for RESULTS.md.
    """
    try:
        holdout_ids = set(read_json(STAGE_DIR / "holdout_id_set.json"))
        stats = read_json(STAGE_DIR / "assemble_stats.json")
        targets = stats["targets"]
        expected_sizes = stats["pool_sizes"]

        translation_pool = [x for x in read_jsonl(STAGE_DIR / "pool_translation.jsonl") if x["rule_id"] not in holdout_ids]
        kql_pool = [x for x in translation_pool if x["task_type"] == "sigma_to_kql"]
        spl_pool = [x for x in translation_pool if x["task_type"] == "sigma_to_spl"]
        wazuh_pool = [x for x in read_jsonl(STAGE_DIR / "pool_wazuh.jsonl") if x["rule_id"] not in holdout_ids]
        authoring_pool = [x for x in read_jsonl(STAGE_DIR / "pool_authoring.jsonl") if x["rule_id"] not in holdout_ids]
        explanation_pool = [x for x in read_jsonl(STAGE_DIR / "pool_explanation.jsonl") if x["rule_id"] not in holdout_ids]
        fptuning_pool = [x for x in read_jsonl(STAGE_DIR / "pool_fptuning.jsonl") if x["rule_id"] not in holdout_ids]
        yara_authoring_pool = read_jsonl(STAGE_DIR / "pool_yara_authoring.jsonl")

        got_sizes = {
            "sigma_to_kql": len(kql_pool), "sigma_to_spl": len(spl_pool),
            "sigma_to_wazuh": len(wazuh_pool), "sigma_authoring": len(authoring_pool),
            "sigma_explanation": len(explanation_pool), "sigma_fptuning": len(fptuning_pool),
            "yara_authoring": len(yara_authoring_pool),
        }
        for k, v in got_sizes.items():
            if expected_sizes.get(k) != v:
                raise RuntimeError(f"pool-size mismatch for {k}: reconstructed {v}, assemble_stats.json says {expected_sizes.get(k)}")

        import random as _random
        sample_rng = _random.Random(42)  # SEED constant from build_dataset.py, replayed

        def sample_pool(pool, k):
            return list(pool) if k >= len(pool) else sample_rng.sample(pool, k)

        # Replay the exact draw sequence up to and including yara_authoring
        # (draws after it -- yara_explanation, general_mix -- don't affect
        # yara_authoring's already-consumed draw and aren't needed here).
        sample_pool(kql_pool, targets["sigma_to_kql"])
        sample_pool(spl_pool, targets["sigma_to_spl"])
        sample_pool(wazuh_pool, targets["sigma_to_wazuh"])
        sample_pool(authoring_pool, targets["sigma_authoring"])
        sample_pool(explanation_pool, targets["sigma_explanation"])
        sample_pool(fptuning_pool, targets["sigma_fptuning"])
        used_yara = sample_pool(yara_authoring_pool, targets["yara_authoring"])

        def item_key(it):
            return f"{it['source']}:{it['file']}:{it['rule_name']}"

        used_keys = {item_key(it) for it in used_yara}
        unused = [it for it in yara_authoring_pool if item_key(it) not in used_keys]

        eval_rng = _random.Random(seed)
        chosen = eval_rng.sample(unused, min(n, len(unused)))
        chosen_keys = {item_key(it) for it in chosen}
        qual_extra = next(it for it in unused if item_key(it) not in chosen_keys)

        log(f"select_yara_eval_items: reconstructed {len(used_yara)} used / {len(unused)} unused "
            f"of {len(yara_authoring_pool)} yara_authoring pool items; sampled {len(chosen)} for eval.")
        return chosen, qual_extra, None

    except Exception as e:
        log(f"WARNING: yara eval-set reconstruction failed ({e!r}); falling back to the brief's stated "
            f"fallback -- signature-base rules excluded from the authoring pool for lacking descriptions.")
        sigbase_dir = REPO / "data" / "signature-base" / "yara"
        from build_dataset import split_yara_rules, extract_yara_meta  # local import, only needed on fallback path
        import glob as _glob
        no_desc = []
        for fpath in sorted(_glob.glob(str(sigbase_dir / "*.yar"))):
            text = Path(fpath).read_text(encoding="utf-8", errors="replace")
            try:
                blocks = split_yara_rules(text)
            except Exception:
                continue
            for name, block_text in blocks:
                meta = extract_yara_meta(block_text)
                if not (meta.get("description") or meta.get("Description")):
                    no_desc.append({"source": "signature-base-nodesc", "file": Path(fpath).name,
                                     "rule_name": name, "rule_text": block_text, "meta": meta})
        import random as _random
        eval_rng = _random.Random(seed)
        chosen = eval_rng.sample(no_desc, min(n, len(no_desc)))
        qual_extra = eval_rng.choice(no_desc)
        caveat = (f"YARA eval-set reconstruction from dataset/_stage/ FAILED ({e}); fell back to "
                  f"signature-base rules with no `description` meta field (excluded from the training "
                  f"authoring pool for that reason) -- prompts below are built from rule NAME + a strings "
                  f"summary instead of a natural-language description, per the brief's fallback clause.")
        return chosen, qual_extra, caveat


def yara_prompt_for_item(item: dict) -> str:
    if item.get("description"):
        return YARA_AUTHORING_TEMPLATES[0].format(description=item["description"])
    # fallback-path item: no description, build a prompt from name + string count
    rule_text = item.get("rule_text", "")
    string_count = rule_text.count("$")
    desc = f"a threat matching the pattern indicators of a rule named '{item['rule_name']}' (~{string_count} string indicator(s))"
    return YARA_AUTHORING_TEMPLATES[0].format(description=desc)


# ---------------------------------------------------------------------------
# v0.2 (Task 4c): the DEDICATED 40-rule YARA holdout (dataset/yara_holdout_ids.json,
# built by build_dataset_v02.py's `yara_holdout` stage) -- unlike v0.1's
# select_yara_eval_items() above (which had to *reconstruct* an "unused" set
# after the fact because v0.1 never carved out a real YARA holdout), v0.2 has a
# proper held-out set that was excluded from training from the start. 30 of
# its 40 items are flagged from_v01_eval=true -- these are the exact 30
# identities v0.1's `yara` stage evaluated, recovered from eval/results.json
# and folded into this holdout by build_dataset_v02.py -- so filtering on that
# flag gives the direct, apples-to-apples v0.1-vs-v0.2 subset without a second
# model run.
# ---------------------------------------------------------------------------
def yara_holdout_item_key(item: dict) -> str:
    return f"{item['source']}:{item['file']}:{item['rule_name']}"


def load_yara_holdout_items() -> list[dict]:
    """All 40 dataset/yara_holdout_ids.json entries, each with its rule_text
    read from dataset/yara_holdout/<filename>, sorted by identity key for a
    stable, fully reproducible iteration order (same discipline as
    load_holdout_items() above)."""
    ids = read_json(YARA_HOLDOUT_IDS_PATH)
    items = []
    for key in sorted(ids.keys()):
        info = ids[key]
        rule_text = (YARA_HOLDOUT_DIR / info["filename"]).read_text(encoding="utf-8")
        items.append({"key": key, "rule_text": rule_text, **info})
    return items


def select_yara_repair_items(holdout_items: list[dict], n: int = 15, seed: int = 42) -> list[dict]:
    rng = random.Random(seed)
    return rng.sample(holdout_items, min(n, len(holdout_items)))


def select_yara_iocs_items(holdout_items: list[dict], exclude_keys: set, n: int = 10, seed: int = 42) -> list[dict]:
    remaining = [it for it in holdout_items if it["key"] not in exclude_keys]
    rng = random.Random(seed)
    return rng.sample(remaining, min(n, len(remaining)))


def pick_yara_qual_holdout_item(holdout_items: list[dict], exclude_keys: set, seed: int = 42) -> dict:
    """One further holdout item, disjoint from the repair(15) + from_iocs(10)
    selections, for the qualitative base-vs-tuned YARA-authoring side-by-side."""
    remaining = sorted((it for it in holdout_items if it["key"] not in exclude_keys), key=yara_holdout_item_key)
    rng = random.Random(seed)
    return rng.choice(remaining)


# ---------------------------------------------------------------------------
# Model load / free
# ---------------------------------------------------------------------------
def get_text_tokenizer(tokenizer_or_processor):
    """Same gotcha as train.py: FastLanguageModel.from_pretrained returns a
    MuseGlimmerProcessor; the plain tokenizer for all text-only ops is `.tokenizer`."""
    return getattr(tokenizer_or_processor, "tokenizer", tokenizer_or_processor)


def load_tuned_model(adapter_dir: Path):
    import torch
    from unsloth import FastLanguageModel

    log(f"Loading TUNED model (base + adapter) from {adapter_dir} ...")
    model, proc = FastLanguageModel.from_pretrained(
        model_name=str(adapter_dir),
        max_seq_length=MAX_SEQ_LENGTH,
        load_in_4bit=True,
        dtype=torch.bfloat16,
        device_map="sequential",
    )
    tok = get_text_tokenizer(proc)
    FastLanguageModel.for_inference(model)
    log(f"Tuned model loaded. type={type(model).__name__} has_peft={hasattr(model, 'peft_config')}")
    return model, tok


def load_base_model():
    import torch
    from unsloth import FastLanguageModel

    log(f"Loading BASE model (no adapter) from {BASE_MODEL_NAME} ...")
    model, proc = FastLanguageModel.from_pretrained(
        model_name=BASE_MODEL_NAME,
        max_seq_length=MAX_SEQ_LENGTH,
        load_in_4bit=True,
        dtype=torch.bfloat16,
        device_map="sequential",
    )
    tok = get_text_tokenizer(proc)
    FastLanguageModel.for_inference(model)
    log(f"Base model loaded. type={type(model).__name__}")
    return model, tok


def free_model(model) -> None:
    import torch

    del model
    gc.collect()
    torch.cuda.empty_cache()
    log("Model freed, CUDA cache emptied.")


def generate(model, tok, user_content: str, max_new_tokens: int, do_sample: bool = False,
             temperature=None, top_p=None, top_k=None, seed=None):
    import torch

    if seed is not None:
        torch.manual_seed(seed)
    messages = [{"role": "user", "content": user_content}]
    inputs = tok.apply_chat_template(
        messages, tokenize=True, add_generation_prompt=True,
        reasoning_strength=REASONING_STRENGTH, return_tensors="pt",
    ).to(model.device)
    gen_kwargs = dict(input_ids=inputs, max_new_tokens=max_new_tokens, do_sample=do_sample)
    if do_sample:
        gen_kwargs.update(temperature=temperature, top_p=top_p, top_k=top_k)
    t0 = time.time()
    with torch.no_grad():
        out_ids = model.generate(**gen_kwargs)
    elapsed = time.time() - t0
    gen_text = tok.decode(out_ids[0][inputs.shape[1]:], skip_special_tokens=False)
    return gen_text, elapsed


# ---------------------------------------------------------------------------
# Stage 1: translation
# ---------------------------------------------------------------------------
def run_translation_stage(model, tok, holdout_items: list[dict], max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    log(f"=== Stage: translation ({len(holdout_items)} rules x 2 backends, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    per_backend = {"kql": [], "spl": []}

    for item in holdout_items:
        for backend, target_name, fence, truth_key in (
            ("kql", "Microsoft 365 Defender Advanced Hunting KQL", "kql", "kql_truth"),
            ("spl", "Splunk SPL", "spl", "spl_truth"),
        ):
            user = TRANSLATION_TEMPLATES[0].format(target=target_name, yaml=item["raw_yaml"], description=item["description"])
            gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
            code, malformed = extract_code_block(gen_text)
            truth = item[truth_key]
            exact = code.strip() == truth.strip()
            normalized = normalize_match(code) == normalize_match(truth)

            gen_path = GEN_DIR / "translation" / f"{item['rule_id']}_{backend}.txt"
            write_text(gen_path, gen_text)

            per_backend[backend].append({
                "rule_id": item["rule_id"], "filename": item["filename"],
                "exact": exact, "normalized": normalized, "malformed_fence": malformed,
                "seconds": round(elapsed, 2),
                "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
                "extracted": code, "ground_truth": truth,
            })
            log(f"  [{backend}] {item['rule_id']} exact={exact} normalized={normalized} "
                f"malformed={malformed} ({elapsed:.1f}s)")

    result = {"n_rules": len(holdout_items), "timing_seconds": round(time.time() - t_start, 1)}
    for backend in ("kql", "spl"):
        items = per_backend[backend]
        n = len(items)
        mismatches = [it for it in items if not it["normalized"]][:3]
        result[backend] = {
            "n": n,
            "exact_match_pct": round(100 * sum(i["exact"] for i in items) / n, 1) if n else 0.0,
            "normalized_match_pct": round(100 * sum(i["normalized"] for i in items) / n, 1) if n else 0.0,
            "malformed_fence_count": sum(i["malformed_fence"] for i in items),
            "mismatch_examples": [
                {"rule_id": i["rule_id"], "filename": i["filename"],
                 "generated": i["extracted"], "ground_truth": i["ground_truth"]}
                for i in mismatches
            ],
            "items": [{k: v for k, v in i.items() if k not in ("extracted", "ground_truth")} for i in items],
        }
    log(f"Translation stage done in {result['timing_seconds']}s. "
        f"KQL exact={result['kql']['exact_match_pct']}% norm={result['kql']['normalized_match_pct']}% | "
        f"SPL exact={result['spl']['exact_match_pct']}% norm={result['spl']['normalized_match_pct']}%")
    return result


# ---------------------------------------------------------------------------
# Stage 1b: translation truncation footnote -- re-run ONLY the items that had
# malformed_fence=true in the committed @512 translation results, at a larger
# max_new_tokens, to test whether they were truncations. Does NOT touch/
# overwrite results["translation"] (the committed headline 80%/90% numbers);
# writes to results["translation_truncation_footnote"] instead.
# ---------------------------------------------------------------------------
def run_translation_truncation_footnote(model, tok, holdout_items: list[dict], existing_translation: dict, max_new_tokens: int) -> dict:
    holdout_by_id = {it["rule_id"]: it for it in holdout_items}
    targets = []  # (backend, rule_id, filename)
    for backend in ("kql", "spl"):
        for it in existing_translation.get(backend, {}).get("items", []):
            if it.get("malformed_fence"):
                targets.append((backend, it["rule_id"], it["filename"]))

    log(f"=== Stage: translation_truncation_footnote ({len(targets)} items, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    items_out = []
    for backend, rule_id, filename in targets:
        item = holdout_by_id[rule_id]
        target_name = "Microsoft 365 Defender Advanced Hunting KQL" if backend == "kql" else "Splunk SPL"
        truth = item["kql_truth"] if backend == "kql" else item["spl_truth"]
        user = TRANSLATION_TEMPLATES[0].format(target=target_name, yaml=item["raw_yaml"], description=item["description"])
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)
        exact = code.strip() == truth.strip()
        normalized = normalize_match(code) == normalize_match(truth)

        gen_path = GEN_DIR / "translation" / f"{rule_id}_{backend}_footnote{max_new_tokens}.txt"
        write_text(gen_path, gen_text)

        items_out.append({
            "rule_id": rule_id, "filename": filename, "backend": backend,
            "malformed_fence_at_512": True, "malformed_fence_now": malformed,
            "exact": exact, "normalized": normalized, "seconds": round(elapsed, 2),
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
            "generated": code, "ground_truth": truth,
        })
        log(f"  [{backend}] {rule_id} malformed_now={malformed} exact={exact} normalized={normalized} ({elapsed:.1f}s)")

    n = len(items_out)
    resolved = [i for i in items_out if not i["malformed_fence_now"]]
    result = {
        "max_new_tokens": max_new_tokens,
        "n_items": n,
        "timing_seconds": round(time.time() - t_start, 1),
        "resolved_fence_count": len(resolved),
        "became_exact_count": sum(1 for i in items_out if i["exact"]),
        "became_normalized_count": sum(1 for i in items_out if i["normalized"]),
        "note": (f"Re-run of the {n} translation generations that had malformed_fence=true in the "
                 f"committed @512 results, at max_new_tokens={max_new_tokens}. Does NOT change the "
                 "headline exact/normalised percentages recorded in results['translation'] (@512) -- "
                 "footnote only, per the brief."),
        "items": items_out,
    }
    log(f"Translation truncation footnote done in {result['timing_seconds']}s. "
        f"{result['resolved_fence_count']}/{n} fences resolved, {result['became_exact_count']}/{n} became exact.")
    return result


# ---------------------------------------------------------------------------
# Stage 2: sigma authoring
# ---------------------------------------------------------------------------
def run_sigma_authoring_stage(model, tok, holdout_items: list[dict], max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    log(f"=== Stage: sigma_authoring ({len(holdout_items)} rules, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    EVAL_TMP.mkdir(parents=True, exist_ok=True)
    items_out = []

    for item in holdout_items:
        tags_s = ", ".join(item["tags"]) if item["tags"] else "none recorded"
        user = AUTHORING_TEMPLATES[0].format(
            description=item["description"], logsource_str=logsource_str(item["logsource"]), tags_str=tags_s,
        )
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)

        gen_path = GEN_DIR / "sigma_authoring" / f"{item['rule_id']}.txt"
        write_text(gen_path, gen_text)

        parses_ok = True
        try:
            yaml.safe_load(code)
        except Exception:
            parses_ok = False

        tmp_yml = EVAL_TMP / f"authoring_{item['rule_id']}.yml"
        tmp_yml.write_text(code, encoding="utf-8")

        check_proc = subprocess.run([SIGMA_BIN, "check", str(tmp_yml)], capture_output=True, text=True, cwd=str(REPO))
        passes_check = check_proc.returncode == 0

        convert_proc = subprocess.run(
            [SIGMA_BIN, "convert", "-t", "kusto", "-p", "microsoft_365_defender", str(tmp_yml)],
            capture_output=True, text=True, cwd=str(REPO),
        )
        converts_ok = convert_proc.returncode == 0

        sim = recall_similarity(code, item["raw_yaml"])
        near_copy = sim > RECALL_NEAR_COPY_THRESHOLD

        items_out.append({
            "rule_id": item["rule_id"], "filename": item["filename"],
            "parses_as_yaml": parses_ok, "passes_sigma_check": passes_check, "converts_ok": converts_ok,
            "malformed_fence": malformed, "recall_similarity": round(sim, 4), "near_copy": near_copy,
            "seconds": round(elapsed, 2),
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
            "check_stderr_tail": (check_proc.stderr or "")[-300:] if not passes_check else "",
            "convert_stderr_tail": (convert_proc.stderr or "")[-300:] if not converts_ok else "",
        })
        log(f"  {item['rule_id']} yaml={parses_ok} check={passes_check} convert={converts_ok} "
            f"sim={sim:.3f} near_copy={near_copy} ({elapsed:.1f}s)")

    n = len(items_out)
    sims = [i["recall_similarity"] for i in items_out]
    near_copies = [i for i in items_out if i["near_copy"]]
    result = {
        "n_rules": n,
        "max_new_tokens": max_new_tokens,
        "timing_seconds": round(time.time() - t_start, 1),
        "parses_as_yaml_pct": round(100 * sum(i["parses_as_yaml"] for i in items_out) / n, 1) if n else 0.0,
        "passes_sigma_check_pct": round(100 * sum(i["passes_sigma_check"] for i in items_out) / n, 1) if n else 0.0,
        "converts_ok_pct": round(100 * sum(i["converts_ok"] for i in items_out) / n, 1) if n else 0.0,
        "recall_similarity": {
            "mean": round(statistics.mean(sims), 4) if sims else 0.0,
            "median": round(statistics.median(sims), 4) if sims else 0.0,
            "near_copy_threshold": RECALL_NEAR_COPY_THRESHOLD,
            "near_copy_count": len(near_copies),
            "near_copy_pct": round(100 * len(near_copies) / n, 1) if n else 0.0,
            "near_copy_rule_ids": [i["rule_id"] for i in near_copies],
        },
        "items": items_out,
    }
    log(f"Sigma authoring stage done in {result['timing_seconds']}s. "
        f"yaml={result['parses_as_yaml_pct']}% check={result['passes_sigma_check_pct']}% "
        f"convert={result['converts_ok_pct']}% near_copy={result['recall_similarity']['near_copy_pct']}%")
    return result


# ---------------------------------------------------------------------------
# Stage 3: yara
# ---------------------------------------------------------------------------
def run_yara_stage(model, tok, max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    chosen, qual_extra, caveat = select_yara_eval_items(n=30, seed=42)
    log(f"=== Stage: yara ({len(chosen)} prompts, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    import yara

    # Non-default max_new_tokens (e.g. the 1536 truncation re-eval) writes to a
    # separate generations subdir so it never overwrites the committed @512
    # raw outputs -- same "append, never overwrite" rule as results.json.
    gen_subdir = "yara" if max_new_tokens == METRIC_MAX_NEW_TOKENS else f"yara_{max_new_tokens}"

    items_out = []
    for idx, item in enumerate(chosen):
        user = yara_prompt_for_item(item)
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)

        safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", item["rule_name"])[:60]
        gen_path = GEN_DIR / gen_subdir / f"{idx:02d}_{safe_name}.txt"
        write_text(gen_path, gen_text)

        compile_ok, compile_err = _try_yara_compile(yara, code)
        compile_ok_fixed, compile_err_fixed = (compile_ok, compile_err)
        used_auto_import = False
        if not compile_ok:
            imports = needed_imports(code)
            if imports:
                fixed_source = "".join(f'import "{imp}"\n' for imp in sorted(imports)) + code
                compile_ok_fixed, compile_err_fixed = _try_yara_compile(yara, fixed_source)
                used_auto_import = True

        error_category = None if compile_ok else categorize_yara_error(compile_err or "")

        items_out.append({
            "index": idx, "source": item.get("source"), "file": item.get("file"), "rule_name": item.get("rule_name"),
            "description": item.get("description", ""), "malformed_fence": malformed,
            "compile_ok": compile_ok, "compile_ok_with_auto_import": compile_ok_fixed,
            "used_auto_import_retry": used_auto_import, "error_category": error_category,
            "compile_error": (compile_err or "")[:300] if not compile_ok else "",
            "seconds": round(elapsed, 2),
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
        })
        log(f"  [{idx:02d}] {item.get('rule_name')} compile={compile_ok} "
            f"(with_auto_import={compile_ok_fixed}) ({elapsed:.1f}s)")

    # §4's YARA-authoring qualitative prompt reuses this stage's reconstruction
    # (guaranteed disjoint from the 30 above) -- stash it for the qualitative stage.
    write_json(STAGE_DIR / "_eval_yara_qual_extra.json", qual_extra)

    n = len(items_out)
    error_counts: dict[str, int] = {}
    for i in items_out:
        if i["error_category"]:
            error_counts[i["error_category"]] = error_counts.get(i["error_category"], 0) + 1

    result = {
        "n_prompts": n,
        "max_new_tokens": max_new_tokens,
        "timing_seconds": round(time.time() - t_start, 1),
        "selection_method": "reconstructed_unused_from_training_pool" if caveat is None else "fallback_no_description_rules",
        "selection_caveat": caveat,
        "compile_ok_pct": round(100 * sum(i["compile_ok"] for i in items_out) / n, 1) if n else 0.0,
        "compile_ok_with_auto_import_pct": round(100 * sum(i["compile_ok_with_auto_import"] for i in items_out) / n, 1) if n else 0.0,
        "error_categories": error_counts,
        "items": items_out,
    }
    log(f"YARA stage done in {result['timing_seconds']}s. compile={result['compile_ok_pct']}% "
        f"(with_auto_import={result['compile_ok_with_auto_import_pct']}%)")
    return result


def _try_yara_compile(yara_module, source: str):
    try:
        yara_module.compile(source=source)
        return True, None
    except Exception as e:
        return False, str(e)


# ---------------------------------------------------------------------------
# Stage 3 (v0.2): yara_holdout -- the dedicated 40-rule holdout, as opposed to
# v0.1's run_yara_stage() above which had to reconstruct an "unused" 30-item
# set after the fact. Reports BOTH the full 40 (the true v0.2 capability
# number) and the from_v01_eval=true 30-item subset (the direct, same-prompts
# comparison against v0.1's committed yara/yara_<N> numbers) from a single run
# -- no need to generate twice.
# ---------------------------------------------------------------------------
def run_yara_holdout_stage(model, tok, max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    holdout_items = load_yara_holdout_items()
    log(f"=== Stage: yara_holdout ({len(holdout_items)} prompts, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    import yara

    gen_subdir = "yara_holdout" if max_new_tokens == METRIC_MAX_NEW_TOKENS else f"yara_holdout_{max_new_tokens}"

    items_out = []
    for idx, item in enumerate(holdout_items):
        user = yara_prompt_for_item(item)
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)

        safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", item["rule_name"])[:60]
        gen_path = GEN_DIR / gen_subdir / f"{idx:02d}_{safe_name}.txt"
        write_text(gen_path, gen_text)

        compile_ok, compile_err = _try_yara_compile(yara, code)
        compile_ok_fixed, compile_err_fixed = (compile_ok, compile_err)
        used_auto_import = False
        if not compile_ok:
            imports = needed_imports(code)
            if imports:
                fixed_source = "".join(f'import "{imp}"\n' for imp in sorted(imports)) + code
                compile_ok_fixed, compile_err_fixed = _try_yara_compile(yara, fixed_source)
                used_auto_import = True

        error_category = None if compile_ok else categorize_yara_error(compile_err or "")

        items_out.append({
            "index": idx, "key": item["key"], "source": item["source"], "file": item["file"],
            "rule_name": item["rule_name"], "description": item.get("description", ""),
            "from_v01_eval": bool(item.get("from_v01_eval")), "malformed_fence": malformed,
            "compile_ok": compile_ok, "compile_ok_with_auto_import": compile_ok_fixed,
            "used_auto_import_retry": used_auto_import, "error_category": error_category,
            "compile_error": (compile_err or "")[:300] if not compile_ok else "",
            "seconds": round(elapsed, 2),
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
        })
        log(f"  [{idx:02d}] {item['rule_name']} from_v01_eval={item.get('from_v01_eval')} "
            f"compile={compile_ok} (with_auto_import={compile_ok_fixed}) ({elapsed:.1f}s)")

    def _summarize(subset: list[dict]) -> dict:
        n = len(subset)
        error_counts: dict[str, int] = {}
        for i in subset:
            if i["error_category"]:
                error_counts[i["error_category"]] = error_counts.get(i["error_category"], 0) + 1
        return {
            "n_prompts": n,
            "compile_ok_pct": round(100 * sum(i["compile_ok"] for i in subset) / n, 1) if n else 0.0,
            "compile_ok_with_auto_import_pct": round(100 * sum(i["compile_ok_with_auto_import"] for i in subset) / n, 1) if n else 0.0,
            "error_categories": error_counts,
        }

    subset30 = [i for i in items_out if i["from_v01_eval"]]
    result = {
        "max_new_tokens": max_new_tokens,
        "timing_seconds": round(time.time() - t_start, 1),
        "selection_method": "dataset/yara_holdout_ids.json (dedicated 40-rule holdout, excluded from v0.2 training)",
        "all_40": _summarize(items_out),
        "v01_comparable_30": _summarize(subset30),
        "items": items_out,
    }
    log(f"YARA holdout stage done in {result['timing_seconds']}s. "
        f"all-40 compile={result['all_40']['compile_ok_pct']}% | "
        f"30-subset (v0.1-comparable) compile={result['v01_comparable_30']['compile_ok_pct']}%")
    return result


# ---------------------------------------------------------------------------
# Stage 4 (v0.2, NEW): yara_repair -- corrupt a holdout rule with one of the
# dataset's 4 real error classes (dataset/_stage/pool_yara_repair.jsonl's own
# CORRUPTION_CLASSES, imported from build_dataset_v02.py, not reimplemented),
# capture yara-python's real compiler error, prompt the model to fix it --
# same prompt shape (YARA_REPAIR_TEMPLATES[0]) as the yara_repair training
# task. Primary metric: % of fixes that compile. Secondary/informational:
# whether the fix preserves the original rule's string identifiers (a cheap,
# not-exhaustive semantic-drift signal -- flagged, not scored, per the brief).
# ---------------------------------------------------------------------------
STRING_ID_RE = re.compile(r"\$[A-Za-z0-9_]+")


def run_yara_repair_stage(model, tok, max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    holdout_items = load_yara_holdout_items()
    chosen = select_yara_repair_items(holdout_items, n=15, seed=42)
    log(f"=== Stage: yara_repair ({len(chosen)} prompts, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    import yara

    class_cycle = itertools.cycle(CORRUPTION_CLASSES)
    corrupt_rng = random.Random(42)

    items_out = []
    for idx, item in enumerate(chosen):
        assigned = next(class_cycle)
        classes_to_try = [assigned] + [c for c in CORRUPTION_CLASSES if c != assigned]
        picked = None
        for cls in classes_to_try:
            r = CORRUPTORS[cls](item["rule_text"], corrupt_rng)
            if r is None:
                continue
            corrupted_text, desc = r
            ok, err = compile_or_error(corrupted_text)
            if ok:
                continue  # corruption didn't actually break it -- try the next class
            picked = (cls, corrupted_text, desc, err)
            break
        if picked is None:
            log(f"  WARNING: no corruption class broke {item['rule_name']}, skipping")
            continue
        cls, corrupted_text, desc, compiler_error = picked

        user = YARA_REPAIR_TEMPLATES[0].format(corrupted=corrupted_text, error=compiler_error)
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)

        safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", item["rule_name"])[:60]
        gen_path = GEN_DIR / "yara_repair" / f"{idx:02d}_{cls}_{safe_name}.txt"
        write_text(gen_path, gen_text)

        compile_ok, compile_err = _try_yara_compile(yara, code)

        orig_ids = set(STRING_ID_RE.findall(item["rule_text"]))
        fixed_ids = set(STRING_ID_RE.findall(code))
        identifiers_preserved = orig_ids.issubset(fixed_ids) if orig_ids else True
        sim_to_original = recall_similarity(code, item["rule_text"])
        preserves_semantics = compile_ok and identifiers_preserved

        items_out.append({
            "index": idx, "key": item["key"], "rule_name": item["rule_name"],
            "corruption_class": cls, "corruption_desc": desc, "compiler_error_shown": compiler_error,
            "malformed_fence": malformed, "compile_ok": compile_ok,
            "compile_error": (compile_err or "")[:300] if not compile_ok else "",
            "identifiers_preserved": identifiers_preserved, "similarity_to_original": round(sim_to_original, 4),
            "compiles_and_preserves_semantics": preserves_semantics,
            "seconds": round(elapsed, 2),
            "corrupted_rule_text": corrupted_text,
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
        })
        log(f"  [{idx:02d}] {item['rule_name']} class={cls} compile={compile_ok} "
            f"ids_preserved={identifiers_preserved} sim={sim_to_original:.3f} ({elapsed:.1f}s)")

    n = len(items_out)
    class_counts: dict[str, int] = {}
    for i in items_out:
        class_counts[i["corruption_class"]] = class_counts.get(i["corruption_class"], 0) + 1
    drifted = [i for i in items_out if i["compile_ok"] and not i["identifiers_preserved"]]

    result = {
        "n_prompts": n,
        "max_new_tokens": max_new_tokens,
        "timing_seconds": round(time.time() - t_start, 1),
        "selection": "15 of the 40-rule yara_holdout, random.Random(42).sample -- deterministic corruption "
                     "(random.Random(42) drives which corruptor variant runs, cycling CORRUPTION_CLASSES for "
                     "class assignment, falling back to the next class if a given corruptor doesn't apply or "
                     "doesn't actually break compilation)",
        "compile_ok_pct": round(100 * sum(i["compile_ok"] for i in items_out) / n, 1) if n else 0.0,
        "compiles_and_preserves_semantics_pct": round(100 * sum(i["compiles_and_preserves_semantics"] for i in items_out) / n, 1) if n else 0.0,
        "corruption_class_counts": class_counts,
        "semantic_drift_count": len(drifted),
        "semantic_drift_rule_names": [i["rule_name"] for i in drifted],
        "items": items_out,
    }
    log(f"YARA repair stage done in {result['timing_seconds']}s. compile={result['compile_ok_pct']}% "
        f"compile_and_preserves={result['compiles_and_preserves_semantics_pct']}%")
    return result


# ---------------------------------------------------------------------------
# Stage 5 (v0.2, NEW): yara_from_iocs -- description + extracted IOCs (same
# shape as the yara_from_iocs training task: extract_indicators()/
# format_indicators(), imported from build_dataset_v02.py) -> ask for a rule.
# Metrics: compile rate + % of the prompt's own IOCs actually present in the
# generated rule (i.e. the model used what it was given, didn't hallucinate
# or drop indicators).
# ---------------------------------------------------------------------------
def _ioc_present_in_code(indicator: dict, code: str) -> bool:
    val = indicator["value"]
    if not val:
        return False
    if indicator["kind"] in ("text", "hash"):
        return val.lower() in code.lower()
    # hex/regex patterns -- whitespace inside the pattern is not semantically
    # meaningful (YARA hex strings tolerate reformatting), so compare with
    # runs of whitespace collapsed rather than requiring a byte-exact substring.
    norm_val = re.sub(r"\s+", "", val)
    norm_code = re.sub(r"\s+", "", code)
    return bool(norm_val) and norm_val in norm_code


def run_yara_from_iocs_stage(model, tok, max_new_tokens: int = METRIC_MAX_NEW_TOKENS) -> dict:
    holdout_items = load_yara_holdout_items()
    repair_chosen = select_yara_repair_items(holdout_items, n=15, seed=42)
    exclude_keys = {it["key"] for it in repair_chosen}
    chosen = select_yara_iocs_items(holdout_items, exclude_keys, n=10, seed=42)
    log(f"=== Stage: yara_from_iocs ({len(chosen)} prompts, max_new_tokens={max_new_tokens}) ===")
    t_start = time.time()
    import yara

    items_out = []
    for idx, item in enumerate(chosen):
        indicators = extract_indicators(item["rule_text"])
        if not indicators:
            log(f"  WARNING: no indicators extracted for {item['rule_name']}, skipping")
            continue
        ioc_list = format_indicators(indicators)
        description = item.get("description") or item["rule_name"]
        user = YARA_FROM_IOCS_TEMPLATES[0].format(description=description, ioc_list=ioc_list)
        gen_text, elapsed = generate(model, tok, user, max_new_tokens, do_sample=False)
        code, malformed = extract_code_block(gen_text)

        safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", item["rule_name"])[:60]
        gen_path = GEN_DIR / "yara_from_iocs" / f"{idx:02d}_{safe_name}.txt"
        write_text(gen_path, gen_text)

        compile_ok, compile_err = _try_yara_compile(yara, code)
        present_flags = [_ioc_present_in_code(ind, code) for ind in indicators]
        n_present = sum(present_flags)
        ioc_presence_pct = round(100 * n_present / len(indicators), 1)

        items_out.append({
            "index": idx, "key": item["key"], "rule_name": item["rule_name"],
            "description": description, "n_indicators": len(indicators),
            "malformed_fence": malformed, "compile_ok": compile_ok,
            "compile_error": (compile_err or "")[:300] if not compile_ok else "",
            "n_indicators_present": n_present, "ioc_presence_pct": ioc_presence_pct,
            "seconds": round(elapsed, 2),
            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/"),
        })
        log(f"  [{idx:02d}] {item['rule_name']} compile={compile_ok} "
            f"iocs_present={n_present}/{len(indicators)} ({elapsed:.1f}s)")

    n = len(items_out)
    presence_pcts = [i["ioc_presence_pct"] for i in items_out]
    result = {
        "n_prompts": n,
        "max_new_tokens": max_new_tokens,
        "timing_seconds": round(time.time() - t_start, 1),
        "selection": "10 of the 40-rule yara_holdout, disjoint from the 15 yara_repair items, "
                     "random.Random(42).sample over the remaining 25",
        "compile_ok_pct": round(100 * sum(i["compile_ok"] for i in items_out) / n, 1) if n else 0.0,
        "mean_ioc_presence_pct": round(statistics.mean(presence_pcts), 1) if presence_pcts else 0.0,
        "median_ioc_presence_pct": round(statistics.median(presence_pcts), 1) if presence_pcts else 0.0,
        "items": items_out,
    }
    log(f"YARA from-IOCs stage done in {result['timing_seconds']}s. compile={result['compile_ok_pct']}% "
        f"mean_ioc_presence={result['mean_ioc_presence_pct']}%")
    return result


# ---------------------------------------------------------------------------
# Stage 4/5: qualitative before/after + chat regression
# ---------------------------------------------------------------------------
def build_qualitative_prompts(holdout_items: list[dict], yara_qual_item_override: dict | None = None) -> list[dict]:
    translation_item = holdout_items[0]
    authoring_item = holdout_items[1]
    explanation_item = holdout_items[2]
    fptuning_item = next(
        (it for it in holdout_items
         if [fp for fp in it["falsepositives"] if fp.strip().lower() not in FALSEPOSITIVE_NOISE_VALUES]),
        holdout_items[3],
    )

    if yara_qual_item_override is not None:
        # v0.2 (Task 4c): source the qualitative YARA-authoring prompt from the
        # real dedicated 40-rule holdout instead of v0.1's after-the-fact pool
        # reconstruction (see pick_yara_qual_holdout_item()).
        yara_qual_item = yara_qual_item_override
    else:
        try:
            yara_qual_item = read_json(STAGE_DIR / "_eval_yara_qual_extra.json")
        except FileNotFoundError:
            chosen, yara_qual_item, _ = select_yara_eval_items(n=30, seed=42)

    tags_s = ", ".join(authoring_item["tags"]) if authoring_item["tags"] else "none recorded"
    fps = fptuning_item["falsepositives"]

    prompts = [
        {
            "name": "translation_sigma_to_kql",
            "source_file": translation_item["filename"],
            "prompt": TRANSLATION_TEMPLATES[0].format(
                target="Microsoft 365 Defender Advanced Hunting KQL",
                yaml=translation_item["raw_yaml"], description=translation_item["description"]),
        },
        {
            "name": "sigma_authoring",
            "source_file": authoring_item["filename"],
            "prompt": AUTHORING_TEMPLATES[0].format(
                description=authoring_item["description"],
                logsource_str=logsource_str(authoring_item["logsource"]), tags_str=tags_s),
        },
        {
            "name": "explanation",
            "source_file": explanation_item["filename"],
            "prompt": EXPLANATION_TEMPLATES[0].format(yaml=explanation_item["raw_yaml"], title=explanation_item["title"]),
        },
        {
            "name": "fp_tuning",
            "source_file": fptuning_item["filename"],
            "prompt": FPTUNING_TEMPLATES[0].format(
                yaml=fptuning_item["raw_yaml"], title=fptuning_item["title"], fp_text=fp_text_from_list(fps)),
        },
        {
            "name": "yara_authoring",
            "source_file": f"{yara_qual_item.get('source')}:{yara_qual_item.get('file')}:{yara_qual_item.get('rule_name')}",
            "prompt": yara_prompt_for_item(yara_qual_item),
        },
    ]
    return prompts


CHAT_REGRESSION_PROMPTS = [
    "Explain how DNS works to a 10-year-old.",
    "Write a Python function that checks whether a given string is a palindrome, ignoring case and spaces.",
    "Summarize the following paragraph in one sentence: 'The Great Barrier Reef, located off the coast of "
    "Queensland, Australia, is the world's largest coral reef system, composed of over 2,900 individual reefs "
    "and 900 islands stretching for over 2,300 kilometres. It is so large that it can be seen from outer space, "
    "and is the world's biggest single structure made by living organisms.'",
]


def run_qualitative_tuned_half(model, tok, holdout_items: list[dict], yara_qual_item_override: dict | None = None):
    log("=== Stage: qualitative (tuned half, 5 prompts + 3 chat-regression prompts) ===")
    t_start = time.time()
    prompts = build_qualitative_prompts(holdout_items, yara_qual_item_override=yara_qual_item_override)
    tuned_outputs = []
    for p in prompts:
        gen_text, elapsed = generate(
            model, tok, p["prompt"], QUALITATIVE_MAX_NEW_TOKENS, seed=QUALITATIVE_SEED, **QUALITATIVE_SAMPLING,
        )
        gen_path = GEN_DIR / "qualitative" / f"{p['name']}_tuned.txt"
        write_text(gen_path, gen_text)
        tuned_outputs.append({"seconds": round(elapsed, 2), "output": gen_text,
                               "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/")})
        log(f"  [tuned] {p['name']} ({elapsed:.1f}s)")

    chat_items = []
    for i, prompt in enumerate(CHAT_REGRESSION_PROMPTS):
        gen_text, elapsed = generate(model, tok, prompt, CHAT_REGRESSION_MAX_NEW_TOKENS, do_sample=False)
        gen_path = GEN_DIR / "chat_regression" / f"{i:02d}.txt"
        write_text(gen_path, gen_text)
        chat_items.append({"prompt": prompt, "output": gen_text, "seconds": round(elapsed, 2),
                            "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/")})
        log(f"  [chat_regression] prompt {i} ({elapsed:.1f}s)")

    return prompts, tuned_outputs, chat_items, round(time.time() - t_start, 1)


def run_qualitative_base_half(model, tok, prompts: list[dict]):
    log("=== Stage: qualitative (base half, 5 prompts) ===")
    t_start = time.time()
    base_outputs = []
    for p in prompts:
        gen_text, elapsed = generate(
            model, tok, p["prompt"], QUALITATIVE_MAX_NEW_TOKENS, seed=QUALITATIVE_SEED, **QUALITATIVE_SAMPLING,
        )
        gen_path = GEN_DIR / "qualitative" / f"{p['name']}_base.txt"
        write_text(gen_path, gen_text)
        base_outputs.append({"seconds": round(elapsed, 2), "output": gen_text,
                              "generation_file": str(gen_path.relative_to(REPO)).replace("\\", "/")})
        log(f"  [base] {p['name']} ({elapsed:.1f}s)")
    return base_outputs, round(time.time() - t_start, 1)


# ---------------------------------------------------------------------------
# results.json merge + RESULTS.md rendering
# ---------------------------------------------------------------------------
def load_existing_results(path: Path = RESULTS_JSON) -> dict:
    if path.exists():
        return read_json(path)
    return {}


def fence_lang(backend: str) -> str:
    return backend


def render_results_md(results: dict, adapter_dir: Path) -> str:
    lines = []
    lines.append("# Task 4 evaluation results -- Glimmer-Sentry-30B\n")
    lines.append(f"Adapter checkpoint: `{adapter_dir}` | Base model: `{BASE_MODEL_NAME}`\n")
    lines.append(
        "Decoding: metric stages (translation, sigma authoring, YARA) use **greedy** decoding "
        "(`do_sample=False`, `max_new_tokens=512`). Qualitative before/after uses "
        "`temperature=1.0, top_p=0.95, top_k=64`, seed 42 (fixed per-prompt, identical for base and tuned "
        "so only the model differs). Chat-regression uses greedy (not brief-mandated; chosen for "
        "reproducibility since it's a sanity check, not a creativity demo).\n"
    )
    lines.append(
        "Prompt templates: reused verbatim from `scripts/build_dataset.py`'s own template banks "
        "(imported, not reimplemented) -- template index `[0]` (the canonical, plain phrasing) from each "
        "bank was used for every generation, deterministically, so results are exactly reproducible.\n"
    )
    lines.append(
        "Normalisation for the 'normalised match' metric: collapse internal whitespace runs to one space, "
        "strip trailing whitespace and trailing semicolons. Nothing semantic (no reordering, no case-folding).\n"
    )

    # --- headline summary table, before any detail sections ---
    t = results.get("translation")
    sa_h = results.get("sigma_authoring")
    y_h = results.get("yara")
    y1536_h = next((results[k] for k in results if re.fullmatch(r"yara_\d+", k)), None)
    cr_h = results.get("chat_regression")
    if any((t, sa_h, y_h, cr_h)):
        lines.append("## Headline summary\n")
        lines.append("| Metric | Result |")
        lines.append("|---|---|")
        if t:
            lines.append(f"| KQL translation (50 holdout) -- exact / normalised | {t['kql']['exact_match_pct']}% / {t['kql']['normalized_match_pct']}% |")
            lines.append(f"| SPL translation (50 holdout) -- exact / normalised | {t['spl']['exact_match_pct']}% / {t['spl']['normalized_match_pct']}% |")
        if sa_h:
            lines.append(f"| Sigma authoring (50 holdout) -- parses / `sigma check` / `sigma convert` | {sa_h['parses_as_yaml_pct']}% / {sa_h['passes_sigma_check_pct']}% / {sa_h['converts_ok_pct']}% |")
            rs_h = sa_h.get("recall_similarity", {})
            lines.append(f"| Sigma authoring memorization (near-copy of real holdout rule) | {rs_h.get('near_copy_count', '?')}/{sa_h.get('n_rules', '?')} ({rs_h.get('near_copy_pct', '?')}%) -- **confirmed novel construction, not recall** |")
        if y_h:
            lines.append(f"| YARA compile rate @{METRIC_MAX_NEW_TOKENS} tokens (30 prompts) | {y_h['compile_ok_pct']}% |")
        if y1536_h:
            lines.append(f"| YARA compile rate @{y1536_h.get('max_new_tokens', '1536')} tokens (same 30 prompts) | {y1536_h['compile_ok_pct']}% |")
        if cr_h:
            lines.append(f"| Chat-regression spot check (3 general prompts, tuned model) | {len(cr_h['items'])}/{len(cr_h['items'])} coherent, no collapse into task-specific output -- general ability intact (see §5) |")
        lines.append("")

    # --- caveat, prominent, near the top ---
    if "sigma_authoring" in results:
        sa = results["sigma_authoring"]
        rs = sa.get("recall_similarity", {})
        lines.append("## Memorization check: sigma-authoring is confirmed novel construction, not recall\n")
        lines.append(
            f"**Headline: {rs.get('near_copy_count', '?')}/{sa.get('n_rules', '?')} "
            f"({rs.get('near_copy_pct', '?')}%) of the sigma-authoring generations below are near-copies "
            f"(similarity > {rs.get('near_copy_threshold', 0.9)}) of the real holdout rule they were meant "
            f"to be inventing.** Mean similarity across all {sa.get('n_rules', '?')}: {rs.get('mean', '?')}, "
            f"median {rs.get('median', '?')} -- i.e. the model is not echoing the ground truth back, it is "
            "genuinely generating.\n\n"
            "This check exists because the **base** model (before our fine-tune) was separately observed "
            "reproducing a held-out Sigma rule near-verbatim, including its real UUID, during a "
            "post-training generation review -- recall from the base model's own pretraining on public "
            "SigmaHQ rules, **not** a leak from our training set (independently verified leak-free, zero "
            "holdout IDs present in `dataset/dataset.jsonl.gz`). Given that precedent, every one of the 50 "
            "sigma-authoring generations below was checked against the real holdout rule text "
            "(normalised, case-folded edit similarity via `difflib.SequenceMatcher`) rather than assuming "
            "the parses/`sigma check`/`sigma convert` percentages reflect pure generation skill -- they do.\n"
        )
        if rs.get("near_copy_rule_ids"):
            lines.append(f"Near-copy rule IDs: {', '.join(rs['near_copy_rule_ids'])}\n")

    # --- §1 translation ---
    if "translation" in results:
        t = results["translation"]
        lines.append("## 1. Translation accuracy (holdout rules x 2 backends)\n")
        lines.append(f"Timing: {t.get('timing_seconds')}s for {t.get('n_rules')} rules x 2 backends.\n")
        lines.append("| Backend | n | Exact match | Normalised match | Malformed fence |")
        lines.append("|---|---|---|---|---|")
        for backend in ("kql", "spl"):
            b = t[backend]
            lines.append(f"| {backend.upper()} | {b['n']} | {b['exact_match_pct']}% | {b['normalized_match_pct']}% | {b['malformed_fence_count']} |")
        lines.append("")
        for backend in ("kql", "spl"):
            b = t[backend]
            lines.append(f"### {backend.upper()} mismatch examples (up to 3)\n")
            if not b["mismatch_examples"]:
                lines.append("_No mismatches -- 100% normalised match._\n")
            for ex in b["mismatch_examples"]:
                lines.append(f"**{ex['rule_id']}** (`{ex['filename']}`)\n")
                lines.append(f"Ground truth:\n```{fence_lang(backend)}\n{ex['ground_truth']}\n```\n")
                lines.append(f"Generated:\n```{fence_lang(backend)}\n{ex['generated']}\n```\n")

    # --- §1 footnote: translation truncation re-eval ---
    if "translation_truncation_footnote" in results:
        f = results["translation_truncation_footnote"]
        lines.append("### 1b. Truncation footnote: malformed-fence items re-run at a higher token cap\n")
        lines.append(
            f"{f['note']}\n\n"
            f"Re-run at `max_new_tokens={f['max_new_tokens']}`: **{f['resolved_fence_count']}/{f['n_items']}** "
            f"malformed fences resolved, **{f['became_exact_count']}/{f['n_items']}** became exact matches, "
            f"**{f['became_normalized_count']}/{f['n_items']}** became normalised matches. "
            f"Timing: {f['timing_seconds']}s for {f['n_items']} items.\n"
        )
        lines.append("| Rule ID | Backend | Malformed @512 | Malformed now | Exact | Normalised | Seconds |")
        lines.append("|---|---|---|---|---|---|---|")
        for it in f["items"]:
            lines.append(
                f"| {it['rule_id']} | {it['backend'].upper()} | {it['malformed_fence_at_512']} | "
                f"{it['malformed_fence_now']} | {it['exact']} | {it['normalized']} | {it['seconds']} |"
            )
        lines.append("")

    # --- §2 sigma authoring ---
    if "sigma_authoring" in results:
        sa = results["sigma_authoring"]
        lines.append("## 2. Sigma authoring validity (50 holdout rules)\n")
        lines.append(f"Timing: {sa.get('timing_seconds')}s for {sa.get('n_rules')} rules.\n")
        lines.append("| Metric | % |")
        lines.append("|---|---|")
        lines.append(f"| Parses as YAML | {sa['parses_as_yaml_pct']}% |")
        lines.append(f"| Passes `sigma check` | {sa['passes_sigma_check_pct']}% |")
        lines.append(f"| Converts with `sigma convert -t kusto -p microsoft_365_defender` | {sa['converts_ok_pct']}% |")
        lines.append(f"| Near-copy of real holdout rule (similarity > {sa['recall_similarity']['near_copy_threshold']}) | {sa['recall_similarity']['near_copy_pct']}% |")
        lines.append("")

    # --- §3 yara ---
    if "yara" in results:
        y = results["yara"]
        lines.append("## 3. YARA validity (30 prompts)\n")
        lines.append(f"Selection method: `{y['selection_method']}`.")
        if y.get("selection_caveat"):
            lines.append(f"\n**Caveat**: {y['selection_caveat']}")
        lines.append(f"\n\nTiming: {y.get('timing_seconds')}s for {y.get('n_prompts')} prompts.\n")
        lines.append(f"Compile success (as generated): **{y['compile_ok_pct']}%**")
        lines.append(f"Compile success (with a courtesy auto-added `import` retry on failures that reference pe./math./hash./elf./dotnet. without importing them): **{y['compile_ok_with_auto_import_pct']}%**\n")
        if y["error_categories"]:
            lines.append("| Error category | Count |")
            lines.append("|---|---|")
            for cat, cnt in sorted(y["error_categories"].items(), key=lambda kv: -kv[1]):
                lines.append(f"| {cat} | {cnt} |")
            lines.append("")

    # --- §3b additional YARA runs at higher token caps (e.g. yara_1536) ---
    # (No eval()/exec() involved anywhere in this file -- "re-eval" below is short for
    # "re-evaluation", a rerun of the YARA metric stage at a different --max_new_tokens.)
    for rkey in sorted(k for k in results if re.fullmatch(r"yara_\d+", k)):
        y2 = results[rkey]
        mnt2 = y2.get("max_new_tokens", rkey.split("_", 1)[1])
        lines.append(f"### 3b. YARA re-run at max_new_tokens={mnt2} (same 30 prompts, truncation check)\n")
        base_pct = results.get("yara", {}).get("compile_ok_pct")
        lines.append(
            f"Same 30 prompts, same selection, decoding otherwise unchanged (greedy) -- only the token cap "
            f"differs from the committed @{METRIC_MAX_NEW_TOKENS} run. "
            + (f"@{METRIC_MAX_NEW_TOKENS} compile rate was **{base_pct}%**; " if base_pct is not None else "")
            + f"@{mnt2} compile rate is **{y2['compile_ok_pct']}%** "
            f"(with_auto_import: **{y2['compile_ok_with_auto_import_pct']}%**). "
            f"Timing: {y2.get('timing_seconds')}s for {y2.get('n_prompts')} prompts. "
            f"The committed headline @{METRIC_MAX_NEW_TOKENS} number in `results['yara']` above is unchanged "
            "by this re-eval -- both are reported per the brief's methodology-honesty requirement.\n"
        )
        if y2["error_categories"]:
            lines.append("| Error category | Count |")
            lines.append("|---|---|")
            for cat, cnt in sorted(y2["error_categories"].items(), key=lambda kv: -kv[1]):
                lines.append(f"| {cat} | {cnt} |")
            lines.append("")

    # --- §4 qualitative ---
    if "qualitative" in results:
        q = results["qualitative"]
        lines.append("## 4. Qualitative before/after (base vs tuned, 5 prompts)\n")
        for p in q["prompts"]:
            lines.append(f"### {p['name']} (source: `{p['source_file']}`)\n")
            lines.append(f"**Prompt:**\n```\n{p['prompt']}\n```\n")
            lines.append(f"**BASE model** ({p['base']['seconds']}s):\n```\n{p['base']['output']}\n```\n")
            lines.append(f"**TUNED model** ({p['tuned']['seconds']}s):\n```\n{p['tuned']['output']}\n```\n")

    # --- §5 chat regression ---
    if "chat_regression" in results:
        cr = results["chat_regression"]
        lines.append("## 5. Chat-regression spot check (3 prompts, tuned model, greedy)\n")
        lines.append("Sanity gate, not a benchmark -- confirms coherent, non-collapsed general ability.\n")
        for i, item in enumerate(cr["items"]):
            lines.append(f"### Prompt {i}\n```\n{item['prompt']}\n```\n")
            lines.append(f"**Output** ({item['seconds']}s):\n```\n{item['output']}\n```\n")

    if "timings" in results:
        lines.append("## Timing summary\n")
        lines.append("| Stage | Seconds |")
        lines.append("|---|---|")
        for k, v in results["timings"].items():
            lines.append(f"| {k} | {v} |")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# v0.2 (Task 4c): renders the v0.1-vs-v0.2 comparison + new-capability report.
# Takes the v0.1 results dict (read-only, untouched -- eval/results.json) and
# the v0.2 results dict (eval/results_v02.json) and produces a Markdown
# fragment that gets appended AFTER the unmodified render_results_md() output
# in eval/RESULTS.md, per the brief ("Keep all v0.1 content").
# ---------------------------------------------------------------------------
def _pct_row(lines: list, label: str, v1, v2) -> None:
    try:
        delta = round(float(v2) - float(v1), 1)
        delta_s = f"{'+' if delta >= 0 else ''}{delta}pp"
    except (TypeError, ValueError):
        delta_s = "?"
    v1_s = f"{v1}%" if isinstance(v1, (int, float)) else "--"
    v2_s = f"{v2}%" if isinstance(v2, (int, float)) else "--"
    lines.append(f"| {label} | {v1_s} | {v2_s} | {delta_s} |")


def render_v02_report(v01: dict, v02: dict, adapter_dir_v02) -> str:
    lines = []
    lines.append("\n---\n")
    lines.append("# v0.2 evaluation and v0.1 comparison (Task 4c)\n")
    lines.append(f"v0.2 adapter checkpoint: `{adapter_dir_v02}` | Base model: `{BASE_MODEL_NAME}`\n")
    lines.append(
        "Everything in this section comes from `eval/results_v02.json`, a separate file from the v0.1 "
        "`eval/results.json` rendered above -- the v0.1 numbers are load-bearing and are never overwritten. "
        "Decoding conventions match v0.1 (greedy for all metric stages, `temperature=1.0/top_p=0.95/top_k=64` "
        "seed 42 for qualitative), and every table cell below states its decode budget (@512 or @1536) "
        "explicitly, per the brief.\n"
    )

    t1, t2 = v01.get("translation", {}), v02.get("translation", {})
    sa1 = v01.get("sigma_authoring", {})
    sa2_512 = v02.get("sigma_authoring", {})
    sa2_1536 = v02.get("sigma_authoring_1536", {})
    y1_512 = v01.get("yara", {})
    y1_1536 = next((v01[k] for k in v01 if re.fullmatch(r"yara_\d+", k)), {})
    yh_512 = v02.get("yara_holdout", {})
    yh_1536 = v02.get("yara_holdout_1536", {})
    repair = v02.get("yara_repair", {})
    iocs = v02.get("yara_from_iocs", {})
    qual2 = v02.get("qualitative", {})
    cr2 = v02.get("chat_regression", {})

    # --- v0.2 headline ---
    lines.append("## v0.2 headline summary\n")
    lines.append("| Metric | v0.2 result |")
    lines.append("|---|---|")
    if t2:
        lines.append(f"| KQL translation @512 (50 holdout) -- exact / normalised | {t2.get('kql', {}).get('exact_match_pct', '?')}% / {t2.get('kql', {}).get('normalized_match_pct', '?')}% |")
        lines.append(f"| SPL translation @512 (50 holdout) -- exact / normalised | {t2.get('spl', {}).get('exact_match_pct', '?')}% / {t2.get('spl', {}).get('normalized_match_pct', '?')}% |")
    if sa2_512:
        lines.append(f"| Sigma authoring @512 -- parses / check / convert | {sa2_512.get('parses_as_yaml_pct')}% / {sa2_512.get('passes_sigma_check_pct')}% / {sa2_512.get('converts_ok_pct')}% |")
    if sa2_1536:
        lines.append(f"| Sigma authoring @1536 -- parses / check / convert | {sa2_1536.get('parses_as_yaml_pct')}% / {sa2_1536.get('passes_sigma_check_pct')}% / {sa2_1536.get('converts_ok_pct')}% |")
    if yh_512:
        lines.append(f"| YARA compile @512 -- full 40-rule holdout | {yh_512.get('all_40', {}).get('compile_ok_pct')}% |")
        lines.append(f"| YARA compile @512 -- 30-item v0.1-comparable subset | {yh_512.get('v01_comparable_30', {}).get('compile_ok_pct')}% |")
    if yh_1536:
        lines.append(f"| YARA compile @1536 -- full 40-rule holdout | {yh_1536.get('all_40', {}).get('compile_ok_pct')}% |")
        lines.append(f"| YARA compile @1536 -- 30-item v0.1-comparable subset | {yh_1536.get('v01_comparable_30', {}).get('compile_ok_pct')}% |")
    if repair:
        lines.append(f"| **NEW** YARA repair @{repair.get('max_new_tokens')} -- compiles / compiles+preserves-semantics | {repair.get('compile_ok_pct')}% / {repair.get('compiles_and_preserves_semantics_pct')}% |")
    if iocs:
        lines.append(f"| **NEW** IOC-grounded authoring @{iocs.get('max_new_tokens')} -- compiles / mean IOC-presence | {iocs.get('compile_ok_pct')}% / {iocs.get('mean_ioc_presence_pct')}% |")
    if qual2:
        lines.append(f"| Qualitative base-vs-tuned side-by-sides | {len(qual2.get('prompts', []))}/5 generated (see below) |")
    if cr2:
        lines.append(f"| Chat-regression spot check (tuned v0.2) | {len(cr2.get('items', []))}/{len(cr2.get('items', []))} coherent |")
    lines.append("")

    # --- comparison table ---
    lines.append("## v0.1 vs v0.2 comparison (every shared metric, decode budget labeled per cell)\n")
    lines.append("| Metric | v0.1 | v0.2 | Delta |")
    lines.append("|---|---|---|---|")
    if t1 and t2:
        _pct_row(lines, "KQL exact @512", t1.get("kql", {}).get("exact_match_pct"), t2.get("kql", {}).get("exact_match_pct"))
        _pct_row(lines, "KQL normalised @512", t1.get("kql", {}).get("normalized_match_pct"), t2.get("kql", {}).get("normalized_match_pct"))
        _pct_row(lines, "SPL exact @512", t1.get("spl", {}).get("exact_match_pct"), t2.get("spl", {}).get("exact_match_pct"))
        _pct_row(lines, "SPL normalised @512", t1.get("spl", {}).get("normalized_match_pct"), t2.get("spl", {}).get("normalized_match_pct"))
    if sa1 and sa2_512:
        _pct_row(lines, "Sigma authoring parses as YAML @512", sa1.get("parses_as_yaml_pct"), sa2_512.get("parses_as_yaml_pct"))
        _pct_row(lines, "Sigma authoring `sigma check` @512", sa1.get("passes_sigma_check_pct"), sa2_512.get("passes_sigma_check_pct"))
        _pct_row(lines, "Sigma authoring `sigma convert` @512", sa1.get("converts_ok_pct"), sa2_512.get("converts_ok_pct"))
        rs1, rs2 = sa1.get("recall_similarity", {}), sa2_512.get("recall_similarity", {})
        if rs1 and rs2:
            _pct_row(lines, "Sigma authoring near-copy rate @512 (memorization)", rs1.get("near_copy_pct"), rs2.get("near_copy_pct"))
    if sa2_1536:
        lines.append(f"| Sigma authoring `sigma convert` @1536 | -- (v0.1 never ran authoring @1536) | {sa2_1536.get('converts_ok_pct')}% | n/a |")
    if y1_512 and yh_512:
        _pct_row(lines, "YARA compile @512 (same 30 rule identities)", y1_512.get("compile_ok_pct"), yh_512.get("v01_comparable_30", {}).get("compile_ok_pct"))
    if y1_1536 and yh_1536:
        _pct_row(lines, "YARA compile @1536 (same 30 rule identities)", y1_1536.get("compile_ok_pct"), yh_1536.get("v01_comparable_30", {}).get("compile_ok_pct"))
    if yh_512:
        lines.append(f"| YARA compile @512, full 40-rule dedicated holdout | -- (v0.1 had no dedicated holdout) | {yh_512.get('all_40', {}).get('compile_ok_pct')}% | n/a |")
    if yh_1536:
        lines.append(f"| YARA compile @1536, full 40-rule dedicated holdout | -- (v0.1 had no dedicated holdout) | {yh_1536.get('all_40', {}).get('compile_ok_pct')}% | n/a |")
    lines.append(
        "\nThe 30-item comparison rows use the exact same 30 rule identities v0.1's `yara`/`yara_<N>` stages "
        "evaluated (`from_v01_eval: true` in `dataset/yara_holdout_ids.json`, recovered from `eval/results.json` "
        "by `build_dataset_v02.py`'s `yara_holdout` stage) -- same prompts, same descriptions, only the model "
        "differs. The full-40 rows are v0.2-only: v0.1 never had a dedicated, excluded-from-training YARA "
        "holdout (its `yara` stage had to reconstruct an 'unused' set after the fact -- see "
        "`select_yara_eval_items()`), so there is no v0.1 equivalent for the full 40.\n"
    )

    # --- YARA holdout detail ---
    if yh_512 or yh_1536:
        lines.append("## YARA holdout detail (dedicated 40-rule holdout)\n")
        for label, yh in (("@512", yh_512), ("@1536", yh_1536)):
            if not yh:
                continue
            lines.append(f"### {label}\n")
            lines.append(f"Timing: {yh.get('timing_seconds')}s for {yh.get('all_40', {}).get('n_prompts')} prompts.\n")
            a40, s30 = yh.get("all_40", {}), yh.get("v01_comparable_30", {})
            lines.append(f"Full 40: compile **{a40.get('compile_ok_pct')}%** (with auto-import retry: {a40.get('compile_ok_with_auto_import_pct')}%)\n")
            lines.append(f"30-item v0.1-comparable subset: compile **{s30.get('compile_ok_pct')}%** (with auto-import retry: {s30.get('compile_ok_with_auto_import_pct')}%)\n")
            if a40.get("error_categories"):
                lines.append("| Error category (full 40) | Count |")
                lines.append("|---|---|")
                for cat, cnt in sorted(a40["error_categories"].items(), key=lambda kv: -kv[1]):
                    lines.append(f"| {cat} | {cnt} |")
                lines.append("")

    # --- YARA repair (NEW) ---
    if repair:
        lines.append("## NEW capability: YARA repair\n")
        lines.append(
            f"15 of the 40 holdout rules (`random.Random(42).sample`), each deterministically corrupted with "
            f"one of the dataset's real error classes (`{', '.join(CORRUPTION_CLASSES)}`, imported from "
            "`build_dataset_v02.py` -- the exact corruption functions the `yara_repair` training task used), "
            "the real yara-python compiler error captured and shown to the model, prompted to fix it "
            f"(`YARA_REPAIR_TEMPLATES[0]`, same prompt shape as training) at max_new_tokens="
            f"{repair.get('max_new_tokens')}.\n\n"
            f"**Compile rate: {repair.get('compile_ok_pct')}%** ({sum(1 for i in repair.get('items', []) if i['compile_ok'])}/{repair.get('n_prompts')}). "
            f"Compiles AND preserves the original rule's string identifiers (semantic-drift proxy, informational "
            f"not the headline metric): **{repair.get('compiles_and_preserves_semantics_pct')}%**.\n"
        )
        if repair.get("corruption_class_counts"):
            lines.append("| Corruption class | Count |")
            lines.append("|---|---|")
            for cls, cnt in sorted(repair["corruption_class_counts"].items(), key=lambda kv: -kv[1]):
                lines.append(f"| {cls} | {cnt} |")
            lines.append("")
        if repair.get("semantic_drift_rule_names"):
            lines.append(
                f"**Semantic drift flagged** (compiled but did not preserve all original string identifiers) "
                f"in {repair['semantic_drift_count']}/{repair.get('n_prompts')}: "
                f"{', '.join(repair['semantic_drift_rule_names'])}\n"
            )
        lines.append("### Repair examples\n")
        for it in repair.get("items", []):
            lines.append(
                f"**{it['rule_name']}** (`{it['corruption_class']}`) -- compile={it['compile_ok']}, "
                f"identifiers_preserved={it['identifiers_preserved']}, similarity_to_original={it['similarity_to_original']}, "
                f"{it['seconds']}s\n"
            )
            lines.append(f"Corruption: {it['corruption_desc']}\n")
            lines.append(f"Compiler error shown to model:\n```\n{it['compiler_error_shown']}\n```\n")
            if not it["compile_ok"]:
                lines.append(f"Model's fix still fails to compile:\n```\n{it['compile_error']}\n```\n")

    # --- YARA from-IOCs (NEW) ---
    if iocs:
        lines.append("## NEW capability: IOC-grounded YARA authoring\n")
        lines.append(
            "10 of the 40 holdout rules (disjoint from the 15 repair items, `random.Random(42).sample` over "
            "the remaining 25), indicators extracted from each rule's own `strings:` section "
            "(`extract_indicators()`/`format_indicators()`, imported from `build_dataset_v02.py`), prompted "
            f"as description + IOC list (`YARA_FROM_IOCS_TEMPLATES[0]`, same shape as training) at "
            f"max_new_tokens={iocs.get('max_new_tokens')}.\n\n"
            f"**Compile rate: {iocs.get('compile_ok_pct')}%**. **Mean IOC-presence: {iocs.get('mean_ioc_presence_pct')}%** "
            f"(median {iocs.get('median_ioc_presence_pct')}%) -- the share of the prompt's own indicators the "
            "model actually used in its generated rule (case-insensitive substring for text/hash indicators, "
            "whitespace-collapsed substring for hex/regex).\n"
        )
        lines.append("### IOC-grounded examples\n")
        for it in iocs.get("items", []):
            lines.append(
                f"**{it['rule_name']}** -- compile={it['compile_ok']}, "
                f"IOCs present={it['n_indicators_present']}/{it['n_indicators']} ({it['ioc_presence_pct']}%), {it['seconds']}s\n"
            )

    # --- qualitative (v0.2) ---
    if qual2:
        lines.append("## Qualitative before/after (base vs v0.2 tuned, 5 prompts)\n")
        for p in qual2.get("prompts", []):
            lines.append(f"### {p['name']} (source: `{p['source_file']}`)\n")
            lines.append(f"**Prompt:**\n```\n{p['prompt']}\n```\n")
            lines.append(f"**BASE model** ({p['base']['seconds']}s):\n```\n{p['base']['output']}\n```\n")
            lines.append(f"**v0.2 TUNED model** ({p['tuned']['seconds']}s):\n```\n{p['tuned']['output']}\n```\n")

    # --- chat regression (v0.2) ---
    if cr2:
        lines.append("## Chat-regression spot check (3 prompts, v0.2 tuned model, greedy)\n")
        lines.append("Sanity gate, not a benchmark -- same 3 prompts as v0.1.\n")
        for i, item in enumerate(cr2.get("items", [])):
            lines.append(f"### Prompt {i}\n```\n{item['prompt']}\n```\n")
            lines.append(f"**Output** ({item['seconds']}s):\n```\n{item['output']}\n```\n")

    if "timings" in v02:
        lines.append("## v0.2 timing summary\n")
        lines.append("| Stage | Seconds |")
        lines.append("|---|---|")
        for k, v in v02["timings"].items():
            lines.append(f"| {k} | {v} |")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
    global GEN_DIR  # reassigned below from --gen_dir; must precede any use of the name in this function
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument(
        "--stage",
        choices=[
            "translation", "sigma_authoring", "yara", "qualitative",
            "qualitative_tuned", "qualitative_base", "translation_footnote", "render", "all",
            "yara_holdout", "yara_repair", "yara_from_iocs",
        ],
        required=True,
        help=(
            "translation/sigma_authoring/yara/qualitative -- original Task 4 stages (qualitative loads "
            "BOTH base and tuned models sequentially in one process; this is what hung on the base half "
            "on 2026-08-11, see task-4-report.md -- prefer qualitative_tuned + qualitative_base as two "
            "separate process invocations instead). "
            "qualitative_tuned -- loads ONLY the tuned model, runs the 5 qualitative prompts + the 3 "
            "chat-regression prompts, stashes prompts+tuned-outputs in results.json under "
            "'_qualitative_pending' for qualitative_base to pick up. "
            "qualitative_base -- fresh-process fix for the hang: loads ONLY the base model (no prior "
            "model load in this process), reads '_qualitative_pending' from results.json (must exist -- "
            "run qualitative_tuned first, in a separate invocation), generates the base-model half, "
            "merges into results['qualitative']. "
            "translation_footnote -- Task 4b footnote: re-runs ONLY the translation items that had "
            "malformed_fence=true in the committed results['translation'] (@512), at --max_new_tokens, "
            "writing to results['translation_truncation_footnote'] -- never touches results['translation']. "
            "render -- no model load, no generation: just re-renders eval/RESULTS.md from the current "
            "results file(s) (useful after hand-editing render_results_md()/render_v02_report(), or after "
            "several separate stage invocations, to regenerate the merged Markdown without spending GPU time). "
            "yara_holdout (Task 4c/v0.2, NEW) -- the DEDICATED 40-rule holdout (dataset/yara_holdout_ids.json), "
            "reports both the full 40 and the from_v01_eval=true 30-item v0.1-comparable subset from one run. "
            "yara_repair (Task 4c/v0.2, NEW) -- 15 holdout rules corrupted with a real error class, model "
            "asked to fix; scored on compile rate. "
            "yara_from_iocs (Task 4c/v0.2, NEW) -- 10 holdout rules, description+IOC-list prompt; scored on "
            "compile rate plus percentage of prompt IOCs present in output."
        ),
    )
    ap.add_argument("--adapter_dir", type=Path, default=ADAPTER_CKPT_DEFAULT)
    ap.add_argument(
        "--max_new_tokens", type=int, default=METRIC_MAX_NEW_TOKENS,
        help=(
            f"Generation cap for translation/sigma_authoring/yara/translation_footnote/yara_holdout/"
            f"yara_repair/yara_from_iocs (default {METRIC_MAX_NEW_TOKENS}, matches the brief-mandated metric "
            "decoding config -- keeps current behavior unchanged when omitted). Passing a non-default value "
            "never overwrites the committed @512 results: yara/yara_holdout/sigma_authoring write to "
            "results['<stage>_<N>'] instead of results['<stage>'], and their raw generation .txt files go to "
            "<gen_dir>/<stage>_<N>/ instead of <gen_dir>/<stage>/. translation_footnote always writes to its "
            "own results['translation_truncation_footnote'] key regardless of N. yara_repair/yara_from_iocs "
            "always write to their own fixed keys (results['yara_repair']/['yara_from_iocs']) regardless of N "
            "-- the brief mandates a single @1536 run for each, not a dual small/large comparison."
        ),
    )
    ap.add_argument(
        "--results_path", type=Path, default=RESULTS_JSON,
        help=(
            "Where this invocation's stage results are read from and written to (default eval/results.json, "
            "the v0.1 file, unchanged from prior behavior). Task 4c (v0.2): pass --results_path "
            "eval/results_v02.json so v0.2 runs NEVER touch the v0.1 file. Regardless of this flag, --stage "
            "render (and the auto-render every stage does at the end) always reads BOTH eval/results.json and "
            "eval/results_v02.json (if it exists) and writes the combined eval/RESULTS.md -- v0.1 content "
            "first, unchanged, then a v0.1-vs-v0.2 comparison section if v0.2 data is present."
        ),
    )
    ap.add_argument(
        "--gen_dir", type=Path, default=GEN_DIR,
        help="Where raw generation .txt files are written (default eval/generations/). Task 4c (v0.2): pass "
             "--gen_dir eval/generations_v02 per the brief.",
    )
    ap.add_argument(
        "--yara_qual_from_holdout", action="store_true",
        help="Qualitative stage only: source the yara_authoring side-by-side prompt from the dedicated "
             "40-rule holdout (disjoint from the yara_repair/yara_from_iocs selections) instead of v0.1's "
             "pool-reconstruction fallback. Use for v0.2 qualitative runs.",
    )
    args = ap.parse_args()
    mnt = args.max_new_tokens

    GEN_DIR = args.gen_dir
    results_path = args.results_path

    EVAL_DIR.mkdir(parents=True, exist_ok=True)
    results = load_existing_results(results_path)
    results.setdefault("meta", {})
    results["meta"].update({
        "adapter_checkpoint": str(args.adapter_dir),
        "base_model": BASE_MODEL_NAME,
        "max_seq_length": MAX_SEQ_LENGTH,
        "reasoning_strength": REASONING_STRENGTH,
        "last_run_stage": args.stage,
        "last_run_at": time.strftime("%Y-%m-%d %H:%M:%S"),
    })
    results.setdefault("timings", {})

    holdout_items = load_holdout_items()
    log(f"Loaded {len(holdout_items)} holdout items.")

    stage_list = ["translation", "sigma_authoring", "yara", "qualitative"] if args.stage == "all" else [args.stage]
    t_overall = time.time()

    needs_tuned = any(
        s in stage_list
        for s in ("translation", "sigma_authoring", "yara", "qualitative", "qualitative_tuned", "translation_footnote",
                   "yara_holdout", "yara_repair", "yara_from_iocs")
    )

    if needs_tuned:
        t0 = time.time()
        model, tok = load_tuned_model(args.adapter_dir)
        results["timings"]["model_load_tuned_s"] = round(time.time() - t0, 1)

        if "translation" in stage_list:
            key = "translation" if mnt == METRIC_MAX_NEW_TOKENS else f"translation_{mnt}"
            results[key] = run_translation_stage(model, tok, holdout_items, max_new_tokens=mnt)
            write_json(results_path, results)
        if "translation_footnote" in stage_list:
            if "translation" not in results:
                raise SystemExit("translation_footnote requires results['translation'] (@512) to already exist -- run --stage translation first.")
            results["translation_truncation_footnote"] = run_translation_truncation_footnote(
                model, tok, holdout_items, results["translation"], max_new_tokens=mnt,
            )
            write_json(results_path, results)
        if "sigma_authoring" in stage_list:
            key = "sigma_authoring" if mnt == METRIC_MAX_NEW_TOKENS else f"sigma_authoring_{mnt}"
            results[key] = run_sigma_authoring_stage(model, tok, holdout_items, max_new_tokens=mnt)
            write_json(results_path, results)
        if "yara" in stage_list:
            key = "yara" if mnt == METRIC_MAX_NEW_TOKENS else f"yara_{mnt}"
            results[key] = run_yara_stage(model, tok, max_new_tokens=mnt)
            write_json(results_path, results)
        if "yara_holdout" in stage_list:
            key = "yara_holdout" if mnt == METRIC_MAX_NEW_TOKENS else f"yara_holdout_{mnt}"
            results[key] = run_yara_holdout_stage(model, tok, max_new_tokens=mnt)
            write_json(results_path, results)
        if "yara_repair" in stage_list:
            results["yara_repair"] = run_yara_repair_stage(model, tok, max_new_tokens=mnt)
            write_json(results_path, results)
        if "yara_from_iocs" in stage_list:
            results["yara_from_iocs"] = run_yara_from_iocs_stage(model, tok, max_new_tokens=mnt)
            write_json(results_path, results)
        if "qualitative" in stage_list:
            tuned_qual_prompts, tuned_qual_outputs, chat_items, qual_tuned_s = run_qualitative_tuned_half(model, tok, holdout_items)
            results["timings"]["qualitative_tuned_s"] = qual_tuned_s
            results["chat_regression"] = {"items": chat_items}
            write_json(results_path, results)
        if "qualitative_tuned" in stage_list:
            yara_qual_override = None
            if args.yara_qual_from_holdout:
                yh_items = load_yara_holdout_items()
                repair_sel = select_yara_repair_items(yh_items, n=15, seed=42)
                iocs_sel = select_yara_iocs_items(yh_items, {it["key"] for it in repair_sel}, n=10, seed=42)
                excl = {it["key"] for it in repair_sel} | {it["key"] for it in iocs_sel}
                yara_qual_override = pick_yara_qual_holdout_item(yh_items, excl, seed=42)
            tuned_qual_prompts, tuned_qual_outputs, chat_items, qual_tuned_s = run_qualitative_tuned_half(
                model, tok, holdout_items, yara_qual_item_override=yara_qual_override,
            )
            results["timings"]["qualitative_tuned_s"] = qual_tuned_s
            results["chat_regression"] = {"items": chat_items}
            results["_qualitative_pending"] = {"prompts": tuned_qual_prompts, "tuned": tuned_qual_outputs}
            write_json(results_path, results)
            log("Tuned half stashed in results['_qualitative_pending']. Run "
                "'--stage qualitative_base' as a SEPARATE fresh-process invocation next.")

        free_model(model)

    if "qualitative" in stage_list:
        t0 = time.time()
        base_model, base_tok = load_base_model()
        results["timings"]["model_load_base_s"] = round(time.time() - t0, 1)
        base_outputs, qual_base_s = run_qualitative_base_half(base_model, base_tok, tuned_qual_prompts)
        results["timings"]["qualitative_base_s"] = qual_base_s
        free_model(base_model)

        merged_prompts = []
        for p, tuned_o, base_o in zip(tuned_qual_prompts, tuned_qual_outputs, base_outputs):
            merged_prompts.append({**p, "tuned": tuned_o, "base": base_o})
        results["qualitative"] = {"sampling": QUALITATIVE_SAMPLING, "seed": QUALITATIVE_SEED, "prompts": merged_prompts}
        write_json(results_path, results)

    if "qualitative_base" in stage_list:
        pending = results.get("_qualitative_pending")
        if not pending:
            raise SystemExit(
                "qualitative_base requires results['_qualitative_pending'] -- run "
                "'--stage qualitative_tuned' first, in a separate invocation."
            )
        t0 = time.time()
        base_model, base_tok = load_base_model()
        results["timings"]["model_load_base_s"] = round(time.time() - t0, 1)
        base_outputs, qual_base_s = run_qualitative_base_half(base_model, base_tok, pending["prompts"])
        results["timings"]["qualitative_base_s"] = qual_base_s
        free_model(base_model)

        merged_prompts = []
        for p, tuned_o, base_o in zip(pending["prompts"], pending["tuned"], base_outputs):
            merged_prompts.append({**p, "tuned": tuned_o, "base": base_o})
        results["qualitative"] = {"sampling": QUALITATIVE_SAMPLING, "seed": QUALITATIVE_SEED, "prompts": merged_prompts}
        del results["_qualitative_pending"]
        write_json(results_path, results)

    results["timings"]["last_invocation_total_s"] = round(time.time() - t_overall, 1)
    write_json(results_path, results)

    # Render eval/RESULTS.md: v0.1 content (eval/results.json) FIRST, unchanged
    # from render_results_md()'s own output, then -- if eval/results_v02.json
    # exists -- the v0.1-vs-v0.2 comparison + new-capability sections appended
    # after it. This works whether `results_path` pointed at the v0.1 or v0.2
    # file this invocation: whichever one wasn't just written is re-read fresh
    # from disk, so neither file's content ever depends on which stage ran.
    v01_results = results if results_path == RESULTS_JSON else load_existing_results(RESULTS_JSON)
    v02_results = results if results_path == RESULTS_V02_JSON else load_existing_results(RESULTS_V02_JSON)

    if v01_results:
        v01_adapter = v01_results.get("meta", {}).get("adapter_checkpoint", str(ADAPTER_CKPT_DEFAULT))
        md = render_results_md(v01_results, v01_adapter)
    else:
        md = "# Task 4 evaluation results -- Glimmer-Sentry-30B\n\n_No v0.1 results yet (eval/results.json missing)._\n"
    if v02_results:
        v02_adapter = v02_results.get("meta", {}).get("adapter_checkpoint", "?")
        md += render_v02_report(v01_results, v02_results, v02_adapter)

    write_text(RESULTS_MD, md)
    log(f"Wrote {results_path} and {RESULTS_MD}")
    log("Done.")


if __name__ == "__main__":
    main()