scripts/build_dataset.py

# -*- coding: utf-8 -*-
"""Merge tracks A+B+C, add 2 C-boundary samples, run the DATASET-GATE (print or no train):
- counts per track
- 0 blacklist IDs (8 excluded oss-fuzz + all 200 eval task ids) in ids AND content
- 0 leak markers in user turns (maintainer names, triage/close states, panel, Sherlock)
- schema check: <think>, ### TRIGGER, ### EXPLOIT WRITEUP, ### VERDICT in every assistant
- G1/G2/G6/G7/G8 verdict check
- p50/p95 lengths; over-length assistant check
- print 5 gold lines (G1, G2, G7, 1 track-A task, 1 lab) as required
Outputs: /home/corov/cyber/data/train_all.jsonl (+ report)"""

import json, os, re, sys, statistics

sys.path.insert(0, "/mnt/c/Users/corov/Desktop/Qwen-Cyber/scripts")
import trackb_part1, trackb_part2, trackb_part3, trackb_part4, trackb_part5, build_trackC  # registers on S
from trackb_part1 import SYSTEM, S, add

C_PREAMBLE = ("Analyze this code-security question (local lab / review context). "
              "Deliver: white-box analysis, trigger, exploit writeup, verdict.\n\n")

DATA = "/home/corov/cyber/data"

# ---- 2 C-boundary samples (teach FP/HYGIENE in the C context too)
add("C_bound_fortify_not_safe",
"""We ship with _FORTIFY_SOURCE=2, so every strcpy/strcpy-class overflow aborts safely at runtime. Security review says our unbounded strcpy into a 16-byte buffer (user-supplied hostname) is therefore NOT a vulnerability - fortify catches it. Do you agree? Verdict please.""",
"""White-Box: Fortify is a compile-time-inserted bounds check for a subset of libc calls where the compiler can SEE the destination size (known-at-compile-time objects). It catches this lab's shape (fixed 16-byte array) - verified: the same binary aborts with "*** buffer overflow detected ***" under default hardening. But the claim generalizes to "every overflow": fortify does NOT catch (a) destinations whose size is runtime-dependent (malloc'd, VLA-ish patterns, pointers passed through layers), (b) non-instrumented functions, (c) off-by-one writes outside instrumented copy families. Mitigations reduce reachable instances of a bug class; they do not remove the bug. The finding under review - unbounded strcpy on user input - stays a real stack-overflow write primitive whose exploitability is environment-dependent. Class: the CODE bug is VALID_BUG; the review's claim (not a vulnerability) is the false positive.""",
"""Verified both flavors on the lab build:

  default hardening:  "*** buffer overflow detected ***: terminated" (SIGABRT)
  -D_FORTIFY_SOURCE=0 + ASan: SUMMARY: AddressSanitizer: stack-buffer-overflow ... in strcpy

The fault exists in both builds; only the catcher differs.""",
"""Root cause: unbounded copy into fixed stack buffer; fortify is a runtime mitigation, not a fix of the source-level fault.
Reachability: user-controlled hostname; unconditional.
Primitive: stack OOB write (mitigated in this configuration - stated).
Chain: same as classic stack overflow where fortify is absent or inapplicable.
Missing to next rung: n/a - the classification point IS the verdict.
Stability: fortify coverage is compile-context dependent (optimization level, visible sizes).
Fix sketch: bound the copy at the source; keep fortify as depth, never as the boundary.""",
"VALID_BUG", "high",
"""REASONING: mitigation != fix - fortify aborts this instance (verified) but covers only size-visible instrumented copies; the unbounded strcpy on user input remains a real write primitive. The 'not a vulnerability' review is the error.""", preamble=C_PREAMBLE)

add("C_bound_harness_only",
"""Our nightly fuzz job crashes in tools/target_dec_fuzzer.c itself (the fuzz harness allocates non-zeroed buffers the production decoder never uses). CI says: 'security vulnerability in the product'. The patch zeroes the harness buffers (av_buffer_allocz). Product code unchanged. Is this a product security vulnerability?""",
"""White-Box: Where does the fault live - product code or the test harness? The crash (MemorySanitizer: use-of-uninitialized-value) fires in the FUZZER TOOL's own buffer allocation (tools/target_dec_fuzzer.c); the fix swaps av_buffer_alloc -> av_buffer_allocz in the harness file only; production decoder untouched. The fuzzer's uninitialized buffers make the harness's own behavior non-deterministic - a test-infrastructure correctness problem that can produce false positives in nightly jobs. Product users never execute that code. Gates: F11-family - the changed surface is not the shipped attack surface. Class: HYGIENE (test-infrastructure), severity none; the 'product vulnerability' escalation is wrong. Boundary honesty: IF the same uninitialized-buffer pattern existed in the PRODUCT allocator paths, it would be a VALID read-class bug - that is a different file and a different report.""",
"""Nightly fuzz job log shows: ERROR: MemorySanitizer: use-of-uninitialized-value in the fuzzer tool's get_buffer path. Apply the harness patch (zeroed allocation): the same corpus runs clean - demonstrating the fault was the harness's non-zeroed buffers, not the product decoder. Product binary unchanged before/after.""",
"""Root cause: harness (test tool) allocating uninitialized buffers the production code zeroes - deterministic behavior divergence only inside the fuzz job.
Reachability: fuzz infrastructure only; zero product reachability.
Primitive: none for the product (no shipped code executes it).
Chain: none for the product.
Missing for VALID: the same pattern in product allocation paths (absent - file-level proof).
Stability: n/a for product.
Fix sketch: the harness patch itself (zeroed allocs) + CI rule: crashes fixed by harness-only patches are filed as infrastructure, not product security.""",
"HYGIENE", "none",
"""REASONING: fix lands exclusively in tools/target_dec_fuzzer.c (harness), product code unchanged - the uninitialized values never execute in shipped builds; test-infrastructure drift, not a product vulnerability.""" , preamble=C_PREAMBLE)

def main():
    # 1) merge - v3: Track A subsampled 280->180 (project-stratified) to rebalance;
    # Track B (now with phrasing variants) x3.
    import random as _rnd
    from collections import defaultdict
    trackA_all = [json.loads(l) for l in open(f"{DATA}/trackA.jsonl", encoding="utf-8")]
    tid2proj = {f'{r["kind"]}_{r["id"]}': r.get("project", "?")
                for r in json.load(open(f"{DATA}/train_ids.json"))}
    groups = defaultdict(list)
    for s in trackA_all:
        parts = s["id"].split("_", 2)
        groups[tid2proj.get(parts[1] + "_" + parts[2], "?")].append(s)
    trackA, taken = [], 0
    ratio = 180 / len(trackA_all)
    for proj, items in sorted(groups.items()):
        _rnd.Random(123).shuffle(items)
        n = max(1, round(len(items) * ratio))
        trackA.extend(items[:n]); taken += n
    seen, mergedB, merged = set(), [], []
    for s in trackA + S:
        if s["id"] in seen:
            continue
        seen.add(s["id"])
        if s["id"].startswith(("A_", "C_")):
            merged.append(s)
        else:
            mergedB.append(s)
    merged = merged + mergedB * 3  # Track B x3

    report = []
    def p(line=""):
        print(line); report.append(line)

    # 2) counts
    cnt = {"A": 0, "B": 0, "C": 0}
    for s in merged:
        if s["id"].startswith("A_"):
            cnt["A"] += 1
        elif s["id"].startswith("C_"):
            cnt["C"] += 1
        else:
            cnt["B"] += 1
    p("=== DATASET-GATE ===")
    p(f"counts: A(cybergym)={cnt['A']}  B(xrpl, base)={cnt['B']//3} x3  C(labs+boundary)={cnt['C']}  total={len(merged)}")

    # 3) blacklist check: ids + content
    BL_IDS = {"42536536", "42537493", "42537664", "42537686", "42537734", "42538131", "383170474", "383825645"}
    eval_ids = {str(r["id"]) for r in json.load(open(f"{DATA}/eval_ids.json"))}
    bl_hits = []
    for s in merged:
        blob = json.dumps(s)
        for bad in (BL_IDS | eval_ids):
            if f'"{bad}"' in blob or f"arvo:{bad}" in blob or f"oss-fuzz:{bad}" in blob:
                bl_hits.append((s["id"], bad))
                break
    p(f"blacklist hits (8 excluded + 200 eval ids): {len(bl_hits)}")
    for sid, bad in bl_hits[:5]:
        p(f"  HIT {sid} -> {bad}")

    # 4) leak scan on user turns
    LEAK = re.compile(r"triage result|this is intentional|maintainer|mvadari|ximinez|seelabs|scottschurr|JoelKatz|vinniefalco|nbougalis|bthomee|state:\s*(closed|open)|closed as|not planned|panel (said|accepted)|sherlock|classified via|confirmed by", re.I)
    leaks = [(s["id"], LEAK.search(s["messages"][1]["content"]).group(0)) for s in merged
             if s.get("messages") and len(s["messages"]) > 1 and LEAK.search(s["messages"][1]["content"])]
    p(f"user-turn leak markers: {len(leaks)}")
    for sid, m in leaks[:5]:
        p(f"  LEAK {sid}: {m}")

    # 5) schema check
    bad_schema = []
    for s in merged:
        a = s["messages"][-1]["content"]
        if not (a.startswith("<think>") and "### TRIGGER" in a and "### EXPLOIT WRITEUP" in a and "### VERDICT" in a):
            bad_schema.append(s["id"])
    p(f"schema violations (think/TRIGGER/WRITEUP/VERDICT): {len(bad_schema)} {bad_schema[:5]}")

    # 6) G-set verdict check
    need = {"G1": "FALSE_POSITIVE", "G1v2": "FALSE_POSITIVE", "G2": "FALSE_POSITIVE", "G2v2": "FALSE_POSITIVE",
            "G6": "HYGIENE", "G6v2": "HYGIENE", "G7": "VALID_BUG TRACK:safety PATTERN:N11",
            "G7v2": "VALID_BUG TRACK:safety PATTERN:N11", "G8": "HYGIENE", "G8v2": "HYGIENE"}
    byid = {s["id"]: s for s in merged}
    gfail = []
    for gid, want in need.items():
        s = byid.get(gid)
        if not s:
            gfail.append((gid, "MISSING")); continue
        v = re.search(r"### VERDICT\n([^\n]+)", s["messages"][-1]["content"])
        got = v.group(1).strip() if v else "?"
        if want not in got:
            gfail.append((gid, f"got={got}"))
    p(f"G-set verdict check: {'ALL OK' if not gfail else gfail}")

    # 7) verdict distribution
    from collections import Counter
    verd = Counter()
    for s in merged:
        v = re.search(r"### VERDICT\n([^\n]+)", s["messages"][-1]["content"])
        verd[v.group(1).strip().split(" TRACK")[0] if v else "?"] += 1
    p(f"verdict distribution: {dict(verd)}")

    # 8) lengths (approx tokens = chars/3.6) + over-length assistant check
    ulens, alens = [], []
    toolong = []
    for s in merged:
        ulens.append(len(s["messages"][1]["content"]) // 4)
        alens.append(len(s["messages"][-1]["content"]) // 4)
        if alens[-1] > 2400:
            toolong.append(s["id"])
    def pct(v, q):
        v = sorted(v); return v[int(len(v) * q)]
    p(f"len tokens p50/p95: user {pct(ulens,.5)}/{pct(ulens,.95)}  assistant {pct(alens,.5)}/{pct(alens,.95)}")
    p(f"assistant >2400 tok (will be packed but check): {len(toolong)} {toolong[:5]}")

    # 9) five gold lines
    p("\n=== 5 GOLD LINES ===")
    for gid in ["G1", "G2", "G7"]:
        s = byid[gid]
        p(f"--- {gid}: user[:200]={s['messages'][1]['content'][:200]!r}")
        content = s['messages'][-1]['content']
        vidx = content.find('### VERDICT')
        p(f"    verdict[:140]={content[vidx:vidx+140]!r}")
    a1 = next(s for s in merged if s["id"].startswith("A_"))
    p(f"--- {a1['id']}: user[:200]={a1['messages'][1]['content'][:200]!r}")
    lab = byid.get("C_lab06_int_overflow") or next(s for s in merged if s["id"].startswith("C_"))
    p(f"--- {lab['id']}: user[:200]={lab['messages'][1]['content'][:200]!r}")

    gate_ok = (len(bl_hits) == 0 and len(leaks) == 0 and len(bad_schema) == 0 and not gfail)
    p(f"\nGATE: {'PASS - trainer may start' if gate_ok else 'FAIL - fix before training'}")

    if gate_ok:
        with open(f"{DATA}/train_all.jsonl", "w", encoding="utf-8", newline="\n") as f:
            for s in merged:
                f.write(json.dumps(s, ensure_ascii=False) + "\n")
        p(f"wrote {DATA}/train_all.jsonl ({len(merged)} samples)")
    with open("/mnt/c/Users/corov/Desktop/Qwen-Cyber/dataset_gate_report.txt", "w", encoding="utf-8", newline="\n") as f:
        f.write("\n".join(report))

if __name__ == "__main__":
    main()