code/training/configs/datasets.yaml

# Dataset acquisition + adapter manifest for the CyberGym SFT mix.
#
# This is the machine-readable contract consumed by:
#   - training/scripts/hf_download.py       (fetch raw split -> data/download/<key>/raw.jsonl)
#   - training/scripts/build_sft_dataset.py (adapt -> normalize -> dedup -> data card)
#
# Schemas below were verified against the live Hugging Face datasets-server on
# 2026-06-27 (see `python training/scripts/hf_download.py --probe <hf_id>`).
# `group` / `target_token_fraction` mirror training/configs/data_mix.yaml.
#
# think field legend:
#   present         -> source already ships <think> reasoning in the answer
#   needs_synthesis -> reasoning must be backfilled (rejection sampling teacher pass)
#   mixed           -> some rows have <think>, some do not (decided per row at build time)
#
# auth field legend:
#   public  -> anonymous datasets-server/download works
#   gated   -> datasets-server returned 401 on 2026-06-27; needs `huggingface-cli login`
#              (and possibly access approval) before download

defaults:
  raw_dir: data/download
  sample_cap: null          # null = take the whole split; integer caps rows for cost control
  pilot_sample_cap: 2000    # used when build is invoked with --profile pilot

sources:
  # ---- Tier 1: C/C++ vulnerability core ----------------------------------- #
  primevul:
    hf_id: colin/PrimeVul
    config: default
    train_split: train
    eval_split: test            # reserve for held-out vuln-detection accuracy
    adapter: detection_func_target
    params: {code_field: func, label_field: target, language: "C/C++"}
    group: c_cpp_vulnerability_core
    license: MIT
    think: needs_synthesis
    auth: public
    enabled: true

  diversevul:
    hf_id: bstee615/diversevul
    config: default
    train_split: train
    eval_split: test
    adapter: detection_func_target
    params: {code_field: func, label_field: target, language: "C/C++"}
    group: c_cpp_vulnerability_core
    license: missing            # record provenance; prefer PrimeVul where overlapping
    think: needs_synthesis
    auth: public
    enabled: true

  megavul:
    hf_id: athrv/megavul-vulnerability-detection
    config: default
    train_split: train
    eval_split: test
    adapter: instruction_io
    params: {}
    group: c_cpp_vulnerability_core
    license: missing
    think: needs_synthesis      # output is written analysis but without <think>
    auth: public
    enabled: true

  crossvul:
    hf_id: hitoshura25/crossvul
    config: default
    train_split: train
    eval_split: null
    adapter: vuln_fix_pair
    params: {tasks: [detect, fix]}
    group: c_cpp_vulnerability_core   # multi-language; tag language in metadata
    license: Apache-2.0
    think: needs_synthesis
    auth: public
    enabled: true

  # ---- Tier 2: reasoning <think> core ------------------------------------- #
  seccoderx_reasoning:
    hf_id: SecCoderX/Reasoning_Vulnerability_Detection_SFT_Cold_Start
    config: default
    train_split: train
    eval_split: null
    adapter: instruction_io
    params: {}
    group: reasoning_think_core
    license: missing
    think: present
    auth: gated                 # datasets-server returned 401 on 2026-06-27
    enabled: false              # flip on after `huggingface-cli login` + access check

  openvul_reasoning:
    hf_id: Leopo1d/OpenVul_Rejection_Sampling_Reasoning_SFT
    config: default
    train_split: train
    eval_split: null
    adapter: chatml_conversations
    params: {}
    group: reasoning_think_core
    license: Apache-2.0
    think: present
    auth: gated                 # datasets-server returned 401 on 2026-06-27
    enabled: false

  pentesting_explanations:
    hf_id: theelderemo/pentesting-explanations
    config: default             # also: hacktricks, mitre_attack
    train_split: train
    eval_split: null
    adapter: mcq_think
    params: {}
    group: reasoning_think_core
    license: Apache-2.0
    think: present
    auth: public
    enabled: true

  # ---- Tier 3: agentic tool loop ------------------------------------------ #
  tooluse_multiturn_reasoning:
    hf_id: interstellarninja/tool-use-multiturn-reasoning
    config: default
    train_split: train
    eval_split: null
    adapter: chatml_conversations
    params: {inline_tools: true}
    group: agentic_tool_loop
    license: Apache-2.0
    think: present
    auth: public
    enabled: true

  # ---- Tier 4/5: preference + defensive breadth --------------------------- #
  cybernative_dpo:
    hf_id: CyberNative/Code_Vulnerability_Security_DPO
    config: default
    train_split: train
    eval_split: null
    adapter: dpo_to_sft         # chosen answer only for SFT; full pairs reused for Stage-2 DPO
    params: {}
    group: preference_positives_for_sft
    license: Apache-2.0
    think: needs_synthesis
    auth: public
    enabled: true

  all_cve_records:
    hf_id: AlicanKiraz0/All-CVE-Records-Training-Dataset
    config: default
    train_split: train
    eval_split: null
    adapter: system_user_assistant
    params: {}
    group: defensive_breadth
    license: Apache-2.0
    think: needs_synthesis
    auth: public
    sample_cap: 60000           # tier-5 breadth; cap so it stays <= ~15% of tokens
    enabled: true

# Eval-only sources. NEVER trained on. Used by build_eval_sets.py to produce
# held-out scoring sets for eval_endpoint.py (base baseline + per-epoch checks).
# Held-out C/C++ vuln-detection accuracy comes from the `eval_split` (test) of the
# training sources above (download with `hf_download.py --key <key> --eval`).
eval_only_sources:
  pentest_mcq_eval:
    hf_id: theelderemo/pentesting-explanations
    config: mitre_attack        # disjoint from the trained `default` config
    split: train
    kind: mcq                   # question / choices / answer_idx
    license: Apache-2.0
    auth: public
    sample_cap: 500