| { | |
| "description": "Training pool of the energy-gate encoder and head (rows per source, after the advice-label rule, before the validation split). 'eval:<bench>' = routing texts of an evaluation benchmark, 'fit:pb_<bench>' = the seed-42 half of an official harmful benchmark, 'fit:jb_<technique>' = jailbreak-technique rows, 'fit:wildjailbreak_*' / 'fit:wildguardmix_*' = the patch rows (official train splits; ablation A6 removes these 41,422 rows). Deduplicated by text.", | |
| "n": 108897, | |
| "labels": { | |
| "0": 71340, | |
| "1": 37557 | |
| }, | |
| "by_source": { | |
| "fit:wildjailbreak_adversarial_benign": 12000, | |
| "falsereject_train": 11653, | |
| "fit:wildjailbreak_adversarial_harmful": 10000, | |
| "fit:wildguardmix_vanilla_harmful": 4969, | |
| "fit:wildguardmix_vanilla_benign": 4954, | |
| "fit:wildguardmix_adversarial_harmful": 4782, | |
| "fit:wildguardmix_adversarial_benign": 4717, | |
| "hellaswag_train": 3000, | |
| "math_train": 2999, | |
| "dolly": 2995, | |
| "alpaca": 2986, | |
| "furonghuang-lab/PHTest:default:train": 2980, | |
| "AmazonScience/FalseReject:default:train": 2971, | |
| "orbench_80k": 2963, | |
| "OpenSafetyLab/Salad-Data": 2957, | |
| "sorry-bench/sorry-bench-202406": 2746, | |
| "JailbreakV-28K/JailBreakV-28k:JailBreakV_28K:JailBreakV_28K": 2742, | |
| "arc_easy_train": 2251, | |
| "gsm8k_train": 2000, | |
| "ethics_cs_train": 2000, | |
| "obqa_train": 2000, | |
| "csqa_train": 2000, | |
| "JailbreakV-28K/JailBreakV-28k:RedTeam_2K:RedTeam_2K": 1949, | |
| "declare-lab/HarmfulQA:default:train": 1906, | |
| "mmlu_valdev": 1812, | |
| "orbench_hard_train": 1210, | |
| "AmazonScience/FalseReject:default:test": 1176, | |
| "LibrAI/do-not-answer:default:train": 928, | |
| "fit:pb_mmsafetybench": 778, | |
| "morables_story": 709, | |
| "fit:pb_airbench": 682, | |
| "orbench_toxic": 655, | |
| "walledai/CatHarmfulQA:default:en": 542, | |
| "xstest_rest": 405, | |
| "allenai/coconot:contrast:test": 375, | |
| "https://raw.githubusercontent.com/alexandrasouly/strongreject/main/strongreject_dataset/strongreject_dataset.csv": 271, | |
| "eval:morables": 180, | |
| "eval:airbench": 160, | |
| "eval:ethics_cs": 150, | |
| "fit:pb_harmbench": 134, | |
| "eval:mmsafetybench": 130, | |
| "eval:orbench": 100, | |
| "Bertievidgen/SimpleSafetyTests:default:test": 100, | |
| "JailbreakBench/JBB-Behaviors:behaviors:benign": 100, | |
| "walledai/MaliciousInstruct:default:train": 99, | |
| "JailbreakBench/JBB-Behaviors:behaviors:harmful": 81, | |
| "eval:llm_ethics": 79, | |
| "fit:jb_low_resource_english": 60, | |
| "fit:jb_obfuscation": 60, | |
| "fit:jb_evil_confidant": 60, | |
| "fit:jb_prefix_injection": 60, | |
| "fit:jb_dev_mode_v2": 60, | |
| "fit:jb_multi_shot_5": 60, | |
| "fit:jb_base64": 60, | |
| "fit:jb_base64_input_only": 60, | |
| "fit:jb_base64_output_only": 60, | |
| "fit:jb_leetspeak": 60, | |
| "fit:jb_combination_1": 60, | |
| "fit:jb_evil_system_prompt": 60, | |
| "fit:jb_combination_3": 60, | |
| "fit:jb_low_resource": 60, | |
| "fit:jb_refusal_suppression_inv": 60, | |
| "fit:jb_style_injection_short": 60, | |
| "fit:jb_distractors": 60, | |
| "fit:jb_prefix_injection_hello": 60, | |
| "fit:jb_gcg": 60, | |
| "fit:jb_AIM": 60, | |
| "fit:jb_few_shot_json": 60, | |
| "fit:jb_refusal_suppression": 60, | |
| "eval:harmbench": 50, | |
| "eval:xstest": 45, | |
| "eval:taxonomy": 40, | |
| "fit:phtest": 39, | |
| "fit:falsereject": 11, | |
| "eval:ggb": 9, | |
| "fit:coconot_contrast": 4, | |
| "fit:jbb_harmful": 3 | |
| }, | |
| "advice_label_rule": { | |
| "rule": "Requests for personal legal, financial (incl. accounting) or medical (incl. mental-health) advice are label 0: they are to be answered by the backend model. They are selected by each dataset's own category, in every source group (public collections, benchmark rows, routing texts, patch rows). The same advice requests wrapped in a jailbreak template, an encoding or a persuasion rewrite leave the pool, so that the label of a jailbreak wrapper stays harmful. Matching is by stripped text; a relabelled row keeps its place in the validation split. Code: cpeg/energy/advice.py.", | |
| "categories": { | |
| "MM-SafetyBench": "scenarios 10-Legal_Opinion, 11-Financial_Advice, 12-Health_Consultation (benchmark units and pool rows)", | |
| "AIR-Bench 2024": "cate-idx 2.6.22-2.6.25 (legal, medical / pharmaceutical, accounting, financial advice in heavily regulated industries); 2.6.26 (government services) stays harmful", | |
| "do-not-answer": "specific_harms starting with Legal Advice, Financial Advice, Medical Advice, Consult advice on psychotic disorders, Advice of coping strategies", | |
| "SORRY-Bench": "categories whose name contains medical advice / financial advice / legal consulting advice / legal advice; plain-language mutations relabelled, encoded and persuasion mutations (ascii, atbash, caesar, morse, authority_endorsement, evidence-based_persuasion, expert_endorsement, logical_appeal, misrepresentation, role_play) dropped", | |
| "SALAD-Bench": "base set, 3-category O17 (law advice) and O20 (medical advice)", | |
| "JailBreakV-28K": "policies Health Consultation and Tailored Unlicensed Advice: RedTeam-2K questions and the 28K set's redteam_query relabelled, the 28K set's jailbreak_query dropped" | |
| }, | |
| "selected_texts": { | |
| "relabel": { | |
| "do-not-answer legal / financial / medical / mental-health advice": 75, | |
| "sorry-bench medical / financial / legal advice, plain-language mutations": 329, | |
| "salad-bench O17 law / O20 medical advice": 431, | |
| "jailbreakv health consultation / tailored unlicensed advice": 243 | |
| }, | |
| "drop": { | |
| "jailbreakv advice in a jailbreak template": 663, | |
| "sorry-bench advice, encoded / persuasion mutations": 300 | |
| }, | |
| "relabel_total_incl_benchmark_advice_units": 1487, | |
| "drop_total_after_removing_relabel_texts": 963 | |
| }, | |
| "relabelled_rows_by_source": { | |
| "JailbreakBench/JBB-Behaviors:behaviors:harmful": 1, | |
| "JailbreakV-28K/JailBreakV-28k:RedTeam_2K:RedTeam_2K": 244, | |
| "LibrAI/do-not-answer:default:train": 79, | |
| "OpenSafetyLab/Salad-Data": 30, | |
| "eval:mmsafetybench": 30, | |
| "fit:pb_airbench": 4, | |
| "fit:pb_mmsafetybench": 192, | |
| "fit:wildguardmix_vanilla_harmful": 1, | |
| "sorry-bench/sorry-bench-202406": 86 | |
| }, | |
| "dropped_rows_by_source": { | |
| "JailbreakV-28K/JailBreakV-28k:JailBreakV_28K:JailBreakV_28K": 245, | |
| "sorry-bench/sorry-bench-202406": 82 | |
| }, | |
| "pool_before_rule": { | |
| "n": 109224, | |
| "labels": { | |
| "0": 70673, | |
| "1": 38551 | |
| } | |
| } | |
| }, | |
| "heldout_never_trained": { | |
| "heldout:csqa": 1221, | |
| "heldout:falsereject": 1187, | |
| "heldout:arc_c": 1172, | |
| "heldout:phtest": 500, | |
| "heldout:math500": 500, | |
| "heldout:xstest": 405, | |
| "heldout:coconot_contrast": 379, | |
| "heldout:orbench_hard": 300, | |
| "heldout:gsm8k": 200, | |
| "heldout:simplesafetytests": 100, | |
| "heldout:jbb_harmful": 73 | |
| } | |
| } | |