File size: 6,640 Bytes
264b834 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | {
"description": "Training pool of the energy-gate encoder and head (rows per source, after the advice-label rule, before the validation split). 'eval:<bench>' = routing texts of an evaluation benchmark, 'fit:pb_<bench>' = the seed-42 half of an official harmful benchmark, 'fit:jb_<technique>' = jailbreak-technique rows, 'fit:wildjailbreak_*' / 'fit:wildguardmix_*' = the patch rows (official train splits; ablation A6 removes these 41,422 rows). Deduplicated by text.",
"n": 108897,
"labels": {
"0": 71340,
"1": 37557
},
"by_source": {
"fit:wildjailbreak_adversarial_benign": 12000,
"falsereject_train": 11653,
"fit:wildjailbreak_adversarial_harmful": 10000,
"fit:wildguardmix_vanilla_harmful": 4969,
"fit:wildguardmix_vanilla_benign": 4954,
"fit:wildguardmix_adversarial_harmful": 4782,
"fit:wildguardmix_adversarial_benign": 4717,
"hellaswag_train": 3000,
"math_train": 2999,
"dolly": 2995,
"alpaca": 2986,
"furonghuang-lab/PHTest:default:train": 2980,
"AmazonScience/FalseReject:default:train": 2971,
"orbench_80k": 2963,
"OpenSafetyLab/Salad-Data": 2957,
"sorry-bench/sorry-bench-202406": 2746,
"JailbreakV-28K/JailBreakV-28k:JailBreakV_28K:JailBreakV_28K": 2742,
"arc_easy_train": 2251,
"gsm8k_train": 2000,
"ethics_cs_train": 2000,
"obqa_train": 2000,
"csqa_train": 2000,
"JailbreakV-28K/JailBreakV-28k:RedTeam_2K:RedTeam_2K": 1949,
"declare-lab/HarmfulQA:default:train": 1906,
"mmlu_valdev": 1812,
"orbench_hard_train": 1210,
"AmazonScience/FalseReject:default:test": 1176,
"LibrAI/do-not-answer:default:train": 928,
"fit:pb_mmsafetybench": 778,
"morables_story": 709,
"fit:pb_airbench": 682,
"orbench_toxic": 655,
"walledai/CatHarmfulQA:default:en": 542,
"xstest_rest": 405,
"allenai/coconot:contrast:test": 375,
"https://raw.githubusercontent.com/alexandrasouly/strongreject/main/strongreject_dataset/strongreject_dataset.csv": 271,
"eval:morables": 180,
"eval:airbench": 160,
"eval:ethics_cs": 150,
"fit:pb_harmbench": 134,
"eval:mmsafetybench": 130,
"eval:orbench": 100,
"Bertievidgen/SimpleSafetyTests:default:test": 100,
"JailbreakBench/JBB-Behaviors:behaviors:benign": 100,
"walledai/MaliciousInstruct:default:train": 99,
"JailbreakBench/JBB-Behaviors:behaviors:harmful": 81,
"eval:llm_ethics": 79,
"fit:jb_low_resource_english": 60,
"fit:jb_obfuscation": 60,
"fit:jb_evil_confidant": 60,
"fit:jb_prefix_injection": 60,
"fit:jb_dev_mode_v2": 60,
"fit:jb_multi_shot_5": 60,
"fit:jb_base64": 60,
"fit:jb_base64_input_only": 60,
"fit:jb_base64_output_only": 60,
"fit:jb_leetspeak": 60,
"fit:jb_combination_1": 60,
"fit:jb_evil_system_prompt": 60,
"fit:jb_combination_3": 60,
"fit:jb_low_resource": 60,
"fit:jb_refusal_suppression_inv": 60,
"fit:jb_style_injection_short": 60,
"fit:jb_distractors": 60,
"fit:jb_prefix_injection_hello": 60,
"fit:jb_gcg": 60,
"fit:jb_AIM": 60,
"fit:jb_few_shot_json": 60,
"fit:jb_refusal_suppression": 60,
"eval:harmbench": 50,
"eval:xstest": 45,
"eval:taxonomy": 40,
"fit:phtest": 39,
"fit:falsereject": 11,
"eval:ggb": 9,
"fit:coconot_contrast": 4,
"fit:jbb_harmful": 3
},
"advice_label_rule": {
"rule": "Requests for personal legal, financial (incl. accounting) or medical (incl. mental-health) advice are label 0: they are to be answered by the backend model. They are selected by each dataset's own category, in every source group (public collections, benchmark rows, routing texts, patch rows). The same advice requests wrapped in a jailbreak template, an encoding or a persuasion rewrite leave the pool, so that the label of a jailbreak wrapper stays harmful. Matching is by stripped text; a relabelled row keeps its place in the validation split. Code: cpeg/energy/advice.py.",
"categories": {
"MM-SafetyBench": "scenarios 10-Legal_Opinion, 11-Financial_Advice, 12-Health_Consultation (benchmark units and pool rows)",
"AIR-Bench 2024": "cate-idx 2.6.22-2.6.25 (legal, medical / pharmaceutical, accounting, financial advice in heavily regulated industries); 2.6.26 (government services) stays harmful",
"do-not-answer": "specific_harms starting with Legal Advice, Financial Advice, Medical Advice, Consult advice on psychotic disorders, Advice of coping strategies",
"SORRY-Bench": "categories whose name contains medical advice / financial advice / legal consulting advice / legal advice; plain-language mutations relabelled, encoded and persuasion mutations (ascii, atbash, caesar, morse, authority_endorsement, evidence-based_persuasion, expert_endorsement, logical_appeal, misrepresentation, role_play) dropped",
"SALAD-Bench": "base set, 3-category O17 (law advice) and O20 (medical advice)",
"JailBreakV-28K": "policies Health Consultation and Tailored Unlicensed Advice: RedTeam-2K questions and the 28K set's redteam_query relabelled, the 28K set's jailbreak_query dropped"
},
"selected_texts": {
"relabel": {
"do-not-answer legal / financial / medical / mental-health advice": 75,
"sorry-bench medical / financial / legal advice, plain-language mutations": 329,
"salad-bench O17 law / O20 medical advice": 431,
"jailbreakv health consultation / tailored unlicensed advice": 243
},
"drop": {
"jailbreakv advice in a jailbreak template": 663,
"sorry-bench advice, encoded / persuasion mutations": 300
},
"relabel_total_incl_benchmark_advice_units": 1487,
"drop_total_after_removing_relabel_texts": 963
},
"relabelled_rows_by_source": {
"JailbreakBench/JBB-Behaviors:behaviors:harmful": 1,
"JailbreakV-28K/JailBreakV-28k:RedTeam_2K:RedTeam_2K": 244,
"LibrAI/do-not-answer:default:train": 79,
"OpenSafetyLab/Salad-Data": 30,
"eval:mmsafetybench": 30,
"fit:pb_airbench": 4,
"fit:pb_mmsafetybench": 192,
"fit:wildguardmix_vanilla_harmful": 1,
"sorry-bench/sorry-bench-202406": 86
},
"dropped_rows_by_source": {
"JailbreakV-28K/JailBreakV-28k:JailBreakV_28K:JailBreakV_28K": 245,
"sorry-bench/sorry-bench-202406": 82
},
"pool_before_rule": {
"n": 109224,
"labels": {
"0": 70673,
"1": 38551
}
}
},
"heldout_never_trained": {
"heldout:csqa": 1221,
"heldout:falsereject": 1187,
"heldout:arc_c": 1172,
"heldout:phtest": 500,
"heldout:math500": 500,
"heldout:xstest": 405,
"heldout:coconot_contrast": 379,
"heldout:orbench_hard": 300,
"heldout:gsm8k": 200,
"heldout:simplesafetytests": 100,
"heldout:jbb_harmful": 73
}
}
|