File size: 8,546 Bytes
c870790 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | {
"_comment": "adaptive attack 用的 D1 权重清单；由 tools/pack_attack_detectors.py 生成",
"base_model": "mistralai/Mistral-7B-v0.1",
"selection_rule": "每个家族取 3 seed × 3 run 中 new-FNR 最低者（= 最强的更新后检测器）",
"detectors": {
"browsesafe_clean__important_message": {
"dataset": "browsesafe_clean",
"family": "important_message",
"seed": "32859",
"run": 1,
"new_fnr": 0.062016,
"seed_new_fnr_mean": {
"32859": 0.178295,
"64066": 0.229974,
"7546": 0.219638
},
"run_new_fnr": {
"0": 0.170543,
"1": 0.062016,
"2": 0.302326
},
"experiment": "browsesafe_clean_k1_ng63_s32859_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 19,
"source_path": "artifacts/runs/browsesafe_clean_k1_ng63_s32859_datasentinel_scratch_agent_ckpt/important_message__run1__from_file__scratch/round3"
},
"browsesafe_clean__indirect_hypothetical": {
"dataset": "browsesafe_clean",
"family": "indirect_hypothetical",
"seed": "7546",
"run": 1,
"new_fnr": 0.150538,
"seed_new_fnr_mean": {
"32859": 0.390681,
"64066": 0.365591,
"7546": 0.268817
},
"run_new_fnr": {
"0": 0.247312,
"1": 0.150538,
"2": 0.408602
},
"experiment": "browsesafe_clean_k1_ng63_s7546_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 15,
"source_path": "artifacts/runs/browsesafe_clean_k1_ng63_s7546_datasentinel_scratch_agent_ckpt/indirect_hypothetical__run1__from_file__scratch/round3"
},
"browsesafe_clean__multilanguage": {
"dataset": "browsesafe_clean",
"family": "multilanguage",
"seed": "64066",
"run": 2,
"new_fnr": 0.056338,
"seed_new_fnr_mean": {
"32859": 0.694836,
"64066": 0.288732,
"7546": 0.330986
},
"run_new_fnr": {
"0": 0.169014,
"1": 0.640845,
"2": 0.056338
},
"experiment": "browsesafe_clean_k1_ng63_s64066_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 17,
"source_path": "artifacts/runs/browsesafe_clean_k1_ng63_s64066_datasentinel_scratch_agent_ckpt/multilanguage__run2__from_file__scratch/round3"
},
"browsesafe_clean__system_prompt_exfiltration": {
"dataset": "browsesafe_clean",
"family": "system_prompt_exfiltration",
"seed": "7546",
"run": 1,
"new_fnr": 0.083333,
"seed_new_fnr_mean": {
"32859": 0.256536,
"64066": 0.261438,
"7546": 0.254902
},
"run_new_fnr": {
"0": 0.465686,
"1": 0.083333,
"2": 0.215686
},
"experiment": "browsesafe_clean_k1_ng63_s7546_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 19,
"source_path": "artifacts/runs/browsesafe_clean_k1_ng63_s7546_datasentinel_scratch_agent_ckpt/system_prompt_exfiltration__run1__from_file__scratch/round3"
},
"promptsleuth_aug__Completion_Attack": {
"dataset": "promptsleuth_aug",
"family": "Completion_Attack",
"seed": "32859",
"run": 2,
"new_fnr": 0.018653,
"seed_new_fnr_mean": {
"32859": 0.045941,
"64066": 0.140242,
"7546": 0.149914
},
"run_new_fnr": {
"0": 0.074611,
"1": 0.04456,
"2": 0.018653
},
"experiment": "promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 98,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt/Completion_Attack__run2__from_file__scratch/round3"
},
"promptsleuth_aug__Context_Tampering": {
"dataset": "promptsleuth_aug",
"family": "Context_Tampering",
"seed": "64066",
"run": 1,
"new_fnr": 0.027461,
"seed_new_fnr_mean": {
"32859": 0.198791,
"64066": 0.095855,
"7546": 0.227116
},
"run_new_fnr": {
"0": 0.161658,
"1": 0.027461,
"2": 0.098446
},
"experiment": "promptsleuth_aug_k1_ng63_s64066_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 98,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s64066_datasentinel_scratch_agent_ckpt/Context_Tampering__run1__from_file__scratch/round3"
},
"promptsleuth_aug__Emotional_Manipulation": {
"dataset": "promptsleuth_aug",
"family": "Emotional_Manipulation",
"seed": "7546",
"run": 0,
"new_fnr": 0.011399,
"seed_new_fnr_mean": {
"32859": 0.244905,
"64066": 0.17772,
"7546": 0.051468
},
"run_new_fnr": {
"0": 0.011399,
"1": 0.088601,
"2": 0.054404
},
"experiment": "promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 98,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt/Emotional_Manipulation__run0__from_file__scratch/round3"
},
"promptsleuth_aug__Instruction_Override": {
"dataset": "promptsleuth_aug",
"family": "Instruction_Override",
"seed": "32859",
"run": 1,
"new_fnr": 0.02487,
"seed_new_fnr_mean": {
"32859": 0.092746,
"64066": 0.35734,
"7546": 0.165976
},
"run_new_fnr": {
"0": 0.160104,
"1": 0.02487,
"2": 0.093264
},
"experiment": "promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 97,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt/Instruction_Override__run1__from_file__scratch/round3"
},
"promptsleuth_aug__Instruction_Wrapping": {
"dataset": "promptsleuth_aug",
"family": "Instruction_Wrapping",
"seed": "32859",
"run": 0,
"new_fnr": 0.055959,
"seed_new_fnr_mean": {
"32859": 0.094128,
"64066": 0.178756,
"7546": 0.145941
},
"run_new_fnr": {
"0": 0.055959,
"1": 0.062176,
"2": 0.164249
},
"experiment": "promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 98,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt/Instruction_Wrapping__run0__from_file__scratch/round3"
},
"promptsleuth_aug__Obfuscation": {
"dataset": "promptsleuth_aug",
"family": "Obfuscation",
"seed": "7546",
"run": 1,
"new_fnr": 0.015026,
"seed_new_fnr_mean": {
"32859": 0.036442,
"64066": 0.100518,
"7546": 0.035924
},
"run_new_fnr": {
"0": 0.052332,
"1": 0.015026,
"2": 0.040415
},
"experiment": "promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 99,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt/Obfuscation__run1__from_file__scratch/round3"
},
"promptsleuth_aug__Payload_Splitting": {
"dataset": "promptsleuth_aug",
"family": "Payload_Splitting",
"seed": "7546",
"run": 0,
"new_fnr": 0.03886,
"seed_new_fnr_mean": {
"32859": 0.188083,
"64066": 0.279275,
"7546": 0.094991
},
"run_new_fnr": {
"0": 0.03886,
"1": 0.061658,
"2": 0.184456
},
"experiment": "promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 97,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s7546_datasentinel_scratch_agent_ckpt/Payload_Splitting__run0__from_file__scratch/round3"
},
"promptsleuth_aug__Role_Impersonation": {
"dataset": "promptsleuth_aug",
"family": "Role_Impersonation",
"seed": "32859",
"run": 0,
"new_fnr": 0.096891,
"seed_new_fnr_mean": {
"32859": 0.109499,
"64066": 0.13247,
"7546": 0.250432
},
"run_new_fnr": {
"0": 0.096891,
"1": 0.127461,
"2": 0.104145
},
"experiment": "promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt",
"n_detected_train_samples": 89,
"source_path": "artifacts/runs/promptsleuth_aug_k1_ng63_s32859_datasentinel_scratch_agent_ckpt/Role_Impersonation__run0__from_file__scratch/round3"
}
}
} |