| { |
| "model": "guardspec-classifier-v3b", |
| "labels": [ |
| "benign", |
| "direct_injection", |
| "indirect_injection", |
| "jailbreak", |
| "prompt_extraction", |
| "data_exfiltration", |
| "tool_poisoning", |
| "obfuscation" |
| ], |
| "output": "logits; apply sigmoid per label (multi-label head)", |
| "max_len": 256, |
| "risk_score": "1-p_benign", |
| "risk_score_doc": "risk = 1 - sigmoid(logit_benign)", |
| "risk_threshold": 0.19113081693649292, |
| "risk_threshold_target_benign_fpr": 0.01, |
| "risk_threshold_review": 0.004999309778213501, |
| "risk_threshold_review_target_benign_fpr": 0.05, |
| "validation": { |
| "n": 7572, |
| "n_attack": 1699, |
| "choice": { |
| "1-p_benign": { |
| "val_auc_int8": 0.9964923127124689, |
| "val_auc_fp32": 0.9970819966312653, |
| "val_tpr_at_fpr": 0.9752795762213067, |
| "val_fpr": 0.009875702366763153, |
| "threshold": 0.19113081693649292 |
| }, |
| "max_attack": { |
| "val_auc_int8": 0.9931538939733482, |
| "val_auc_fp32": 0.9932332667917858, |
| "val_tpr_at_fpr": 0.9746909947027663, |
| "val_fpr": 0.009875702366763153, |
| "threshold": 0.16231968253850937 |
| } |
| }, |
| "by_lang_at_threshold": { |
| "val": { |
| "tpr": 0.9858247422680413, |
| "benign_fpr": 0.006717557251908397, |
| "n_attack": 1552, |
| "n_benign": 3275 |
| }, |
| "val_hard": { |
| "tpr": 0.8666666666666667, |
| "benign_fpr": 0.03571428571428571, |
| "n_attack": 45, |
| "n_benign": 784 |
| }, |
| "val_v3": { |
| "tpr": 0.967741935483871, |
| "benign_fpr": 0.0041928721174004195, |
| "n_attack": 31, |
| "n_benign": 477 |
| }, |
| "val_vi_variants": { |
| "tpr": 0.8169014084507042, |
| "benign_fpr": 0.004487658937920718, |
| "n_attack": 71, |
| "n_benign": 1337 |
| }, |
| "en": { |
| "tpr": 0.9775280898876404, |
| "benign_fpr": 0.009337068160597572, |
| "auc": 0.9959760121276975 |
| }, |
| "vi": { |
| "tpr": 0.9736575481256332, |
| "benign_fpr": 0.010184937014205307, |
| "auc": 0.9975380835340802 |
| } |
| } |
| }, |
| "label_thresholds": { |
| "direct_injection": 0.7300132513046265, |
| "indirect_injection": 0.20383548736572266, |
| "jailbreak": 0.2394111305475235, |
| "prompt_extraction": 0.6600154638290405, |
| "data_exfiltration": 0.23522545397281647, |
| "tool_poisoning": 0.8506209850311279, |
| "obfuscation": 0.5216465592384338 |
| }, |
| "label_threshold_method": "max F1 on the validation split, one-vs-rest", |
| "label_validation": { |
| "direct_injection": { |
| "threshold": 0.7300132513046265, |
| "val_f1": 0.9216683621566634, |
| "val_precision": 0.937888198757764, |
| "val_recall": 0.906, |
| "val_auc": 0.9927010746606335 |
| }, |
| "indirect_injection": { |
| "threshold": 0.20383548736572266, |
| "val_f1": 0.9881305637982196, |
| "val_precision": 0.985207100591716, |
| "val_recall": 0.9910714285714286, |
| "val_auc": 0.9990906084656085 |
| }, |
| "jailbreak": { |
| "threshold": 0.2394111305475235, |
| "val_f1": 0.8929159802306426, |
| "val_precision": 0.9377162629757786, |
| "val_recall": 0.8522012578616353, |
| "val_auc": 0.9798317735779697 |
| }, |
| "prompt_extraction": { |
| "threshold": 0.6600154638290405, |
| "val_f1": 0.9232839838492597, |
| "val_precision": 0.9607843137254902, |
| "val_recall": 0.8886010362694301, |
| "val_auc": 0.9851513954162454 |
| }, |
| "data_exfiltration": { |
| "threshold": 0.23522545397281647, |
| "val_f1": 0.9649122807017544, |
| "val_precision": 0.9821428571428571, |
| "val_recall": 0.9482758620689655, |
| "val_auc": 0.997447047517216 |
| }, |
| "tool_poisoning": { |
| "threshold": 0.8506209850311279, |
| "val_f1": 0.9898989898989898, |
| "val_precision": 0.9932432432432432, |
| "val_recall": 0.9865771812080537, |
| "val_auc": 0.9998607628927684 |
| }, |
| "obfuscation": { |
| "threshold": 0.5216465592384338, |
| "val_f1": 0.8615384615384616, |
| "val_precision": 0.9032258064516129, |
| "val_recall": 0.8235294117647058, |
| "val_auc": 0.9870357245977373 |
| } |
| }, |
| "calibrated_on": "validation splits runs/data-v3b-A/val.jsonl, runs/data-v3b-A/val_hard.jsonl, runs/data-v3b-A/val_vi_variants.jsonl, runs/data-v3b-A/val_v3.jsonl, INT8 model (v3b arm A build: train/runs/data-v3b-A, includes owner-supplied and otpkyc validation rows)" |
| } |
|
|
