| { |
| "model_path": "/mnt/artifacts-datai/checkpoints/DEF-attackvla/best.pth", |
| "dataset": "/mnt/forge-data/datasets/lerobot--libero/extracted_frames/observation.images.image", |
| "n_params": 633138, |
| "suites": [ |
| "libero_long", |
| "libero_object", |
| "libero_spatial", |
| "libero_goal" |
| ], |
| "protected_vla_models": [ |
| { |
| "name": "openvla-7b", |
| "type": "openvla", |
| "params": 7000000000, |
| "path": "/mnt/forge-data/models/openvla--openvla-7b/" |
| }, |
| { |
| "name": "pi0fast-base", |
| "type": "pi0_fast", |
| "params": 3000000000, |
| "path": "/mnt/forge-data/models/lerobot--pi0fast-base/" |
| }, |
| { |
| "name": "pi05-base", |
| "type": "pi0", |
| "params": 3000000000, |
| "path": "/mnt/forge-data/models/lerobot--pi05_base/" |
| }, |
| { |
| "name": "smolvla-base", |
| "type": "smolvla", |
| "params": 400000000, |
| "path": "/mnt/forge-data/models/lerobot--smolvla_base/" |
| }, |
| { |
| "name": "bitvla", |
| "type": "bitvla_llava", |
| "params": 3000000000, |
| "path": "/mnt/forge-data/models/hongyuw--bitvla-bitsiglipL-224px-bf16/" |
| }, |
| { |
| "name": "xvla-pt", |
| "type": "xvla", |
| "params": 1500000000, |
| "path": "/mnt/forge-data/models/2toINF--X-VLA-Pt/" |
| } |
| ], |
| "overall_accuracy": 0.976, |
| "overall_tpr": 0.958, |
| "overall_fpr": 0.006, |
| "results": [ |
| { |
| "task_suite": "libero_long", |
| "attack_type": "upa", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 471, |
| "fp": 3, |
| "tn": 497, |
| "fn": 29, |
| "accuracy": 0.968, |
| "tpr": 0.942, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9454827085137367, |
| "latency_ms": 40.75266420841217 |
| }, |
| { |
| "task_suite": "libero_long", |
| "attack_type": "blue_cube", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 478, |
| "fp": 3, |
| "tn": 497, |
| "fn": 22, |
| "accuracy": 0.975, |
| "tpr": 0.956, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9599938094615936, |
| "latency_ms": 5.975306034088135 |
| }, |
| { |
| "task_suite": "libero_long", |
| "attack_type": "noise", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 480, |
| "fp": 3, |
| "tn": 497, |
| "fn": 20, |
| "accuracy": 0.977, |
| "tpr": 0.96, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9620153531432152, |
| "latency_ms": 7.7228546142578125 |
| }, |
| { |
| "task_suite": "libero_long", |
| "attack_type": "checkerboard", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 484, |
| "fp": 3, |
| "tn": 497, |
| "fn": 16, |
| "accuracy": 0.981, |
| "tpr": 0.968, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9678697884082794, |
| "latency_ms": 5.856528878211975 |
| }, |
| { |
| "task_suite": "libero_long", |
| "attack_type": "colored_square", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 471, |
| "fp": 3, |
| "tn": 497, |
| "fn": 29, |
| "accuracy": 0.968, |
| "tpr": 0.942, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9445305317640305, |
| "latency_ms": 6.648823618888855 |
| }, |
| { |
| "task_suite": "libero_object", |
| "attack_type": "upa", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 481, |
| "fp": 3, |
| "tn": 497, |
| "fn": 19, |
| "accuracy": 0.978, |
| "tpr": 0.962, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9614603295922279, |
| "latency_ms": 6.172060966491699 |
| }, |
| { |
| "task_suite": "libero_object", |
| "attack_type": "blue_cube", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 480, |
| "fp": 3, |
| "tn": 497, |
| "fn": 20, |
| "accuracy": 0.977, |
| "tpr": 0.96, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9611484184861183, |
| "latency_ms": 6.092682480812073 |
| }, |
| { |
| "task_suite": "libero_object", |
| "attack_type": "noise", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 478, |
| "fp": 3, |
| "tn": 497, |
| "fn": 22, |
| "accuracy": 0.975, |
| "tpr": 0.956, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9555910006165504, |
| "latency_ms": 6.759926676750183 |
| }, |
| { |
| "task_suite": "libero_object", |
| "attack_type": "checkerboard", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 480, |
| "fp": 3, |
| "tn": 497, |
| "fn": 20, |
| "accuracy": 0.977, |
| "tpr": 0.96, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9586973637342453, |
| "latency_ms": 8.294612169265747 |
| }, |
| { |
| "task_suite": "libero_object", |
| "attack_type": "colored_square", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 475, |
| "fp": 3, |
| "tn": 497, |
| "fn": 25, |
| "accuracy": 0.972, |
| "tpr": 0.95, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9521107524633408, |
| "latency_ms": 9.056717157363892 |
| }, |
| { |
| "task_suite": "libero_spatial", |
| "attack_type": "upa", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 484, |
| "fp": 3, |
| "tn": 497, |
| "fn": 16, |
| "accuracy": 0.981, |
| "tpr": 0.968, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9719937071204185, |
| "latency_ms": 7.455766201019287 |
| }, |
| { |
| "task_suite": "libero_spatial", |
| "attack_type": "blue_cube", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 476, |
| "fp": 3, |
| "tn": 497, |
| "fn": 24, |
| "accuracy": 0.973, |
| "tpr": 0.952, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9540987685322762, |
| "latency_ms": 9.753599762916565 |
| }, |
| { |
| "task_suite": "libero_spatial", |
| "attack_type": "noise", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 482, |
| "fp": 3, |
| "tn": 497, |
| "fn": 18, |
| "accuracy": 0.979, |
| "tpr": 0.964, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9642860442399979, |
| "latency_ms": 8.851349353790283 |
| }, |
| { |
| "task_suite": "libero_spatial", |
| "attack_type": "checkerboard", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 483, |
| "fp": 3, |
| "tn": 497, |
| "fn": 17, |
| "accuracy": 0.98, |
| "tpr": 0.966, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.963277131319046, |
| "latency_ms": 9.599000215530396 |
| }, |
| { |
| "task_suite": "libero_spatial", |
| "attack_type": "colored_square", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 480, |
| "fp": 3, |
| "tn": 497, |
| "fn": 20, |
| "accuracy": 0.977, |
| "tpr": 0.96, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9570039436221123, |
| "latency_ms": 8.878082036972046 |
| }, |
| { |
| "task_suite": "libero_goal", |
| "attack_type": "upa", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 481, |
| "fp": 3, |
| "tn": 497, |
| "fn": 19, |
| "accuracy": 0.978, |
| "tpr": 0.962, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9604860842227936, |
| "latency_ms": 8.672043681144714 |
| }, |
| { |
| "task_suite": "libero_goal", |
| "attack_type": "blue_cube", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 486, |
| "fp": 3, |
| "tn": 497, |
| "fn": 14, |
| "accuracy": 0.983, |
| "tpr": 0.972, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9728344157338142, |
| "latency_ms": 8.557334542274475 |
| }, |
| { |
| "task_suite": "libero_goal", |
| "attack_type": "noise", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 474, |
| "fp": 3, |
| "tn": 497, |
| "fn": 26, |
| "accuracy": 0.971, |
| "tpr": 0.948, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9491961672902107, |
| "latency_ms": 8.128583431243896 |
| }, |
| { |
| "task_suite": "libero_goal", |
| "attack_type": "checkerboard", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 478, |
| "fp": 3, |
| "tn": 497, |
| "fn": 22, |
| "accuracy": 0.975, |
| "tpr": 0.956, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9593797773122787, |
| "latency_ms": 8.788511157035828 |
| }, |
| { |
| "task_suite": "libero_goal", |
| "attack_type": "colored_square", |
| "n_clean": 500, |
| "n_attack": 500, |
| "tp": 478, |
| "fp": 3, |
| "tn": 497, |
| "fn": 22, |
| "accuracy": 0.975, |
| "tpr": 0.956, |
| "fpr": 0.006, |
| "mean_score_clean": 0.04928203532472253, |
| "mean_score_attack": 0.9626386165618896, |
| "latency_ms": 9.380266070365906 |
| } |
| ] |
| } |
