| { |
| "checkpoint": "runs/deepcad_latent/resnet18_transformer_v1_ddp2/best.pt", |
| "ids": "datasets/rescue_deepcad_latent/full_v0_len60_excluding_rescue_test/test_ids.txt", |
| "is_multimodal": false, |
| "text_root": null, |
| "retrieval_mode": "direct", |
| "compare_target": "true", |
| "param_tolerance": 3, |
| "num_samples": 5592, |
| "summary": { |
| "num_samples": 5592, |
| "cmd_token_acc": 0.8391398004815825, |
| "token_exact_acc": 0.5305168253604553, |
| "sequence_cmd_exact_rate": 0.675071530758226, |
| "sequence_exact_rate": 0.13125894134477825, |
| "mean_pred_len": 8.831008583690988, |
| "mean_gt_len": 9.420243204577968, |
| "mean_len_abs_error": 1.8159871244635193, |
| "first_divergence_missing_rate": 0.13125894134477825, |
| "acc_cmd": 0.8634451846250522, |
| "acc_param": 0.741478247165596, |
| "pred_solid_valid_rate": 0.8134835479256081, |
| "gt_solid_valid_rate": 0.9774678111587983, |
| "invalidity_ratio": 0.18651645207439194, |
| "cd_valid_count": 4928, |
| "cd_invalid_count": 664, |
| "cd_mean": 0.12981373850902178, |
| "cd_median": 0.034327806811930595, |
| "by_bucket": { |
| "len_01_06": { |
| "count": 3216, |
| "cmd_token_acc": 0.9740033988990587, |
| "token_exact_acc": 0.6861027642036345, |
| "sequence_exact_rate": 0.1843905472636816, |
| "mean_len_abs_error": 0.17319651741293532, |
| "acc_cmd": 0.9847740464344948, |
| "acc_param": 0.8519142227556925, |
| "pred_solid_valid_rate": 0.970771144278607, |
| "invalidity_ratio": 0.029228855721393, |
| "cd_mean": 0.11898658033303332, |
| "cd_median": 0.019705686610152172 |
| }, |
| "len_07_12": { |
| "count": 1315, |
| "cmd_token_acc": 0.7624808189216025, |
| "token_exact_acc": 0.4048386834849154, |
| "sequence_exact_rate": 0.0714828897338403, |
| "mean_len_abs_error": 1.7931558935361216, |
| "acc_cmd": 0.811738789305329, |
| "acc_param": 0.7042623472758387, |
| "pred_solid_valid_rate": 0.7262357414448669, |
| "invalidity_ratio": 0.2737642585551331, |
| "cd_mean": 0.14735227678297338, |
| "cd_median": 0.06654237348776594 |
| }, |
| "len_13_24": { |
| "count": 790, |
| "cmd_token_acc": 0.5792265561163498, |
| "token_exact_acc": 0.2490031207995815, |
| "sequence_exact_rate": 0.056962025316455694, |
| "mean_len_abs_error": 4.688607594936709, |
| "acc_cmd": 0.6215911324139712, |
| "acc_param": 0.5876221498371336, |
| "pred_solid_valid_rate": 0.4835443037974684, |
| "invalidity_ratio": 0.5164556962025316, |
| "cd_mean": 0.13941998955960228, |
| "cd_median": 0.057110440702972096 |
| }, |
| "len_25_plus": { |
| "count": 271, |
| "cmd_token_acc": 0.36835268346799277, |
| "token_exact_acc": 0.11464303882817876, |
| "sequence_exact_rate": 0.007380073800738007, |
| "mean_len_abs_error": 13.047970479704796, |
| "acc_cmd": 0.3795521643779019, |
| "acc_param": 0.4598471487360376, |
| "pred_solid_valid_rate": 0.33210332103321033, |
| "invalidity_ratio": 0.6678966789667897, |
| "cd_mean": 0.19125942949745178, |
| "cd_median": 0.11478811091909968 |
| } |
| }, |
| "worst_examples": [ |
| { |
| "sample_id": "0044/00449233_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 4, |
| "gt_len": 50, |
| "len_abs_error": 46, |
| "cmd_token_acc": 0.06, |
| "token_exact_acc": 0.04, |
| "sequence_exact": false, |
| "first_divergence_step": 2, |
| "cd": 0.1369912929596886, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0063/00638901_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 6, |
| "gt_len": 48, |
| "len_abs_error": 42, |
| "cmd_token_acc": 0.10416666666666667, |
| "token_exact_acc": 0.020833333333333332, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.8650329199152399, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0046/00461540_00002", |
| "bucket": "len_25_plus", |
| "pred_len": 14, |
| "gt_len": 56, |
| "len_abs_error": 42, |
| "cmd_token_acc": 0.125, |
| "token_exact_acc": 0.017857142857142856, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.01608317231504478, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0055/00550713_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 12, |
| "gt_len": 53, |
| "len_abs_error": 41, |
| "cmd_token_acc": 0.1509433962264151, |
| "token_exact_acc": 0.03773584905660377, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0057/00571486_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 11, |
| "gt_len": 52, |
| "len_abs_error": 41, |
| "cmd_token_acc": 0.15384615384615385, |
| "token_exact_acc": 0.038461538461538464, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.372333608571029, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0094/00947145_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 10, |
| "gt_len": 49, |
| "len_abs_error": 39, |
| "cmd_token_acc": 0.20408163265306123, |
| "token_exact_acc": 0.02040816326530612, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0043/00432451_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 3, |
| "gt_len": 39, |
| "len_abs_error": 36, |
| "cmd_token_acc": 0.02564102564102564, |
| "token_exact_acc": 0.02564102564102564, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.31735212582855776, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0054/00542394_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 11, |
| "gt_len": 46, |
| "len_abs_error": 35, |
| "cmd_token_acc": 0.15217391304347827, |
| "token_exact_acc": 0.021739130434782608, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0024/00244336_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 22, |
| "gt_len": 57, |
| "len_abs_error": 35, |
| "cmd_token_acc": 0.2982456140350877, |
| "token_exact_acc": 0.017543859649122806, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0065/00658127_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 17, |
| "gt_len": 51, |
| "len_abs_error": 34, |
| "cmd_token_acc": 0.21568627450980393, |
| "token_exact_acc": 0.0196078431372549, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.08852288401841818, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0098/00980287_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 18, |
| "gt_len": 50, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.1, |
| "token_exact_acc": 0.04, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0062/00620669_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 6, |
| "gt_len": 38, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.10526315789473684, |
| "token_exact_acc": 0.02631578947368421, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.13130972784010514, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0044/00446170_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 16, |
| "gt_len": 48, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.125, |
| "token_exact_acc": 0.020833333333333332, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0087/00873259_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 10, |
| "gt_len": 42, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.16666666666666666, |
| "token_exact_acc": 0.023809523809523808, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.074099495283291, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0033/00331543_00002", |
| "bucket": "len_25_plus", |
| "pred_len": 12, |
| "gt_len": 44, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.18181818181818182, |
| "token_exact_acc": 0.022727272727272728, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.3762435364793075, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0033/00339538_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 20, |
| "gt_len": 52, |
| "len_abs_error": 32, |
| "cmd_token_acc": 0.19230769230769232, |
| "token_exact_acc": 0.019230769230769232, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.03264505643227296, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0004/00043885_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 12, |
| "gt_len": 43, |
| "len_abs_error": 31, |
| "cmd_token_acc": 0.18604651162790697, |
| "token_exact_acc": 0.023255813953488372, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0048/00486655_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 19, |
| "gt_len": 50, |
| "len_abs_error": 31, |
| "cmd_token_acc": 0.2, |
| "token_exact_acc": 0.02, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": -1.0, |
| "pred_solid_valid": false |
| }, |
| { |
| "sample_id": "0013/00134148_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 6, |
| "gt_len": 36, |
| "len_abs_error": 30, |
| "cmd_token_acc": 0.16666666666666666, |
| "token_exact_acc": 0.08333333333333333, |
| "sequence_exact": false, |
| "first_divergence_step": 2, |
| "cd": 0.1194260991804219, |
| "pred_solid_valid": true |
| }, |
| { |
| "sample_id": "0017/00178584_00001", |
| "bucket": "len_25_plus", |
| "pred_len": 6, |
| "gt_len": 36, |
| "len_abs_error": 30, |
| "cmd_token_acc": 0.16666666666666666, |
| "token_exact_acc": 0.08333333333333333, |
| "sequence_exact": false, |
| "first_divergence_step": 1, |
| "cd": 0.7222374409952519, |
| "pred_solid_valid": true |
| } |
| ] |
| }, |
| "paper_metrics": { |
| "acc_cmd": 0.8634451846250522, |
| "acc_param": 0.741478247165596, |
| "invalidity_ratio": 0.18651645207439194, |
| "cd_mean": 0.12981373850902178, |
| "cd_median": 0.034327806811930595 |
| } |
| } |
