{
  "checkpoint": "runs/deepcad_latent/resnet18_transformer_v1_ddp2/best.pt",
  "ids": "datasets/rescue_deepcad_latent/full_v0_len60_excluding_rescue_test/test_ids.txt",
  "is_multimodal": false,
  "text_root": null,
  "retrieval_mode": "direct",
  "compare_target": "true",
  "param_tolerance": 3,
  "num_samples": 5592,
  "summary": {
    "num_samples": 5592,
    "cmd_token_acc": 0.8391398004815825,
    "token_exact_acc": 0.5305168253604553,
    "sequence_cmd_exact_rate": 0.675071530758226,
    "sequence_exact_rate": 0.13125894134477825,
    "mean_pred_len": 8.831008583690988,
    "mean_gt_len": 9.420243204577968,
    "mean_len_abs_error": 1.8159871244635193,
    "first_divergence_missing_rate": 0.13125894134477825,
    "acc_cmd": 0.8634451846250522,
    "acc_param": 0.741478247165596,
    "pred_solid_valid_rate": 0.8134835479256081,
    "gt_solid_valid_rate": 0.9774678111587983,
    "invalidity_ratio": 0.18651645207439194,
    "cd_valid_count": 4928,
    "cd_invalid_count": 664,
    "cd_mean": 0.12981373850902178,
    "cd_median": 0.034327806811930595,
    "by_bucket": {
      "len_01_06": {
        "count": 3216,
        "cmd_token_acc": 0.9740033988990587,
        "token_exact_acc": 0.6861027642036345,
        "sequence_exact_rate": 0.1843905472636816,
        "mean_len_abs_error": 0.17319651741293532,
        "acc_cmd": 0.9847740464344948,
        "acc_param": 0.8519142227556925,
        "pred_solid_valid_rate": 0.970771144278607,
        "invalidity_ratio": 0.029228855721393,
        "cd_mean": 0.11898658033303332,
        "cd_median": 0.019705686610152172
      },
      "len_07_12": {
        "count": 1315,
        "cmd_token_acc": 0.7624808189216025,
        "token_exact_acc": 0.4048386834849154,
        "sequence_exact_rate": 0.0714828897338403,
        "mean_len_abs_error": 1.7931558935361216,
        "acc_cmd": 0.811738789305329,
        "acc_param": 0.7042623472758387,
        "pred_solid_valid_rate": 0.7262357414448669,
        "invalidity_ratio": 0.2737642585551331,
        "cd_mean": 0.14735227678297338,
        "cd_median": 0.06654237348776594
      },
      "len_13_24": {
        "count": 790,
        "cmd_token_acc": 0.5792265561163498,
        "token_exact_acc": 0.2490031207995815,
        "sequence_exact_rate": 0.056962025316455694,
        "mean_len_abs_error": 4.688607594936709,
        "acc_cmd": 0.6215911324139712,
        "acc_param": 0.5876221498371336,
        "pred_solid_valid_rate": 0.4835443037974684,
        "invalidity_ratio": 0.5164556962025316,
        "cd_mean": 0.13941998955960228,
        "cd_median": 0.057110440702972096
      },
      "len_25_plus": {
        "count": 271,
        "cmd_token_acc": 0.36835268346799277,
        "token_exact_acc": 0.11464303882817876,
        "sequence_exact_rate": 0.007380073800738007,
        "mean_len_abs_error": 13.047970479704796,
        "acc_cmd": 0.3795521643779019,
        "acc_param": 0.4598471487360376,
        "pred_solid_valid_rate": 0.33210332103321033,
        "invalidity_ratio": 0.6678966789667897,
        "cd_mean": 0.19125942949745178,
        "cd_median": 0.11478811091909968
      }
    },
    "worst_examples": [
      {
        "sample_id": "0044/00449233_00001",
        "bucket": "len_25_plus",
        "pred_len": 4,
        "gt_len": 50,
        "len_abs_error": 46,
        "cmd_token_acc": 0.06,
        "token_exact_acc": 0.04,
        "sequence_exact": false,
        "first_divergence_step": 2,
        "cd": 0.1369912929596886,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0063/00638901_00001",
        "bucket": "len_25_plus",
        "pred_len": 6,
        "gt_len": 48,
        "len_abs_error": 42,
        "cmd_token_acc": 0.10416666666666667,
        "token_exact_acc": 0.020833333333333332,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.8650329199152399,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0046/00461540_00002",
        "bucket": "len_25_plus",
        "pred_len": 14,
        "gt_len": 56,
        "len_abs_error": 42,
        "cmd_token_acc": 0.125,
        "token_exact_acc": 0.017857142857142856,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.01608317231504478,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0055/00550713_00001",
        "bucket": "len_25_plus",
        "pred_len": 12,
        "gt_len": 53,
        "len_abs_error": 41,
        "cmd_token_acc": 0.1509433962264151,
        "token_exact_acc": 0.03773584905660377,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0057/00571486_00001",
        "bucket": "len_25_plus",
        "pred_len": 11,
        "gt_len": 52,
        "len_abs_error": 41,
        "cmd_token_acc": 0.15384615384615385,
        "token_exact_acc": 0.038461538461538464,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.372333608571029,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0094/00947145_00001",
        "bucket": "len_25_plus",
        "pred_len": 10,
        "gt_len": 49,
        "len_abs_error": 39,
        "cmd_token_acc": 0.20408163265306123,
        "token_exact_acc": 0.02040816326530612,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0043/00432451_00001",
        "bucket": "len_25_plus",
        "pred_len": 3,
        "gt_len": 39,
        "len_abs_error": 36,
        "cmd_token_acc": 0.02564102564102564,
        "token_exact_acc": 0.02564102564102564,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.31735212582855776,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0054/00542394_00001",
        "bucket": "len_25_plus",
        "pred_len": 11,
        "gt_len": 46,
        "len_abs_error": 35,
        "cmd_token_acc": 0.15217391304347827,
        "token_exact_acc": 0.021739130434782608,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0024/00244336_00001",
        "bucket": "len_25_plus",
        "pred_len": 22,
        "gt_len": 57,
        "len_abs_error": 35,
        "cmd_token_acc": 0.2982456140350877,
        "token_exact_acc": 0.017543859649122806,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0065/00658127_00001",
        "bucket": "len_25_plus",
        "pred_len": 17,
        "gt_len": 51,
        "len_abs_error": 34,
        "cmd_token_acc": 0.21568627450980393,
        "token_exact_acc": 0.0196078431372549,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.08852288401841818,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0098/00980287_00001",
        "bucket": "len_25_plus",
        "pred_len": 18,
        "gt_len": 50,
        "len_abs_error": 32,
        "cmd_token_acc": 0.1,
        "token_exact_acc": 0.04,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0062/00620669_00001",
        "bucket": "len_25_plus",
        "pred_len": 6,
        "gt_len": 38,
        "len_abs_error": 32,
        "cmd_token_acc": 0.10526315789473684,
        "token_exact_acc": 0.02631578947368421,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.13130972784010514,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0044/00446170_00001",
        "bucket": "len_25_plus",
        "pred_len": 16,
        "gt_len": 48,
        "len_abs_error": 32,
        "cmd_token_acc": 0.125,
        "token_exact_acc": 0.020833333333333332,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0087/00873259_00001",
        "bucket": "len_25_plus",
        "pred_len": 10,
        "gt_len": 42,
        "len_abs_error": 32,
        "cmd_token_acc": 0.16666666666666666,
        "token_exact_acc": 0.023809523809523808,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.074099495283291,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0033/00331543_00002",
        "bucket": "len_25_plus",
        "pred_len": 12,
        "gt_len": 44,
        "len_abs_error": 32,
        "cmd_token_acc": 0.18181818181818182,
        "token_exact_acc": 0.022727272727272728,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.3762435364793075,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0033/00339538_00001",
        "bucket": "len_25_plus",
        "pred_len": 20,
        "gt_len": 52,
        "len_abs_error": 32,
        "cmd_token_acc": 0.19230769230769232,
        "token_exact_acc": 0.019230769230769232,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.03264505643227296,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0004/00043885_00001",
        "bucket": "len_25_plus",
        "pred_len": 12,
        "gt_len": 43,
        "len_abs_error": 31,
        "cmd_token_acc": 0.18604651162790697,
        "token_exact_acc": 0.023255813953488372,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0048/00486655_00001",
        "bucket": "len_25_plus",
        "pred_len": 19,
        "gt_len": 50,
        "len_abs_error": 31,
        "cmd_token_acc": 0.2,
        "token_exact_acc": 0.02,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": -1.0,
        "pred_solid_valid": false
      },
      {
        "sample_id": "0013/00134148_00001",
        "bucket": "len_25_plus",
        "pred_len": 6,
        "gt_len": 36,
        "len_abs_error": 30,
        "cmd_token_acc": 0.16666666666666666,
        "token_exact_acc": 0.08333333333333333,
        "sequence_exact": false,
        "first_divergence_step": 2,
        "cd": 0.1194260991804219,
        "pred_solid_valid": true
      },
      {
        "sample_id": "0017/00178584_00001",
        "bucket": "len_25_plus",
        "pred_len": 6,
        "gt_len": 36,
        "len_abs_error": 30,
        "cmd_token_acc": 0.16666666666666666,
        "token_exact_acc": 0.08333333333333333,
        "sequence_exact": false,
        "first_divergence_step": 1,
        "cd": 0.7222374409952519,
        "pred_solid_valid": true
      }
    ]
  },
  "paper_metrics": {
    "acc_cmd": 0.8634451846250522,
    "acc_param": 0.741478247165596,
    "invalidity_ratio": 0.18651645207439194,
    "cd_mean": 0.12981373850902178,
    "cd_median": 0.034327806811930595
  }
}