{
  "counts": {
    "train": 575,
    "valid": 124,
    "test": 124,
    "train_vocab": 39,
    "test_vocab": 35,
    "unseen_test_tools": [
      "mcp__cowork__request_cowork_directory"
    ]
  },
  "top_train_tools": [
    [
      "Bash",
      482
    ],
    [
      "Read",
      387
    ],
    [
      "Edit",
      218
    ],
    [
      "Write",
      97
    ],
    [
      "StructuredOutput",
      87
    ],
    [
      "mcp__ccd_session__mark_chapter",
      85
    ],
    [
      "run_shell_command",
      56
    ],
    [
      "replace",
      33
    ],
    [
      "ToolSearch",
      31
    ],
    [
      "read_file",
      21
    ],
    [
      "grep_search",
      18
    ],
    [
      "Agent",
      17
    ],
    [
      "TaskUpdate",
      16
    ],
    [
      "Grep",
      12
    ],
    [
      "Monitor",
      12
    ],
    [
      "TaskStop",
      10
    ],
    [
      "ScheduleWakeup",
      9
    ],
    [
      "mcp__ccd_session__spawn_task",
      8
    ],
    [
      "mcp__playwright__browser_navigate",
      6
    ],
    [
      "mcp__playwright__browser_take_screenshot",
      6
    ]
  ],
  "top_test_tools": [
    [
      "Bash",
      93
    ],
    [
      "Read",
      75
    ],
    [
      "Edit",
      55
    ],
    [
      "StructuredOutput",
      23
    ],
    [
      "mcp__ccd_session__mark_chapter",
      15
    ],
    [
      "Write",
      14
    ],
    [
      "run_shell_command",
      13
    ],
    [
      "read_file",
      9
    ],
    [
      "Agent",
      7
    ],
    [
      "ToolSearch",
      7
    ],
    [
      "replace",
      5
    ],
    [
      "mcp__playwright__browser_navigate",
      5
    ],
    [
      "grep_search",
      4
    ],
    [
      "mcp__playwright__browser_take_screenshot",
      4
    ],
    [
      "Monitor",
      4
    ],
    [
      "mcp__playwright__browser_resize",
      3
    ],
    [
      "mcp__playwright__browser_evaluate",
      3
    ],
    [
      "list_directory",
      3
    ],
    [
      "TaskUpdate",
      3
    ],
    [
      "TaskStop",
      3
    ]
  ],
  "floor_by_k": [
    {
      "k": 1,
      "tools": [
        "Bash"
      ],
      "jaccard": 0.31837557603686634,
      "exact": 0.12096774193548387,
      "precision": 0.75,
      "recall": 0.31837557603686634,
      "top1": 0.75
    },
    {
      "k": 2,
      "tools": [
        "Bash",
        "Read"
      ],
      "jaccard": 0.43137480798771116,
      "exact": 0.08870967741935484,
      "precision": 0.6774193548387096,
      "recall": 0.5031490015360983,
      "top1": 0.75
    },
    {
      "k": 3,
      "tools": [
        "Bash",
        "Read",
        "Edit"
      ],
      "jaccard": 0.4740975422427035,
      "exact": 0.11290322580645161,
      "precision": 0.5994623655913978,
      "recall": 0.6307987711213517,
      "top1": 0.75
    },
    {
      "k": 4,
      "tools": [
        "Bash",
        "Read",
        "Edit",
        "Write"
      ],
      "jaccard": 0.4057027649769585,
      "exact": 0.016129032258064516,
      "precision": 0.4778225806451613,
      "recall": 0.6584965437788018,
      "top1": 0.75
    },
    {
      "k": 5,
      "tools": [
        "Bash",
        "Read",
        "Edit",
        "Write",
        "StructuredOutput"
      ],
      "jaccard": 0.3831989247311828,
      "exact": 0.0,
      "precision": 0.41935483870967744,
      "recall": 0.714544930875576,
      "top1": 0.75
    },
    {
      "k": 6,
      "tools": [
        "Bash",
        "Read",
        "Edit",
        "Write",
        "StructuredOutput",
        "mcp__ccd_session__mark_chapter"
      ],
      "jaccard": 0.3497064539403249,
      "exact": 0.0,
      "precision": 0.3696236559139785,
      "recall": 0.7408986175115208,
      "top1": 0.75
    },
    {
      "k": 7,
      "tools": [
        "Bash",
        "Read",
        "Edit",
        "Write",
        "StructuredOutput",
        "mcp__ccd_session__mark_chapter",
        "run_shell_command"
      ],
      "jaccard": 0.3145152562491272,
      "exact": 0.0,
      "precision": 0.3317972350230414,
      "recall": 0.8076996927803379,
      "top1": 0.75
    },
    {
      "k": 8,
      "tools": [
        "Bash",
        "Read",
        "Edit",
        "Write",
        "StructuredOutput",
        "mcp__ccd_session__mark_chapter",
        "run_shell_command",
        "replace"
      ],
      "jaccard": 0.2813046720304785,
      "exact": 0.0,
      "precision": 0.29536290322580644,
      "recall": 0.8220814132104455,
      "top1": 0.75
    }
  ],
  "summary": {
    "diffusion_zero_shot": {
      "jaccard": [
        0.07278545826932924,
        0.044809267793138756,
        0.10379544290834614
      ],
      "top1": 0.0967741935483871,
      "exact": 0.008064516129032258,
      "precision": 0.12916666666666665,
      "recall": 0.09401881720430108,
      "pred_n": 1.7741935483870968,
      "missing": 2.685483870967742,
      "extra": 1.4919354838709677
    },
    "diffusion_lora": {
      "jaccard": [
        0.4472606246799795,
        0.3933147721454173,
        0.5016513056835638
      ],
      "top1": 0.75,
      "exact": 0.10483870967741936,
      "precision": 0.5658602150537635,
      "recall": 0.6065956221198157,
      "pred_n": 2.879032258064516,
      "missing": 1.2661290322580645,
      "extra": 1.1774193548387097
    },
    "ar_zero_shot": {
      "jaccard": [
        0.07011919532080822,
        0.04485037811650715,
        0.0987386269644334
      ],
      "top1": 0.11290322580645161,
      "exact": 0.008064516129032258,
      "precision": 0.12575475599669148,
      "recall": 0.11344086021505376,
      "pred_n": 2.379032258064516,
      "missing": 2.6693548387096775,
      "extra": 2.0806451612903225
    },
    "ar_lora": {
      "jaccard": [
        0.4925115207373272,
        0.4320468509984639,
        0.5533218125960062
      ],
      "top1": 0.782258064516129,
      "exact": 0.22580645161290322,
      "precision": 0.7553763440860215,
      "recall": 0.5109927035330262,
      "pred_n": 1.75,
      "missing": 1.5725806451612903,
      "extra": 0.3548387096774194
    },
    "qwen35_zero_shot": {
      "jaccard": [
        0.1381896637945025,
        0.10233294930875576,
        0.17591047792660694
      ],
      "top1": 0.25,
      "exact": 0.008064516129032258,
      "precision": 0.21700372607630672,
      "recall": 0.21192396313364056,
      "pred_n": 2.4274193548387095,
      "missing": 2.2903225806451615,
      "extra": 1.75
    },
    "qwen27_zero_shot": {
      "jaccard": [
        0.19682306940371458,
        0.15791788856304986,
        0.23646662477307637
      ],
      "top1": 0.33064516129032256,
      "exact": 0.008064516129032258,
      "precision": 0.322599846390169,
      "recall": 0.279531490015361,
      "pred_n": 2.282258064516129,
      "missing": 2.1048387096774195,
      "extra": 1.4193548387096775
    },
    "diffusion_aw005": {
      "jaccard": [
        0.4254032258064516,
        0.37307987711213514,
        0.4777073732718894
      ],
      "top1": 0.7580645161290323,
      "exact": 0.08870967741935484,
      "precision": 0.5090693804403482,
      "recall": 0.6351478494623657,
      "pred_n": 3.467741935483871,
      "missing": 1.1612903225806452,
      "extra": 1.6612903225806452
    },
    "diffusion_aw025": {
      "jaccard": [
        0.43988491347362313,
        0.38916963412931155,
        0.4930299539170507
      ],
      "top1": 0.75,
      "exact": 0.08064516129032258,
      "precision": 0.542388928276025,
      "recall": 0.6325940860215054,
      "pred_n": 3.411290322580645,
      "missing": 1.185483870967742,
      "extra": 1.6290322580645162
    },
    "diffusion_r32": {
      "jaccard": [
        0.44411802355350743,
        0.3898361495135689,
        0.49879032258064515
      ],
      "top1": 0.75,
      "exact": 0.12096774193548387,
      "precision": 0.668010752688172,
      "recall": 0.5104646697388633,
      "pred_n": 2.064516129032258,
      "missing": 1.532258064516129,
      "extra": 0.6290322580645161
    },
    "diffusion_expanded": {
      "jaccard": [
        0.4335445468509984,
        0.3826612903225806,
        0.48529185867895547
      ],
      "top1": 0.7580645161290323,
      "exact": 0.08064516129032258,
      "precision": 0.6024193548387097,
      "recall": 0.5542338709677419,
      "pred_n": 2.475806451612903,
      "missing": 1.435483870967742,
      "extra": 0.9435483870967742
    },
    "floor_top3": {
      "jaccard": [
        0.4740975422427035,
        0.4201996927803379,
        0.5277265745007681
      ],
      "top1": 0.75,
      "exact": 0.11290322580645161,
      "precision": 0.5994623655913978,
      "recall": 0.6307987711213517,
      "pred_n": 3.0,
      "missing": 1.1693548387096775,
      "extra": 1.2016129032258065
    },
    "tfidf_1nn": {
      "jaccard": [
        0.500444828469022,
        0.4454237071172555,
        0.5572644649257552
      ],
      "top1": 0.7661290322580645,
      "exact": 0.1532258064516129,
      "precision": 0.6579781105990783,
      "recall": 0.639276113671275,
      "pred_n": 2.814516129032258,
      "missing": 1.1451612903225807,
      "extra": 0.9919354838709677
    },
    "tfidf_1nn_top3vote": {
      "jaccard": [
        0.48733678955453147,
        0.4321716589861751,
        0.5448636712749616
      ],
      "top1": 0.7661290322580645,
      "exact": 0.1693548387096774,
      "precision": 0.6760752688172043,
      "recall": 0.59176267281106,
      "pred_n": 2.2903225806451615,
      "missing": 1.3548387096774193,
      "extra": 0.6774193548387096
    },
    "tfidf_3nn": {
      "jaccard": [
        0.5050179211469534,
        0.45102406554019453,
        0.5590437788018433
      ],
      "top1": 0.7903225806451613,
      "exact": 0.1693548387096774,
      "precision": 0.6533602150537634,
      "recall": 0.6464285714285715,
      "pred_n": 2.7419354838709675,
      "missing": 1.1290322580645162,
      "extra": 0.9032258064516129
    },
    "tfidf_3nn_top3vote": {
      "jaccard": [
        0.510515873015873,
        0.45830773169482847,
        0.5629704301075269
      ],
      "top1": 0.7903225806451613,
      "exact": 0.16129032258064516,
      "precision": 0.6370967741935484,
      "recall": 0.6839093701996928,
      "pred_n": 2.8548387096774195,
      "missing": 1.1290322580645162,
      "extra": 1.0161290322580645
    },
    "tfidf_5nn": {
      "jaccard": [
        0.500502432155658,
        0.4513536866359447,
        0.5495967741935485
      ],
      "top1": 0.7903225806451613,
      "exact": 0.12096774193548387,
      "precision": 0.637768817204301,
      "recall": 0.6762576804915514,
      "pred_n": 2.7983870967741935,
      "missing": 1.1048387096774193,
      "extra": 0.9354838709677419
    },
    "tfidf_5nn_top3vote": {
      "jaccard": [
        0.5122855862775217,
        0.45913978494623653,
        0.5648841525857655
      ],
      "top1": 0.7903225806451613,
      "exact": 0.1774193548387097,
      "precision": 0.6303763440860215,
      "recall": 0.6996351766513057,
      "pred_n": 2.935483870967742,
      "missing": 1.096774193548387,
      "extra": 1.064516129032258
    },
    "train_labelset_oracle": {
      "jaccard": [
        0.9444124423963134,
        0.9213997695852535,
        0.9657834101382489
      ],
      "top1": 0.9354838709677419,
      "exact": 0.8145161290322581,
      "precision": 0.9692492319508448,
      "recall": 0.9717069892473118,
      "pred_n": 3.0,
      "missing": 0.10483870967741936,
      "extra": 0.13709677419354838
    },
    "data_ceiling": {
      "jaccard": [
        0.9959677419354839,
        0.9879032258064516,
        1.0
      ],
      "top1": 1.0,
      "exact": 0.9919354838709677,
      "precision": 1.0,
      "recall": 0.9959677419354839,
      "pred_n": 2.9596774193548385,
      "missing": 0.008064516129032258,
      "extra": 0.0
    }
  },
  "paired_vs_floor": {
    "diffusion_zero_shot": [
      -0.4013120839733743,
      -0.4626536098310292,
      -0.338773681515617
    ],
    "diffusion_lora": [
      -0.02683691756272401,
      -0.06750192012288786,
      0.012679211469534053
    ],
    "ar_zero_shot": [
      -0.4039783469218953,
      -0.46681436842727164,
      -0.33903609831029186
    ],
    "ar_lora": [
      0.01841397849462366,
      -0.03867127496159754,
      0.07795698924731184
    ],
    "qwen35_zero_shot": [
      -0.33590787844820097,
      -0.39298195084485404,
      -0.27843161814936007
    ],
    "qwen27_zero_shot": [
      -0.277274472838989,
      -0.3321507121910348,
      -0.2189315388912163
    ],
    "diffusion_aw005": [
      -0.04869431643625192,
      -0.0894489247311828,
      -0.008787762416794671
    ],
    "diffusion_aw025": [
      -0.034212628769080385,
      -0.06047467317628608,
      -0.007286866359447002
    ],
    "diffusion_r32": [
      -0.029979518689196107,
      -0.08051075268817205,
      0.021873399897593446
    ],
    "diffusion_expanded": [
      -0.040552995391705066,
      -0.07831221198156682,
      -0.0032258064516129024
    ],
    "tfidf_1nn": [
      0.026347286226318482,
      -0.03129800307219662,
      0.08436059907834101
    ],
    "tfidf_1nn_top3vote": [
      0.013239247311827957,
      -0.04418202764976958,
      0.07288786482334869
    ],
    "tfidf_3nn": [
      0.030920378904249875,
      -0.0268721198156682,
      0.0878200204813108
    ],
    "tfidf_3nn_top3vote": [
      0.036418330773169486,
      -0.010615079365079364,
      0.08473502304147465
    ],
    "tfidf_5nn": [
      0.02640488991295443,
      -0.026379288274449565,
      0.07841461853558628
    ],
    "tfidf_5nn_top3vote": [
      0.03818804403481822,
      -0.005081925243215566,
      0.08151881720430107
    ],
    "train_labelset_oracle": [
      0.4703149001536099,
      0.4196524577572965,
      0.5210253456221199
    ],
    "data_ceiling": [
      0.5218701996927804,
      0.468394777265745,
      0.5749135944700461
    ]
  },
  "strata_counts": {
    "freq": {
      "head_only": 44,
      "one_tail": 48,
      "multi_tail": 32
    },
    "card": {
      "1-2": 50,
      "3": 35,
      "4+": 39
    },
    "length": {
      "short": 42,
      "medium": 41,
      "long": 41
    }
  }
}