| { |
| "counts": { |
| "train": 575, |
| "valid": 124, |
| "test": 124, |
| "train_vocab": 39, |
| "test_vocab": 35, |
| "unseen_test_tools": [ |
| "mcp__cowork__request_cowork_directory" |
| ] |
| }, |
| "top_train_tools": [ |
| [ |
| "Bash", |
| 482 |
| ], |
| [ |
| "Read", |
| 387 |
| ], |
| [ |
| "Edit", |
| 218 |
| ], |
| [ |
| "Write", |
| 97 |
| ], |
| [ |
| "StructuredOutput", |
| 87 |
| ], |
| [ |
| "mcp__ccd_session__mark_chapter", |
| 85 |
| ], |
| [ |
| "run_shell_command", |
| 56 |
| ], |
| [ |
| "replace", |
| 33 |
| ], |
| [ |
| "ToolSearch", |
| 31 |
| ], |
| [ |
| "read_file", |
| 21 |
| ], |
| [ |
| "grep_search", |
| 18 |
| ], |
| [ |
| "Agent", |
| 17 |
| ], |
| [ |
| "TaskUpdate", |
| 16 |
| ], |
| [ |
| "Grep", |
| 12 |
| ], |
| [ |
| "Monitor", |
| 12 |
| ], |
| [ |
| "TaskStop", |
| 10 |
| ], |
| [ |
| "ScheduleWakeup", |
| 9 |
| ], |
| [ |
| "mcp__ccd_session__spawn_task", |
| 8 |
| ], |
| [ |
| "mcp__playwright__browser_navigate", |
| 6 |
| ], |
| [ |
| "mcp__playwright__browser_take_screenshot", |
| 6 |
| ] |
| ], |
| "top_test_tools": [ |
| [ |
| "Bash", |
| 93 |
| ], |
| [ |
| "Read", |
| 75 |
| ], |
| [ |
| "Edit", |
| 55 |
| ], |
| [ |
| "StructuredOutput", |
| 23 |
| ], |
| [ |
| "mcp__ccd_session__mark_chapter", |
| 15 |
| ], |
| [ |
| "Write", |
| 14 |
| ], |
| [ |
| "run_shell_command", |
| 13 |
| ], |
| [ |
| "read_file", |
| 9 |
| ], |
| [ |
| "Agent", |
| 7 |
| ], |
| [ |
| "ToolSearch", |
| 7 |
| ], |
| [ |
| "replace", |
| 5 |
| ], |
| [ |
| "mcp__playwright__browser_navigate", |
| 5 |
| ], |
| [ |
| "grep_search", |
| 4 |
| ], |
| [ |
| "mcp__playwright__browser_take_screenshot", |
| 4 |
| ], |
| [ |
| "Monitor", |
| 4 |
| ], |
| [ |
| "mcp__playwright__browser_resize", |
| 3 |
| ], |
| [ |
| "mcp__playwright__browser_evaluate", |
| 3 |
| ], |
| [ |
| "list_directory", |
| 3 |
| ], |
| [ |
| "TaskUpdate", |
| 3 |
| ], |
| [ |
| "TaskStop", |
| 3 |
| ] |
| ], |
| "floor_by_k": [ |
| { |
| "k": 1, |
| "tools": [ |
| "Bash" |
| ], |
| "jaccard": 0.31837557603686634, |
| "exact": 0.12096774193548387, |
| "precision": 0.75, |
| "recall": 0.31837557603686634, |
| "top1": 0.75 |
| }, |
| { |
| "k": 2, |
| "tools": [ |
| "Bash", |
| "Read" |
| ], |
| "jaccard": 0.43137480798771116, |
| "exact": 0.08870967741935484, |
| "precision": 0.6774193548387096, |
| "recall": 0.5031490015360983, |
| "top1": 0.75 |
| }, |
| { |
| "k": 3, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit" |
| ], |
| "jaccard": 0.4740975422427035, |
| "exact": 0.11290322580645161, |
| "precision": 0.5994623655913978, |
| "recall": 0.6307987711213517, |
| "top1": 0.75 |
| }, |
| { |
| "k": 4, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit", |
| "Write" |
| ], |
| "jaccard": 0.4057027649769585, |
| "exact": 0.016129032258064516, |
| "precision": 0.4778225806451613, |
| "recall": 0.6584965437788018, |
| "top1": 0.75 |
| }, |
| { |
| "k": 5, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit", |
| "Write", |
| "StructuredOutput" |
| ], |
| "jaccard": 0.3831989247311828, |
| "exact": 0.0, |
| "precision": 0.41935483870967744, |
| "recall": 0.714544930875576, |
| "top1": 0.75 |
| }, |
| { |
| "k": 6, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit", |
| "Write", |
| "StructuredOutput", |
| "mcp__ccd_session__mark_chapter" |
| ], |
| "jaccard": 0.3497064539403249, |
| "exact": 0.0, |
| "precision": 0.3696236559139785, |
| "recall": 0.7408986175115208, |
| "top1": 0.75 |
| }, |
| { |
| "k": 7, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit", |
| "Write", |
| "StructuredOutput", |
| "mcp__ccd_session__mark_chapter", |
| "run_shell_command" |
| ], |
| "jaccard": 0.3145152562491272, |
| "exact": 0.0, |
| "precision": 0.3317972350230414, |
| "recall": 0.8076996927803379, |
| "top1": 0.75 |
| }, |
| { |
| "k": 8, |
| "tools": [ |
| "Bash", |
| "Read", |
| "Edit", |
| "Write", |
| "StructuredOutput", |
| "mcp__ccd_session__mark_chapter", |
| "run_shell_command", |
| "replace" |
| ], |
| "jaccard": 0.2813046720304785, |
| "exact": 0.0, |
| "precision": 0.29536290322580644, |
| "recall": 0.8220814132104455, |
| "top1": 0.75 |
| } |
| ], |
| "summary": { |
| "diffusion_zero_shot": { |
| "jaccard": [ |
| 0.07278545826932924, |
| 0.044809267793138756, |
| 0.10379544290834614 |
| ], |
| "top1": 0.0967741935483871, |
| "exact": 0.008064516129032258, |
| "precision": 0.12916666666666665, |
| "recall": 0.09401881720430108, |
| "pred_n": 1.7741935483870968, |
| "missing": 2.685483870967742, |
| "extra": 1.4919354838709677 |
| }, |
| "diffusion_lora": { |
| "jaccard": [ |
| 0.4472606246799795, |
| 0.3933147721454173, |
| 0.5016513056835638 |
| ], |
| "top1": 0.75, |
| "exact": 0.10483870967741936, |
| "precision": 0.5658602150537635, |
| "recall": 0.6065956221198157, |
| "pred_n": 2.879032258064516, |
| "missing": 1.2661290322580645, |
| "extra": 1.1774193548387097 |
| }, |
| "ar_zero_shot": { |
| "jaccard": [ |
| 0.07011919532080822, |
| 0.04485037811650715, |
| 0.0987386269644334 |
| ], |
| "top1": 0.11290322580645161, |
| "exact": 0.008064516129032258, |
| "precision": 0.12575475599669148, |
| "recall": 0.11344086021505376, |
| "pred_n": 2.379032258064516, |
| "missing": 2.6693548387096775, |
| "extra": 2.0806451612903225 |
| }, |
| "ar_lora": { |
| "jaccard": [ |
| 0.4925115207373272, |
| 0.4320468509984639, |
| 0.5533218125960062 |
| ], |
| "top1": 0.782258064516129, |
| "exact": 0.22580645161290322, |
| "precision": 0.7553763440860215, |
| "recall": 0.5109927035330262, |
| "pred_n": 1.75, |
| "missing": 1.5725806451612903, |
| "extra": 0.3548387096774194 |
| }, |
| "qwen35_zero_shot": { |
| "jaccard": [ |
| 0.1381896637945025, |
| 0.10233294930875576, |
| 0.17591047792660694 |
| ], |
| "top1": 0.25, |
| "exact": 0.008064516129032258, |
| "precision": 0.21700372607630672, |
| "recall": 0.21192396313364056, |
| "pred_n": 2.4274193548387095, |
| "missing": 2.2903225806451615, |
| "extra": 1.75 |
| }, |
| "qwen27_zero_shot": { |
| "jaccard": [ |
| 0.19682306940371458, |
| 0.15791788856304986, |
| 0.23646662477307637 |
| ], |
| "top1": 0.33064516129032256, |
| "exact": 0.008064516129032258, |
| "precision": 0.322599846390169, |
| "recall": 0.279531490015361, |
| "pred_n": 2.282258064516129, |
| "missing": 2.1048387096774195, |
| "extra": 1.4193548387096775 |
| }, |
| "diffusion_aw005": { |
| "jaccard": [ |
| 0.4254032258064516, |
| 0.37307987711213514, |
| 0.4777073732718894 |
| ], |
| "top1": 0.7580645161290323, |
| "exact": 0.08870967741935484, |
| "precision": 0.5090693804403482, |
| "recall": 0.6351478494623657, |
| "pred_n": 3.467741935483871, |
| "missing": 1.1612903225806452, |
| "extra": 1.6612903225806452 |
| }, |
| "diffusion_aw025": { |
| "jaccard": [ |
| 0.43988491347362313, |
| 0.38916963412931155, |
| 0.4930299539170507 |
| ], |
| "top1": 0.75, |
| "exact": 0.08064516129032258, |
| "precision": 0.542388928276025, |
| "recall": 0.6325940860215054, |
| "pred_n": 3.411290322580645, |
| "missing": 1.185483870967742, |
| "extra": 1.6290322580645162 |
| }, |
| "diffusion_r32": { |
| "jaccard": [ |
| 0.44411802355350743, |
| 0.3898361495135689, |
| 0.49879032258064515 |
| ], |
| "top1": 0.75, |
| "exact": 0.12096774193548387, |
| "precision": 0.668010752688172, |
| "recall": 0.5104646697388633, |
| "pred_n": 2.064516129032258, |
| "missing": 1.532258064516129, |
| "extra": 0.6290322580645161 |
| }, |
| "diffusion_expanded": { |
| "jaccard": [ |
| 0.4335445468509984, |
| 0.3826612903225806, |
| 0.48529185867895547 |
| ], |
| "top1": 0.7580645161290323, |
| "exact": 0.08064516129032258, |
| "precision": 0.6024193548387097, |
| "recall": 0.5542338709677419, |
| "pred_n": 2.475806451612903, |
| "missing": 1.435483870967742, |
| "extra": 0.9435483870967742 |
| }, |
| "floor_top3": { |
| "jaccard": [ |
| 0.4740975422427035, |
| 0.4201996927803379, |
| 0.5277265745007681 |
| ], |
| "top1": 0.75, |
| "exact": 0.11290322580645161, |
| "precision": 0.5994623655913978, |
| "recall": 0.6307987711213517, |
| "pred_n": 3.0, |
| "missing": 1.1693548387096775, |
| "extra": 1.2016129032258065 |
| }, |
| "tfidf_1nn": { |
| "jaccard": [ |
| 0.500444828469022, |
| 0.4454237071172555, |
| 0.5572644649257552 |
| ], |
| "top1": 0.7661290322580645, |
| "exact": 0.1532258064516129, |
| "precision": 0.6579781105990783, |
| "recall": 0.639276113671275, |
| "pred_n": 2.814516129032258, |
| "missing": 1.1451612903225807, |
| "extra": 0.9919354838709677 |
| }, |
| "tfidf_1nn_top3vote": { |
| "jaccard": [ |
| 0.48733678955453147, |
| 0.4321716589861751, |
| 0.5448636712749616 |
| ], |
| "top1": 0.7661290322580645, |
| "exact": 0.1693548387096774, |
| "precision": 0.6760752688172043, |
| "recall": 0.59176267281106, |
| "pred_n": 2.2903225806451615, |
| "missing": 1.3548387096774193, |
| "extra": 0.6774193548387096 |
| }, |
| "tfidf_3nn": { |
| "jaccard": [ |
| 0.5050179211469534, |
| 0.45102406554019453, |
| 0.5590437788018433 |
| ], |
| "top1": 0.7903225806451613, |
| "exact": 0.1693548387096774, |
| "precision": 0.6533602150537634, |
| "recall": 0.6464285714285715, |
| "pred_n": 2.7419354838709675, |
| "missing": 1.1290322580645162, |
| "extra": 0.9032258064516129 |
| }, |
| "tfidf_3nn_top3vote": { |
| "jaccard": [ |
| 0.510515873015873, |
| 0.45830773169482847, |
| 0.5629704301075269 |
| ], |
| "top1": 0.7903225806451613, |
| "exact": 0.16129032258064516, |
| "precision": 0.6370967741935484, |
| "recall": 0.6839093701996928, |
| "pred_n": 2.8548387096774195, |
| "missing": 1.1290322580645162, |
| "extra": 1.0161290322580645 |
| }, |
| "tfidf_5nn": { |
| "jaccard": [ |
| 0.500502432155658, |
| 0.4513536866359447, |
| 0.5495967741935485 |
| ], |
| "top1": 0.7903225806451613, |
| "exact": 0.12096774193548387, |
| "precision": 0.637768817204301, |
| "recall": 0.6762576804915514, |
| "pred_n": 2.7983870967741935, |
| "missing": 1.1048387096774193, |
| "extra": 0.9354838709677419 |
| }, |
| "tfidf_5nn_top3vote": { |
| "jaccard": [ |
| 0.5122855862775217, |
| 0.45913978494623653, |
| 0.5648841525857655 |
| ], |
| "top1": 0.7903225806451613, |
| "exact": 0.1774193548387097, |
| "precision": 0.6303763440860215, |
| "recall": 0.6996351766513057, |
| "pred_n": 2.935483870967742, |
| "missing": 1.096774193548387, |
| "extra": 1.064516129032258 |
| }, |
| "train_labelset_oracle": { |
| "jaccard": [ |
| 0.9444124423963134, |
| 0.9213997695852535, |
| 0.9657834101382489 |
| ], |
| "top1": 0.9354838709677419, |
| "exact": 0.8145161290322581, |
| "precision": 0.9692492319508448, |
| "recall": 0.9717069892473118, |
| "pred_n": 3.0, |
| "missing": 0.10483870967741936, |
| "extra": 0.13709677419354838 |
| }, |
| "data_ceiling": { |
| "jaccard": [ |
| 0.9959677419354839, |
| 0.9879032258064516, |
| 1.0 |
| ], |
| "top1": 1.0, |
| "exact": 0.9919354838709677, |
| "precision": 1.0, |
| "recall": 0.9959677419354839, |
| "pred_n": 2.9596774193548385, |
| "missing": 0.008064516129032258, |
| "extra": 0.0 |
| } |
| }, |
| "paired_vs_floor": { |
| "diffusion_zero_shot": [ |
| -0.4013120839733743, |
| -0.4626536098310292, |
| -0.338773681515617 |
| ], |
| "diffusion_lora": [ |
| -0.02683691756272401, |
| -0.06750192012288786, |
| 0.012679211469534053 |
| ], |
| "ar_zero_shot": [ |
| -0.4039783469218953, |
| -0.46681436842727164, |
| -0.33903609831029186 |
| ], |
| "ar_lora": [ |
| 0.01841397849462366, |
| -0.03867127496159754, |
| 0.07795698924731184 |
| ], |
| "qwen35_zero_shot": [ |
| -0.33590787844820097, |
| -0.39298195084485404, |
| -0.27843161814936007 |
| ], |
| "qwen27_zero_shot": [ |
| -0.277274472838989, |
| -0.3321507121910348, |
| -0.2189315388912163 |
| ], |
| "diffusion_aw005": [ |
| -0.04869431643625192, |
| -0.0894489247311828, |
| -0.008787762416794671 |
| ], |
| "diffusion_aw025": [ |
| -0.034212628769080385, |
| -0.06047467317628608, |
| -0.007286866359447002 |
| ], |
| "diffusion_r32": [ |
| -0.029979518689196107, |
| -0.08051075268817205, |
| 0.021873399897593446 |
| ], |
| "diffusion_expanded": [ |
| -0.040552995391705066, |
| -0.07831221198156682, |
| -0.0032258064516129024 |
| ], |
| "tfidf_1nn": [ |
| 0.026347286226318482, |
| -0.03129800307219662, |
| 0.08436059907834101 |
| ], |
| "tfidf_1nn_top3vote": [ |
| 0.013239247311827957, |
| -0.04418202764976958, |
| 0.07288786482334869 |
| ], |
| "tfidf_3nn": [ |
| 0.030920378904249875, |
| -0.0268721198156682, |
| 0.0878200204813108 |
| ], |
| "tfidf_3nn_top3vote": [ |
| 0.036418330773169486, |
| -0.010615079365079364, |
| 0.08473502304147465 |
| ], |
| "tfidf_5nn": [ |
| 0.02640488991295443, |
| -0.026379288274449565, |
| 0.07841461853558628 |
| ], |
| "tfidf_5nn_top3vote": [ |
| 0.03818804403481822, |
| -0.005081925243215566, |
| 0.08151881720430107 |
| ], |
| "train_labelset_oracle": [ |
| 0.4703149001536099, |
| 0.4196524577572965, |
| 0.5210253456221199 |
| ], |
| "data_ceiling": [ |
| 0.5218701996927804, |
| 0.468394777265745, |
| 0.5749135944700461 |
| ] |
| }, |
| "strata_counts": { |
| "freq": { |
| "head_only": 44, |
| "one_tail": 48, |
| "multi_tail": 32 |
| }, |
| "card": { |
| "1-2": 50, |
| "3": 35, |
| "4+": 39 |
| }, |
| "length": { |
| "short": 42, |
| "medium": 41, |
| "long": 41 |
| } |
| } |
| } |
