[
  {
    "name": "focused-64mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "e63d7d483dde1aa536251a2342d6809ff6d327bf958304ead50fd817e72081d5",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 651392,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.8579595696600513,
        "mean_tokens_per_example": 1272.25,
        "p95_tokens_per_example": 2339
      },
      "json": {
        "bytes": 2782302,
        "tokens": 760588,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6580934750482523,
        "mean_tokens_per_example": 1485.5234375,
        "p95_tokens_per_example": 2608
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 55115,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3956636124467026,
        "mean_tokens_per_example": 107.646484375,
        "p95_tokens_per_example": 255
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 263776,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6609926604391605,
        "mean_tokens_per_example": 515.1875,
        "p95_tokens_per_example": 2443
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 26,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 4,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "focused",
    "size_mib": 64
  },
  {
    "name": "focused-64mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 615688,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.08168422967477,
        "mean_tokens_per_example": 1202.515625,
        "p95_tokens_per_example": 2184
      },
      "json": {
        "bytes": 2782302,
        "tokens": 722785,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.849418568453966,
        "mean_tokens_per_example": 1411.689453125,
        "p95_tokens_per_example": 2514
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 52862,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.5403881805455715,
        "mean_tokens_per_example": 103.24609375,
        "p95_tokens_per_example": 249
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 251703,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.8365931276146887,
        "mean_tokens_per_example": 491.607421875,
        "p95_tokens_per_example": 2304
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 25,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 4,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "focused",
    "size_mib": 64
  },
  {
    "name": "focused-64mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "ce4bf31da579b5be65d25bea6670d071cdbbf2ef95a22ec26807cd8fc997660c",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 601208,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.179990951550877,
        "mean_tokens_per_example": 1174.234375,
        "p95_tokens_per_example": 2144
      },
      "json": {
        "bytes": 2782302,
        "tokens": 707451,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.932854713612674,
        "mean_tokens_per_example": 1381.740234375,
        "p95_tokens_per_example": 2419
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 51789,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.613740369576551,
        "mean_tokens_per_example": 101.150390625,
        "p95_tokens_per_example": 237
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 246543,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.916890765505409,
        "mean_tokens_per_example": 481.529296875,
        "p95_tokens_per_example": 2267
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 25,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 4,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "focused",
    "size_mib": 64
  },
  {
    "name": "original-64mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "97fa4177ad5a038cc18df87a2acb76c3ed3da89a9b7848517442bcbd7029d83d",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 686845,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6588225873377547,
        "mean_tokens_per_example": 1341.494140625,
        "p95_tokens_per_example": 2464
      },
      "json": {
        "bytes": 2782302,
        "tokens": 859400,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2374936001861765,
        "mean_tokens_per_example": 1678.515625,
        "p95_tokens_per_example": 3008
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 64373,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.907305858046075,
        "mean_tokens_per_example": 125.728515625,
        "p95_tokens_per_example": 287
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 317957,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.037146532392745,
        "mean_tokens_per_example": 621.009765625,
        "p95_tokens_per_example": 2919
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 14
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 64
  },
  {
    "name": "original-64mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "cd9109d658834401ef05b4de231cdfd789385bb0d9f294e343e19b447d94b52a",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 639444,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.930045477008151,
        "mean_tokens_per_example": 1248.9140625,
        "p95_tokens_per_example": 2281
      },
      "json": {
        "bytes": 2782302,
        "tokens": 792841,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.5092811799591597,
        "mean_tokens_per_example": 1548.517578125,
        "p95_tokens_per_example": 2779
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 60430,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.0970047989409233,
        "mean_tokens_per_example": 118.02734375,
        "p95_tokens_per_example": 267
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 295090,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.272499915280084,
        "mean_tokens_per_example": 576.34765625,
        "p95_tokens_per_example": 2703
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 6,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 64
  },
  {
    "name": "original-64mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "86de4d58e6eb5417264afe227857e98c0b0431eb7adbfaf6a9ebacac37d77726",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 618301,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.064434636204696,
        "mean_tokens_per_example": 1207.619140625,
        "p95_tokens_per_example": 2199
      },
      "json": {
        "bytes": 2782302,
        "tokens": 762463,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.649097726709362,
        "mean_tokens_per_example": 1489.185546875,
        "p95_tokens_per_example": 2637
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 56136,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3339033775117572,
        "mean_tokens_per_example": 109.640625,
        "p95_tokens_per_example": 254
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 283856,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.4020136970858466,
        "mean_tokens_per_example": 554.40625,
        "p95_tokens_per_example": 2588
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 16,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 26,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 64
  },
  {
    "name": "original-128mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "64356327337221b75b445cc7c761d655a40c1af2bccd8f8ab5453fd1fa1035c0",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 684734,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6701025507715404,
        "mean_tokens_per_example": 1337.37109375,
        "p95_tokens_per_example": 2404
      },
      "json": {
        "bytes": 2782302,
        "tokens": 848790,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.277962746969215,
        "mean_tokens_per_example": 1657.79296875,
        "p95_tokens_per_example": 2970
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 63820,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.9324976496396116,
        "mean_tokens_per_example": 124.6484375,
        "p95_tokens_per_example": 287
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 317026,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.0460656223779754,
        "mean_tokens_per_example": 619.19140625,
        "p95_tokens_per_example": 2882
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 14
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 128
  },
  {
    "name": "original-128mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "ade4e945319da6e9b010ad7f48f19d9489f7ac583448318a91cf302c21aaf321",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 637085,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.9445976596529504,
        "mean_tokens_per_example": 1244.306640625,
        "p95_tokens_per_example": 2256
      },
      "json": {
        "bytes": 2782302,
        "tokens": 791494,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.515253432116984,
        "mean_tokens_per_example": 1545.88671875,
        "p95_tokens_per_example": 2750
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 60206,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.1085273892967478,
        "mean_tokens_per_example": 117.58984375,
        "p95_tokens_per_example": 264
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 293856,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2862422410976806,
        "mean_tokens_per_example": 573.9375,
        "p95_tokens_per_example": 2688
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 6,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 128
  },
  {
    "name": "original-128mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "db13f817c797466237e07a17ca3d62b65dafe334745d6c72714073fa47d36380",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 615615,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.082168238265799,
        "mean_tokens_per_example": 1202.373046875,
        "p95_tokens_per_example": 2185
      },
      "json": {
        "bytes": 2782302,
        "tokens": 761584,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.65330941826509,
        "mean_tokens_per_example": 1487.46875,
        "p95_tokens_per_example": 2625
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 56060,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3384231180877633,
        "mean_tokens_per_example": 109.4921875,
        "p95_tokens_per_example": 255
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 284063,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.399534610280114,
        "mean_tokens_per_example": 554.810546875,
        "p95_tokens_per_example": 2613
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 128
  },
  {
    "name": "original-256mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "e76d8e8a71db29a3e1ed036d8e6813ce180ac568b0310d0cdc01a2c8974c92e6",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 683686,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.675728331427,
        "mean_tokens_per_example": 1335.32421875,
        "p95_tokens_per_example": 2431
      },
      "json": {
        "bytes": 2782302,
        "tokens": 846841,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2855069605746534,
        "mean_tokens_per_example": 1653.986328125,
        "p95_tokens_per_example": 2972
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 64278,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.9116027256604124,
        "mean_tokens_per_example": 125.54296875,
        "p95_tokens_per_example": 289
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 316910,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.047180587548515,
        "mean_tokens_per_example": 618.96484375,
        "p95_tokens_per_example": 2874
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 14
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 256
  },
  {
    "name": "original-256mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "3839415fcdd736bb9322b958a45ba82ba01310e2007a1b139f77e3486568c1b6",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 635593,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.9538572640038514,
        "mean_tokens_per_example": 1241.392578125,
        "p95_tokens_per_example": 2248
      },
      "json": {
        "bytes": 2782302,
        "tokens": 788761,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.527433531830301,
        "mean_tokens_per_example": 1540.548828125,
        "p95_tokens_per_example": 2744
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 59948,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.1219056515646892,
        "mean_tokens_per_example": 117.0859375,
        "p95_tokens_per_example": 264
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 293252,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2930107893552303,
        "mean_tokens_per_example": 572.7578125,
        "p95_tokens_per_example": 2703
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 6,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 256
  },
  {
    "name": "original-256mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "eb4b69846169b0aedd86c72eb2bc7e85b96b074519f7bba8563423cd200baf81",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 613844,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.093945693042532,
        "mean_tokens_per_example": 1198.9140625,
        "p95_tokens_per_example": 2162
      },
      "json": {
        "bytes": 2782302,
        "tokens": 760204,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.659941278919869,
        "mean_tokens_per_example": 1484.7734375,
        "p95_tokens_per_example": 2628
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 55978,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.343313444567509,
        "mean_tokens_per_example": 109.33203125,
        "p95_tokens_per_example": 254
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 283137,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.410652793523983,
        "mean_tokens_per_example": 553.001953125,
        "p95_tokens_per_example": 2581
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 256
  },
  {
    "name": "original-512mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "c3bae921b15a294391e45fe3bf2ce0a2f3ac1b6678c0f32541613106f322078e",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 682997,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6794363664847722,
        "mean_tokens_per_example": 1333.978515625,
        "p95_tokens_per_example": 2411
      },
      "json": {
        "bytes": 2782302,
        "tokens": 845018,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2925949506401047,
        "mean_tokens_per_example": 1650.42578125,
        "p95_tokens_per_example": 2971
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 64290,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.911059262715819,
        "mean_tokens_per_example": 125.56640625,
        "p95_tokens_per_example": 290
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 319399,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.0234346381798316,
        "mean_tokens_per_example": 623.826171875,
        "p95_tokens_per_example": 2883
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 20,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 15
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 512
  },
  {
    "name": "original-512mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "ee276c0fba28e5d4f555f85971e9d99b5cee6b53aa5181bc1c4dd33417572829",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 634379,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.9614236915156398,
        "mean_tokens_per_example": 1239.021484375,
        "p95_tokens_per_example": 2267
      },
      "json": {
        "bytes": 2782302,
        "tokens": 789081,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.5260030339090664,
        "mean_tokens_per_example": 1541.173828125,
        "p95_tokens_per_example": 2729
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 57989,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2273707082377694,
        "mean_tokens_per_example": 113.259765625,
        "p95_tokens_per_example": 264
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 292371,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3029336014857833,
        "mean_tokens_per_example": 571.037109375,
        "p95_tokens_per_example": 2685
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 6,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 512
  },
  {
    "name": "original-512mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "1f4b70d34b608d97a3ad7a9bcd7b0094fa82c652281d39bcc12325c1f16865e2",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 612400,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.103598954931417,
        "mean_tokens_per_example": 1196.09375,
        "p95_tokens_per_example": 2147
      },
      "json": {
        "bytes": 2782302,
        "tokens": 759199,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6647861759565017,
        "mean_tokens_per_example": 1482.810546875,
        "p95_tokens_per_example": 2621
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 55806,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3536178905494034,
        "mean_tokens_per_example": 108.99609375,
        "p95_tokens_per_example": 254
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 282554,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.417690069862752,
        "mean_tokens_per_example": 551.86328125,
        "p95_tokens_per_example": 2588
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 512
  },
  {
    "name": "original-1024mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "020928562c47ea392514da50b9c4886b4e3e24fd01cfdc30501599042ffb1359",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 682180,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.6838429739951333,
        "mean_tokens_per_example": 1332.3828125,
        "p95_tokens_per_example": 2410
      },
      "json": {
        "bytes": 2782302,
        "tokens": 844701,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2938305980459357,
        "mean_tokens_per_example": 1649.806640625,
        "p95_tokens_per_example": 2974
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 64244,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.913143639872984,
        "mean_tokens_per_example": 125.4765625,
        "p95_tokens_per_example": 289
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 317323,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.0432146424936106,
        "mean_tokens_per_example": 619.771484375,
        "p95_tokens_per_example": 2882
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 14
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 1024
  },
  {
    "name": "original-1024mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "17ce952fb54679634b5a723756a217bb536ad1d2f6616917ac0172b7a6ef255c",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 633318,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.9680602793541317,
        "mean_tokens_per_example": 1236.94921875,
        "p95_tokens_per_example": 2219
      },
      "json": {
        "bytes": 2782302,
        "tokens": 787329,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.5338492548858227,
        "mean_tokens_per_example": 1537.751953125,
        "p95_tokens_per_example": 2739
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 57750,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.240727272727273,
        "mean_tokens_per_example": 112.79296875,
        "p95_tokens_per_example": 260
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 292030,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3067903982467555,
        "mean_tokens_per_example": 570.37109375,
        "p95_tokens_per_example": 2677
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 6,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 1024
  },
  {
    "name": "original-1024mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "ce5e83a52cab742ea5edc2440c8defd26c946d05ca2f1d3f64ab3dedc26b744d",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 611455,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.109941042268033,
        "mean_tokens_per_example": 1194.248046875,
        "p95_tokens_per_example": 2151
      },
      "json": {
        "bytes": 2782302,
        "tokens": 757438,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.673306594071066,
        "mean_tokens_per_example": 1479.37109375,
        "p95_tokens_per_example": 2618
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 55758,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3565048961583988,
        "mean_tokens_per_example": 108.90234375,
        "p95_tokens_per_example": 248
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 281364,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.4321448372926175,
        "mean_tokens_per_example": 549.5390625,
        "p95_tokens_per_example": 2566
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 1024
  },
  {
    "name": "original-2048mib-16384",
    "vocab": 16384,
    "execution": "modal-cpu",
    "tokenizer_sha256": "f95ec3899180fd6aef876f92e8fa1e6ec9cd96c180e22b23ead4e88b3b002561",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 682473,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.682261422796213,
        "mean_tokens_per_example": 1332.955078125,
        "p95_tokens_per_example": 2401
      },
      "json": {
        "bytes": 2782302,
        "tokens": 845529,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.2906050531678983,
        "mean_tokens_per_example": 1651.423828125,
        "p95_tokens_per_example": 2970
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 64223,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 2.9140961960668297,
        "mean_tokens_per_example": 125.435546875,
        "p95_tokens_per_example": 289
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 316917,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.0471132820265243,
        "mean_tokens_per_example": 618.978515625,
        "p95_tokens_per_example": 2881
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30,
      "get_user_configuration": 4,
      "\"additionalProperties\":false": 7,
      "\"tool_call_id\":\"call_0123456789\"": 14
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 2048
  },
  {
    "name": "original-2048mib-32768",
    "vocab": 32768,
    "execution": "modal-cpu",
    "tokenizer_sha256": "6f8742023692929139e450a3a64802e355649742e7051f9d8a6e6db47fbc8e96",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 633463,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.967151988356068,
        "mean_tokens_per_example": 1237.232421875,
        "p95_tokens_per_example": 2278
      },
      "json": {
        "bytes": 2782302,
        "tokens": 787465,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.533238937603576,
        "mean_tokens_per_example": 1538.017578125,
        "p95_tokens_per_example": 2730
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 59957,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.121437029871408,
        "mean_tokens_per_example": 117.103515625,
        "p95_tokens_per_example": 268
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 292566,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3007321424909253,
        "mean_tokens_per_example": 571.41796875,
        "p95_tokens_per_example": 2676
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 2048
  },
  {
    "name": "original-2048mib-50000",
    "vocab": 50000,
    "execution": "modal-cpu",
    "tokenizer_sha256": "5e3de8f77cfa5a5820a96bf0205e7cceee7f27c43e16afdc16d452df7a37f724",
    "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
    "metrics": {
      "python": {
        "bytes": 2513044,
        "tokens": 611191,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 4.111716304723074,
        "mean_tokens_per_example": 1193.732421875,
        "p95_tokens_per_example": 2152
      },
      "json": {
        "bytes": 2782302,
        "tokens": 762018,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.651228711132808,
        "mean_tokens_per_example": 1488.31640625,
        "p95_tokens_per_example": 2631
      },
      "git_diff": {
        "bytes": 187152,
        "tokens": 55765,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.3560835649601004,
        "mean_tokens_per_example": 108.916015625,
        "p95_tokens_per_example": 248
      },
      "tool_call": {
        "bytes": 965682,
        "tokens": 282399,
        "examples": 512,
        "roundtrip_failures": 0,
        "unk_tokens": 0,
        "bytes_per_token": 3.4195659333071293,
        "mean_tokens_per_example": 551.560546875,
        "p95_tokens_per_example": 2587
      }
    },
    "fragmentation": {
      "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17,
      "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28,
      "get_user_configuration": 3,
      "\"additionalProperties\":false": 5,
      "\"tool_call_id\":\"call_0123456789\"": 12
    },
    "correctness_passed": true,
    "family": "original",
    "size_mib": 2048
  }
]