| [ |
| { |
| "name": "focused-64mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "e63d7d483dde1aa536251a2342d6809ff6d327bf958304ead50fd817e72081d5", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 651392, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8579595696600513, |
| "mean_tokens_per_example": 1272.25, |
| "p95_tokens_per_example": 2339 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 760588, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6580934750482523, |
| "mean_tokens_per_example": 1485.5234375, |
| "p95_tokens_per_example": 2608 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 55115, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3956636124467026, |
| "mean_tokens_per_example": 107.646484375, |
| "p95_tokens_per_example": 255 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 263776, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6609926604391605, |
| "mean_tokens_per_example": 515.1875, |
| "p95_tokens_per_example": 2443 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 26, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 4, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "focused", |
| "size_mib": 64 |
| }, |
| { |
| "name": "focused-64mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 615688, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.08168422967477, |
| "mean_tokens_per_example": 1202.515625, |
| "p95_tokens_per_example": 2184 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 722785, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.849418568453966, |
| "mean_tokens_per_example": 1411.689453125, |
| "p95_tokens_per_example": 2514 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 52862, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5403881805455715, |
| "mean_tokens_per_example": 103.24609375, |
| "p95_tokens_per_example": 249 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 251703, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8365931276146887, |
| "mean_tokens_per_example": 491.607421875, |
| "p95_tokens_per_example": 2304 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 25, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 4, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "focused", |
| "size_mib": 64 |
| }, |
| { |
| "name": "focused-64mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "ce4bf31da579b5be65d25bea6670d071cdbbf2ef95a22ec26807cd8fc997660c", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 601208, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.179990951550877, |
| "mean_tokens_per_example": 1174.234375, |
| "p95_tokens_per_example": 2144 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 707451, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.932854713612674, |
| "mean_tokens_per_example": 1381.740234375, |
| "p95_tokens_per_example": 2419 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 51789, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.613740369576551, |
| "mean_tokens_per_example": 101.150390625, |
| "p95_tokens_per_example": 237 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 246543, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.916890765505409, |
| "mean_tokens_per_example": 481.529296875, |
| "p95_tokens_per_example": 2267 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 14, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 25, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 4, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "focused", |
| "size_mib": 64 |
| }, |
| { |
| "name": "original-64mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "97fa4177ad5a038cc18df87a2acb76c3ed3da89a9b7848517442bcbd7029d83d", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 686845, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6588225873377547, |
| "mean_tokens_per_example": 1341.494140625, |
| "p95_tokens_per_example": 2464 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 859400, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2374936001861765, |
| "mean_tokens_per_example": 1678.515625, |
| "p95_tokens_per_example": 3008 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 64373, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.907305858046075, |
| "mean_tokens_per_example": 125.728515625, |
| "p95_tokens_per_example": 287 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 317957, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.037146532392745, |
| "mean_tokens_per_example": 621.009765625, |
| "p95_tokens_per_example": 2919 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 14 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 64 |
| }, |
| { |
| "name": "original-64mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "cd9109d658834401ef05b4de231cdfd789385bb0d9f294e343e19b447d94b52a", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 639444, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.930045477008151, |
| "mean_tokens_per_example": 1248.9140625, |
| "p95_tokens_per_example": 2281 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 792841, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5092811799591597, |
| "mean_tokens_per_example": 1548.517578125, |
| "p95_tokens_per_example": 2779 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 60430, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.0970047989409233, |
| "mean_tokens_per_example": 118.02734375, |
| "p95_tokens_per_example": 267 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 295090, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.272499915280084, |
| "mean_tokens_per_example": 576.34765625, |
| "p95_tokens_per_example": 2703 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 6, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 64 |
| }, |
| { |
| "name": "original-64mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "86de4d58e6eb5417264afe227857e98c0b0431eb7adbfaf6a9ebacac37d77726", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 618301, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.064434636204696, |
| "mean_tokens_per_example": 1207.619140625, |
| "p95_tokens_per_example": 2199 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 762463, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.649097726709362, |
| "mean_tokens_per_example": 1489.185546875, |
| "p95_tokens_per_example": 2637 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 56136, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3339033775117572, |
| "mean_tokens_per_example": 109.640625, |
| "p95_tokens_per_example": 254 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 283856, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4020136970858466, |
| "mean_tokens_per_example": 554.40625, |
| "p95_tokens_per_example": 2588 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 16, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 26, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 64 |
| }, |
| { |
| "name": "original-128mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "64356327337221b75b445cc7c761d655a40c1af2bccd8f8ab5453fd1fa1035c0", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 684734, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6701025507715404, |
| "mean_tokens_per_example": 1337.37109375, |
| "p95_tokens_per_example": 2404 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 848790, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.277962746969215, |
| "mean_tokens_per_example": 1657.79296875, |
| "p95_tokens_per_example": 2970 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 63820, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.9324976496396116, |
| "mean_tokens_per_example": 124.6484375, |
| "p95_tokens_per_example": 287 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 317026, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.0460656223779754, |
| "mean_tokens_per_example": 619.19140625, |
| "p95_tokens_per_example": 2882 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 14 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 128 |
| }, |
| { |
| "name": "original-128mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "ade4e945319da6e9b010ad7f48f19d9489f7ac583448318a91cf302c21aaf321", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 637085, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9445976596529504, |
| "mean_tokens_per_example": 1244.306640625, |
| "p95_tokens_per_example": 2256 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 791494, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.515253432116984, |
| "mean_tokens_per_example": 1545.88671875, |
| "p95_tokens_per_example": 2750 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 60206, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.1085273892967478, |
| "mean_tokens_per_example": 117.58984375, |
| "p95_tokens_per_example": 264 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 293856, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2862422410976806, |
| "mean_tokens_per_example": 573.9375, |
| "p95_tokens_per_example": 2688 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 6, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 128 |
| }, |
| { |
| "name": "original-128mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "db13f817c797466237e07a17ca3d62b65dafe334745d6c72714073fa47d36380", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 615615, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.082168238265799, |
| "mean_tokens_per_example": 1202.373046875, |
| "p95_tokens_per_example": 2185 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 761584, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.65330941826509, |
| "mean_tokens_per_example": 1487.46875, |
| "p95_tokens_per_example": 2625 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 56060, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3384231180877633, |
| "mean_tokens_per_example": 109.4921875, |
| "p95_tokens_per_example": 255 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 284063, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.399534610280114, |
| "mean_tokens_per_example": 554.810546875, |
| "p95_tokens_per_example": 2613 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 128 |
| }, |
| { |
| "name": "original-256mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "e76d8e8a71db29a3e1ed036d8e6813ce180ac568b0310d0cdc01a2c8974c92e6", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 683686, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.675728331427, |
| "mean_tokens_per_example": 1335.32421875, |
| "p95_tokens_per_example": 2431 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 846841, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2855069605746534, |
| "mean_tokens_per_example": 1653.986328125, |
| "p95_tokens_per_example": 2972 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 64278, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.9116027256604124, |
| "mean_tokens_per_example": 125.54296875, |
| "p95_tokens_per_example": 289 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 316910, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.047180587548515, |
| "mean_tokens_per_example": 618.96484375, |
| "p95_tokens_per_example": 2874 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 14 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 256 |
| }, |
| { |
| "name": "original-256mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "3839415fcdd736bb9322b958a45ba82ba01310e2007a1b139f77e3486568c1b6", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 635593, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9538572640038514, |
| "mean_tokens_per_example": 1241.392578125, |
| "p95_tokens_per_example": 2248 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 788761, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.527433531830301, |
| "mean_tokens_per_example": 1540.548828125, |
| "p95_tokens_per_example": 2744 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 59948, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.1219056515646892, |
| "mean_tokens_per_example": 117.0859375, |
| "p95_tokens_per_example": 264 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 293252, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2930107893552303, |
| "mean_tokens_per_example": 572.7578125, |
| "p95_tokens_per_example": 2703 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 6, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 256 |
| }, |
| { |
| "name": "original-256mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "eb4b69846169b0aedd86c72eb2bc7e85b96b074519f7bba8563423cd200baf81", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 613844, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.093945693042532, |
| "mean_tokens_per_example": 1198.9140625, |
| "p95_tokens_per_example": 2162 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 760204, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.659941278919869, |
| "mean_tokens_per_example": 1484.7734375, |
| "p95_tokens_per_example": 2628 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 55978, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.343313444567509, |
| "mean_tokens_per_example": 109.33203125, |
| "p95_tokens_per_example": 254 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 283137, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.410652793523983, |
| "mean_tokens_per_example": 553.001953125, |
| "p95_tokens_per_example": 2581 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 256 |
| }, |
| { |
| "name": "original-512mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "c3bae921b15a294391e45fe3bf2ce0a2f3ac1b6678c0f32541613106f322078e", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 682997, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6794363664847722, |
| "mean_tokens_per_example": 1333.978515625, |
| "p95_tokens_per_example": 2411 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 845018, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2925949506401047, |
| "mean_tokens_per_example": 1650.42578125, |
| "p95_tokens_per_example": 2971 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 64290, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.911059262715819, |
| "mean_tokens_per_example": 125.56640625, |
| "p95_tokens_per_example": 290 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 319399, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.0234346381798316, |
| "mean_tokens_per_example": 623.826171875, |
| "p95_tokens_per_example": 2883 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 20, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 15 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 512 |
| }, |
| { |
| "name": "original-512mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "ee276c0fba28e5d4f555f85971e9d99b5cee6b53aa5181bc1c4dd33417572829", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 634379, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9614236915156398, |
| "mean_tokens_per_example": 1239.021484375, |
| "p95_tokens_per_example": 2267 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 789081, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5260030339090664, |
| "mean_tokens_per_example": 1541.173828125, |
| "p95_tokens_per_example": 2729 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 57989, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2273707082377694, |
| "mean_tokens_per_example": 113.259765625, |
| "p95_tokens_per_example": 264 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 292371, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3029336014857833, |
| "mean_tokens_per_example": 571.037109375, |
| "p95_tokens_per_example": 2685 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 6, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 512 |
| }, |
| { |
| "name": "original-512mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "1f4b70d34b608d97a3ad7a9bcd7b0094fa82c652281d39bcc12325c1f16865e2", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 612400, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.103598954931417, |
| "mean_tokens_per_example": 1196.09375, |
| "p95_tokens_per_example": 2147 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 759199, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6647861759565017, |
| "mean_tokens_per_example": 1482.810546875, |
| "p95_tokens_per_example": 2621 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 55806, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3536178905494034, |
| "mean_tokens_per_example": 108.99609375, |
| "p95_tokens_per_example": 254 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 282554, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.417690069862752, |
| "mean_tokens_per_example": 551.86328125, |
| "p95_tokens_per_example": 2588 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 512 |
| }, |
| { |
| "name": "original-1024mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "020928562c47ea392514da50b9c4886b4e3e24fd01cfdc30501599042ffb1359", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 682180, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6838429739951333, |
| "mean_tokens_per_example": 1332.3828125, |
| "p95_tokens_per_example": 2410 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 844701, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2938305980459357, |
| "mean_tokens_per_example": 1649.806640625, |
| "p95_tokens_per_example": 2974 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 64244, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.913143639872984, |
| "mean_tokens_per_example": 125.4765625, |
| "p95_tokens_per_example": 289 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 317323, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.0432146424936106, |
| "mean_tokens_per_example": 619.771484375, |
| "p95_tokens_per_example": 2882 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 14 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 1024 |
| }, |
| { |
| "name": "original-1024mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "17ce952fb54679634b5a723756a217bb536ad1d2f6616917ac0172b7a6ef255c", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 633318, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9680602793541317, |
| "mean_tokens_per_example": 1236.94921875, |
| "p95_tokens_per_example": 2219 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 787329, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5338492548858227, |
| "mean_tokens_per_example": 1537.751953125, |
| "p95_tokens_per_example": 2739 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 57750, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.240727272727273, |
| "mean_tokens_per_example": 112.79296875, |
| "p95_tokens_per_example": 260 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 292030, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3067903982467555, |
| "mean_tokens_per_example": 570.37109375, |
| "p95_tokens_per_example": 2677 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 6, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 1024 |
| }, |
| { |
| "name": "original-1024mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "ce5e83a52cab742ea5edc2440c8defd26c946d05ca2f1d3f64ab3dedc26b744d", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 611455, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.109941042268033, |
| "mean_tokens_per_example": 1194.248046875, |
| "p95_tokens_per_example": 2151 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 757438, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.673306594071066, |
| "mean_tokens_per_example": 1479.37109375, |
| "p95_tokens_per_example": 2618 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 55758, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3565048961583988, |
| "mean_tokens_per_example": 108.90234375, |
| "p95_tokens_per_example": 248 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 281364, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4321448372926175, |
| "mean_tokens_per_example": 549.5390625, |
| "p95_tokens_per_example": 2566 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 1024 |
| }, |
| { |
| "name": "original-2048mib-16384", |
| "vocab": 16384, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "f95ec3899180fd6aef876f92e8fa1e6ec9cd96c180e22b23ead4e88b3b002561", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 682473, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.682261422796213, |
| "mean_tokens_per_example": 1332.955078125, |
| "p95_tokens_per_example": 2401 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 845529, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.2906050531678983, |
| "mean_tokens_per_example": 1651.423828125, |
| "p95_tokens_per_example": 2970 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 64223, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 2.9140961960668297, |
| "mean_tokens_per_example": 125.435546875, |
| "p95_tokens_per_example": 289 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 316917, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.0471132820265243, |
| "mean_tokens_per_example": 618.978515625, |
| "p95_tokens_per_example": 2881 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 18, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 30, |
| "get_user_configuration": 4, |
| "\"additionalProperties\":false": 7, |
| "\"tool_call_id\":\"call_0123456789\"": 14 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 2048 |
| }, |
| { |
| "name": "original-2048mib-32768", |
| "vocab": 32768, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "6f8742023692929139e450a3a64802e355649742e7051f9d8a6e6db47fbc8e96", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 633463, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.967151988356068, |
| "mean_tokens_per_example": 1237.232421875, |
| "p95_tokens_per_example": 2278 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 787465, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.533238937603576, |
| "mean_tokens_per_example": 1538.017578125, |
| "p95_tokens_per_example": 2730 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 59957, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.121437029871408, |
| "mean_tokens_per_example": 117.103515625, |
| "p95_tokens_per_example": 268 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 292566, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3007321424909253, |
| "mean_tokens_per_example": 571.41796875, |
| "p95_tokens_per_example": 2676 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 29, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 2048 |
| }, |
| { |
| "name": "original-2048mib-50000", |
| "vocab": 50000, |
| "execution": "modal-cpu", |
| "tokenizer_sha256": "5e3de8f77cfa5a5820a96bf0205e7cceee7f27c43e16afdc16d452df7a37f724", |
| "evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0", |
| "metrics": { |
| "python": { |
| "bytes": 2513044, |
| "tokens": 611191, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.111716304723074, |
| "mean_tokens_per_example": 1193.732421875, |
| "p95_tokens_per_example": 2152 |
| }, |
| "json": { |
| "bytes": 2782302, |
| "tokens": 762018, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.651228711132808, |
| "mean_tokens_per_example": 1488.31640625, |
| "p95_tokens_per_example": 2631 |
| }, |
| "git_diff": { |
| "bytes": 187152, |
| "tokens": 55765, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3560835649601004, |
| "mean_tokens_per_example": 108.916015625, |
| "p95_tokens_per_example": 248 |
| }, |
| "tool_call": { |
| "bytes": 965682, |
| "tokens": 282399, |
| "examples": 512, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4195659333071293, |
| "mean_tokens_per_example": 551.560546875, |
| "p95_tokens_per_example": 2587 |
| } |
| }, |
| "fragmentation": { |
| "{\"name\":\"read_file\",\"arguments\":{\"path\":\"src/main.py\"}}": 17, |
| "diff --git a/main.py b/main.py\n@@ -1,2 +1,2 @@\n-old\n+new\n": 28, |
| "get_user_configuration": 3, |
| "\"additionalProperties\":false": 5, |
| "\"tool_call_id\":\"call_0123456789\"": 12 |
| }, |
| "correctness_passed": true, |
| "family": "original", |
| "size_mib": 2048 |
| } |
| ] |
|
|
