jfrick's picture
Publish code-focused 32K tokenizer and promote it to the default
0ebdb4f verified
Raw History Blame Contribute Delete
1.53 kB
{
"seed": 20261016,
"target_bytes": 67108864,
"weights_percent": {
"python": 40,
"json": 20,
"git_diff": 20,
"tool_call": 20
},
"actual_bytes": {
"python": 26847131,
"json": 13424847,
"git_diff": 13421882,
"tool_call": 13422435
},
"training_examples": {
"python": 5059,
"json": 2290,
"git_diff": 33758,
"tool_call": 6555
},
"fitting_parents": 14911,
"evaluation_parents_per_category": 512,
"train_sha256": "cdc34355495f1c21bba4742ac11e9bcc0a0d0df836ab49eed9a1e01f97ab9a09",
"evaluation_sha256": "a9a82723c1e78b46ec52b33b81551a61a5498e46ac73a6f09e4c0da9e27775a0",
"source_train_sha256": "5e1c04e310375d9922e008142e7d9813f56e541918b729dd7764eaec3ffbadea",
"source_validation_sha256": "611010954c9c48ccad9b9a28f7377d56ab180373b5e7eb76fe87ab723b2b8040",
"implementation_sha256": "cd1f6e5351aad5acce1a1021af649c4dd8a099c2b0d506e7cc67cd5aee230163",
"limitations": [
"Python excerpts from the existing pinned codeparrot corpus, not necessarily complete parseable files",
"JSON, tool calls and patches are generated controlled fixtures, not independent production traces",
"Fitting/evaluation parents are disjoint; generator templates are shared",
"Each category quota includes its final whole example, so 64 MiB is approximate",
"Unified diffs cover insertions, deletions and identifier replacements, but not binary patches or renames",
"One fitting seed; no model-level tool-call or code-generation correctness measured"
]
}