climbmix-code-v1: 32K BPE fitted on the ClimbMix plus code recipe

config.json

@@ -0,0 +1,16 @@
+{
+  "format": "huggingface-tokenizers-json",
+  "vocab_size": 32768,
+  "special_tokens": {
+    "<|pad|>": 0,
+    "<|bos|>": 1,
+    "<|eos|>": 2,
+    "<|unk|>": 3
+  },
+  "normalization": null,
+  "pretokenization_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "add_prefix_space": false,
+  "automatic_bos_eos": false,
+  "padding": null,
+  "truncation": null
+}

evaluation.json

@@ -0,0 +1,81 @@
+{
+  "eval_sha256": "71e377e1008755b83876ebab70f509df46974387d270ca218ceb7a7263ade543",
+  "tokenizers": {
+    "climbmix-code-v1": {
+      "tokenizer_sha256": "48aa6fb6d01d04067570637ca8a5ecced608f5df5585f9ce44a12c05c5281ae5",
+      "sources": {
+        "climbmix": {
+          "bytes": 14743727,
+          "tokens": 3217386,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 4.582517298204194
+        },
+        "code_curated": {
+          "bytes": 3682462,
+          "tokens": 1023424,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 3.598178272153086
+        },
+        "code_mit": {
+          "bytes": 1733957,
+          "tokens": 503413,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 3.4444025084771352
+        }
+      },
+      "digits": [
+        "1",
+        "2",
+        "3",
+        "4",
+        "5",
+        "\u0120+",
+        "\u0120",
+        "6",
+        "7",
+        "8",
+        "\u0120=",
+        "\u0120",
+        "1",
+        "3",
+        "0",
+        "2",
+        "3"
+      ]
+    },
+    "structured-code-v1": {
+      "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9",
+      "sources": {
+        "climbmix": {
+          "bytes": 14743727,
+          "tokens": 4022037,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 3.6657362923314727
+        },
+        "code_curated": {
+          "bytes": 3682462,
+          "tokens": 1046393,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 3.519195942633408
+        },
+        "code_mit": {
+          "bytes": 1733957,
+          "tokens": 514537,
+          "roundtrip_failures": 0,
+          "bytes_per_token": 3.3699364671539658
+        }
+      },
+      "digits": [
+        "123",
+        "45",
+        "\u0120+",
+        "\u0120",
+        "678",
+        "\u0120=",
+        "\u0120",
+        "130",
+        "23"
+      ]
+    }
+  }
+}

freeze-manifest.json

@@ -0,0 +1,12 @@
+{
+  "id": "climbmix-code-bpe-v1",
+  "status": "frozen",
+  "vocab_size": 32768,
+  "files": {
+    "config.json": "ef1f7e15eb8451f989ed2b380a09ce84aa3960ea03d2e472aa1f25a12cdb9a18",
+    "sample.json": "cf6e2241aba3ea6f3c779e54087cb01ffa3394cf5e5562807c244561d3de5cf9",
+    "tokenizer.json": "48aa6fb6d01d04067570637ca8a5ecced608f5df5585f9ce44a12c05c5281ae5",
+    "evaluation.json": "57a207587ae46a9a4293629a52f5653b6de980353f320b639c52e926c1c6ddc8"
+  },
+  "contract": "Immutable. Record tokenizer SHA256 in shards and checkpoints. Prior tokenizer data must be retokenized in a new version."
+}

sample.json

@@ -0,0 +1,30 @@
+{
+  "recipe_sha256": "320b4e5219aba8a78dd675847041653c55eb96f74ef57abdc54d472f2c90ea5f",
+  "train_bytes_target": 268435456,
+  "eval_bytes_target": 16777216,
+  "files_per_source": 64,
+  "sources": {
+    "climbmix": {
+      "train": 228529615,
+      "validation": 14743727,
+      "documents": 81269,
+      "files": 64
+    },
+    "code_curated": {
+      "train": 31454416,
+      "validation": 3682462,
+      "documents": 5130,
+      "files": 64
+    },
+    "code_mit": {
+      "train": 13964697,
+      "validation": 1733957,
+      "documents": 1586,
+      "files": 54
+    }
+  },
+  "train_sha256": "f4b611bc901131b6dd4a8824b30024fb7e38c3f56be6a83a1fed482549f4f1df",
+  "eval_sha256": "71e377e1008755b83876ebab70f509df46974387d270ca218ceb7a7263ade543",
+  "recipe_id": "climbmix-code-v1",
+  "fit_seconds": 31.667878392000034
+}