Upload folder using huggingface_hub

bpe128k_dclm-edu150M/raw_tok_name.txt

@@ -0,0 +1 @@
+bpe320000_2025-09-20T17-01-07

bpe128k_dclm-edu150M/special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "eos_token": "<|eos|>"
+}

bpe128k_dclm-edu150M/tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}