Upload recipe/train_config.yaml with huggingface_hub

recipe/train_config.yaml

@@ -0,0 +1,129 @@
+recipe_version: hf-bytelevel-bpe-v1
+algorithm:
+  model: BPE
+  pre_tokenizer: ByteLevel
+  byte_level_use_regex: false
+  byte_level_add_prefix_space: false
+  normalizer: null
+  decoder: ByteLevel
+  post_processor: ByteLevel
+trainer:
+  vocab_size: 256000
+  min_frequency: 2
+  initial_alphabet: ByteLevel.alphabet()
+  special_tokens:
+  - <|pad|>
+  - <|startoftext|>
+  - <|endoftext|>
+  - <|unk|>
+  - <|im_start|>
+  - <|im_end|>
+  - <|tool_call_start|>
+  - <|tool_call_end|>
+  - <|tool_response_start|>
+  - <|tool_response_end|>
+  - <|reasoning_start|>
+  - <|reasoning_end|>
+  - <|fim_prefix|>
+  - <|fim_middle|>
+  - <|fim_suffix|>
+  - <|fim_pad|>
+  - <|repo_name|>
+  - <|file_sep|>
+  - <|lang:all|>
+  - <|lang:amu|>
+  - <|lang:amu0|>
+  - <|lang:azg|>
+  - <|lang:azn0|>
+  - <|lang:azz|>
+  - <|lang:cco|>
+  - <|lang:cco0|>
+  - <|lang:chd|>
+  - <|lang:chf|>
+  - <|lang:chq|>
+  - <|lang:ctp0|>
+  - <|lang:ctu|>
+  - <|lang:cux|>
+  - <|lang:de|>
+  - <|lang:en-US|>
+  - <|lang:eng|>
+  - <|lang:es-MX|>
+  - <|lang:fr|>
+  - <|lang:hch|>
+  - <|lang:hus|>
+  - <|lang:huv|>
+  - <|lang:lac|>
+  - <|lang:mam|>
+  - <|lang:mau|>
+  - <|lang:maz|>
+  - <|lang:meh|>
+  - <|lang:mfy|>
+  - <|lang:mix|>
+  - <|lang:mmc|>
+  - <|lang:nch|>
+  - <|lang:ntp|>
+  - <|lang:ocu|>
+  - <|lang:ots|>
+  - <|lang:pei|>
+  - <|lang:pia|>
+  - <|lang:pmq|>
+  - <|lang:poi|>
+  - <|lang:pt-BR|>
+  - <|lang:pua|>
+  - <|lang:quc|>
+  - <|lang:sei|>
+  - <|lang:spa|>
+  - <|lang:tar0|>
+  - <|lang:toj|>
+  - <|lang:top|>
+  - <|lang:tpp|>
+  - <|lang:tpx|>
+  - <|lang:trs|>
+  - <|lang:tsz|>
+  - <|lang:tzh|>
+  - <|lang:tzo|>
+  - <|lang:unk|>
+  - <|lang:var|>
+  - <|lang:yaq|>
+  - <|lang:yua|>
+  - <|lang:zai|>
+  - <|lang:zam|>
+  - <|lang:zoc|>
+  - <|lang:zoq|>
+  - <|reserved_0|>
+  - <|reserved_1|>
+  - <|reserved_2|>
+  - <|reserved_3|>
+  - <|reserved_4|>
+  - <|reserved_5|>
+  - <|reserved_6|>
+  - <|reserved_7|>
+  - <|reserved_8|>
+  - <|reserved_9|>
+  - <|reserved_10|>
+  - <|reserved_11|>
+  - <|reserved_12|>
+  - <|reserved_13|>
+  - <|reserved_14|>
+  - <|reserved_15|>
+corpus_weights:
+  exotic: 0.7
+  english: 0.1
+  spanish: 0.1
+  code: 0.1
+corpus_bytes:
+  modern: 24824308
+  old: 10197249
+  eng: 5002329
+  spa: 5000693
+  code: 5002607
+  exotic: 35021557
+sources:
+  exotic: tachiwin/tokenizer_train
+  english: uonlp/CulturaX:en
+  spanish: uonlp/CulturaX:es
+  code: bigcode/the-stack-smol
+checkpoint_strategy:
+  type: completed-artifact-resume
+  description: Corpus preparation and final tokenizer are resumable via Hub. HF BpeTrainer
+    does not expose an internal merge-state checkpoint.