add: tokenizer/config files

config.json

@@ -0,0 +1,55 @@
+{
+  "model_type": "Transformer",
+  "_name_or_path": "Transformer-340M-0408",
+  "architectures": ["TransformerForCausalLM"],
+  "dim": 1024,
+  "num_heads": 4,
+  "num_layers": 11,
+  "seq_len": 4096,
+  "vocab_size": 200064,
+  "inter_dim": 4096,
+  "mlp_scale": 12,
+  "bias": false,
+  "weight_tying": true,
+  "rope_theta": 10000.0,
+  "num_epochs": 1,
+  "global_bsz": 524288,
+  "bsz": 8,
+  "warmup_steps": 1907,
+  "eval_period": 50,
+  "save_period": 1000,
+  "max_lr": 4.0e-4,
+  "min_lr": 4.0e-5,
+  "max_norm": 1.0,
+  "fsdp": true,
+  "ddp": false,
+  "reshard_after_forward_policy": "default",
+  "mixed_precision": true,
+  "torch_dtype": "bfloat16",
+  "cpu_offload": false,
+  "sharding_strategy": "full_shard",
+  "state_dict_type": "full",
+  "auto_wrap_policy": "partial",
+  "backward_prefetch": "backward_pre",
+  "forward_prefetch": false,
+  "sync_module_states": true,
+  "use_orig_params": true,
+  "device_id": null,
+  "precision": {
+    "param": "bfloat16",
+    "reduce": "bfloat16",
+    "buffer": "bfloat16"
+  },
+  "fsdp_modules": [
+    "AttentionLayer"
+  ],
+  "num_workers": 0,
+  "snapshot_every_n_steps": 50,
+  "use_activation_checkpointing": true,
+  "torch_compile": true,
+  "torch_compile_kwargs": {
+    "mode": "default",
+    "fullgraph": true
+  },
+  "enable_compiled_autograd": false
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 128000,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}