Upload lambda-360m midtrained model

model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9cd49fa043f982a708fe6f7b538ff9f2bbb0f275d1e3fa2af4330403ccc760c
+size 1439512299

model_config.json

@@ -0,0 +1 @@
+{"max_len": 1024, "d_model": 960, "num_layers": 32, "num_heads": 15, "num_kv_heads": 5, "d_ff": 4096, "learning_rate": 0.0001, "batch_size": 48, "gradient_accumulation_steps": 2, "devices": "auto", "device_count": 2, "global_batch_size": 96, "effective_batch_size": 96, "global_effective_batch_size": 192, "lr_schedule": "warmup_cosine", "lr_warmup_steps": 1000, "min_learning_rate": 2e-05, "min_learning_rate_ratio": 0.2, "pad_token_id": 0, "bos_token_id": 2, "eos_token_id": 3, "train_dataset_signature": "44bbfca21e62efa8", "validation_dataset_signature": "8fb1014e14cb23c7", "train_dataset_case": {"name": "synthetic-textbook-jp-train", "genre": "textbook", "language": "ja", "dataset_path": "KeisukeMiyamoto/SyntheticTextbook-jp", "config_name": "default", "split": "train", "text_column": "rewrite"}, "validation_dataset_case": {"name": "synthetic-textbook-jp-validation", "genre": "textbook", "language": "ja", "dataset_path": "KeisukeMiyamoto/SyntheticTextbook-jp", "config_name": "default", "split": "validation", "text_column": "rewrite"}, "validation_cache_path": "models/lambda-360m-midtrained/validation-cache-8fb1014e14cb23c7-bucket-packing-v1-len1024-samples6144.pt", "validation_sample_count": 6144, "packing_version": "bucket-packing-v1", "shuffle_buffer_size": 10000, "base_shuffle_seed": 17, "shuffle_seed": 17, "trained_steps": 20480, "training_precision": "fp8", "float8_recipe": "tensorwise", "midtraining_source_model": "models/lambda-360m", "midtraining_max_steps": 20480}

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "pad_token": "|<pad>|",
+  "unk_token": "|<unknown>|",
+  "bos_token": "|<bos>|",
+  "eos_token": "|<eos>|",
+  "sep_token": "|<sep>|",
+  "cls_token": "|<cls>|",
+  "mask_token": "|<mask>|",
+  "extra_special_tokens": [
+    "|<system>|",
+    "|<user>|",
+    "|<assistant>|",
+    "|<thinking>|",
+    "|<end_of_thinking>|",
+    "|<end_of_turn>|"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "|<bos>|",
+  "cls_token": "|<cls>|",
+  "eos_token": "|<eos>|",
+  "extra_special_tokens": [
+    "|<system>|",
+    "|<user>|",
+    "|<assistant>|",
+    "|<thinking>|",
+    "|<end_of_thinking>|",
+    "|<end_of_turn>|"
+  ],
+  "mask_token": "|<mask>|",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "|<pad>|",
+  "sep_token": "|<sep>|",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "|<unknown>|"
+}