Uploading the base model

config.json

@@ -0,0 +1,54 @@
+{
+  "sequence_len": 8192,
+  "vocab_size": 131072,
+  "n_layer": 16,
+  "n_head": 8,
+  "n_kv_head": 8,
+  "n_embd": 1024,
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "moe_layer_interval": 3,
+  "moe_group_size": 4,
+  "moe_expert_intermediate_size": 1792,
+  "moe_adjugate_intermediate_size": 0,
+  "moe_adjugate_scale": 0.05,
+  "moe_router_aux_loss_coef": 0.015,
+  "moe_router_bias_lr": 0.001,
+  "moe_activation_checkpoint": true,
+  "moe_capacity_factor": 0.75,
+  "rotary_scaling_type": "yarn",
+  "rotary_scale_factor": 4.0,
+  "residual_scale": -1.0,
+  "attn_dropout": 0.01,
+  "label_smoothing": 0.0,
+  "z_loss_weight": 0.0,
+  "use_flash_attention": true,
+  "domain_router_dim": 32,
+  "num_domain_tags": 128,
+  "domain_router_features": {
+    "dataset": {
+      "capacity": 128,
+      "mode": "one_hot"
+    },
+    "quality": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "specialty": {
+      "capacity": 64,
+      "mode": "one_hot"
+    },
+    "modality": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "language": {
+      "capacity": 32,
+      "mode": "one_hot"
+    },
+    "origin": {
+      "capacity": 8,
+      "mode": "one_hot"
+    }
+  }
+}

meta.json

@@ -0,0 +1,153 @@
+{
+  "step": 45444,
+  "val_bpb": 0.6811688407173832,
+  "model_config": {
+    "sequence_len": 8192,
+    "vocab_size": 131072,
+    "n_layer": 16,
+    "n_head": 8,
+    "n_kv_head": 8,
+    "n_embd": 1024,
+    "moe_num_experts": 8,
+    "moe_top_k": 2,
+    "moe_layer_interval": 3,
+    "moe_group_size": 4,
+    "moe_expert_intermediate_size": 1792,
+    "moe_adjugate_intermediate_size": 0,
+    "moe_adjugate_scale": 0.05,
+    "moe_router_aux_loss_coef": 0.015,
+    "moe_router_bias_lr": 0.001,
+    "moe_activation_checkpoint": true,
+    "moe_capacity_factor": 0.75,
+    "rotary_scaling_type": "yarn",
+    "rotary_scale_factor": 4.0,
+    "residual_scale": -1.0,
+    "attn_dropout": 0.01,
+    "label_smoothing": 0.0,
+    "z_loss_weight": 0.0,
+    "use_flash_attention": true,
+    "domain_router_dim": 32,
+    "num_domain_tags": 128,
+    "domain_router_features": {
+      "dataset": {
+        "capacity": 128,
+        "mode": "one_hot"
+      },
+      "quality": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "specialty": {
+        "capacity": 64,
+        "mode": "one_hot"
+      },
+      "modality": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "language": {
+        "capacity": 32,
+        "mode": "one_hot"
+      },
+      "origin": {
+        "capacity": 8,
+        "mode": "one_hot"
+      }
+    }
+  },
+  "user_config": {
+    "run": "continous_pretraining",
+    "device_type": "",
+    "depth": 16,
+    "max_seq_len": 8192,
+    "moe_num_experts": 8,
+    "moe_top_k": 2,
+    "moe_layer_interval": 3,
+    "moe_group_size": 4,
+    "moe_expert_intermediate_size": 1792,
+    "moe_adjugate_intermediate_size": 0,
+    "moe_adjugate_scale": 0.05,
+    "moe_router_aux_loss_coef": 0.015,
+    "moe_router_bias_lr": 0.001,
+    "moe_activation_checkpoint": true,
+    "moe_capacity_factor": 0.75,
+    "moe_router_bias_reset_interval": 0,
+    "domain_router_dim": 32,
+    "domain_tag_capacity": 128,
+    "quality_tag_capacity": 32,
+    "specialty_tag_capacity": 64,
+    "modality_tag_capacity": 32,
+    "language_tag_capacity": 32,
+    "origin_tag_capacity": 8,
+    "domain_router_feature_modes": "dataset:one_hot,quality:one_hot,specialty:one_hot,modality:one_hot,language:one_hot,origin:one_hot",
+    "rotary_scaling_type": "yarn",
+    "rotary_scale_factor": 4.0,
+    "residual_scale": -1.0,
+    "attn_dropout": 0.01,
+    "label_smoothing": 0.0,
+    "z_loss_weight": 0.0,
+    "use_flash_attention": true,
+    "use_distributed_muon": false,
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20,
+    "device_batch_size": 1,
+    "total_batch_size": 524288,
+    "max_grad_accum_steps": 0,
+    "embedding_lr": 0.2,
+    "unembedding_lr": 0.004,
+    "weight_decay": 0.0,
+    "freeze_embedding_optimizers": false,
+    "matrix_lr": 0.02,
+    "grad_clip": 1.0,
+    "warmup_ratio": 0.0,
+    "warmdown_ratio": 0.2,
+    "final_lr_frac": 0.0,
+    "eval_every": 500,
+    "eval_tokens": "[redacted]",
+    "core_metric_every": 1000,
+    "core_metric_max_per_task": 2500,
+    "sample_every": 1000,
+    "micro_eval_every": 250,
+    "micro_eval_tokens": "[redacted]",
+    "skip_initial_eval": false,
+    "fast_dev_run": false,
+    "fast_dev_num_iterations": 200,
+    "fast_dev_max_grad_accum_steps": 2,
+    "fast_dev_eval_tokens_multiplier": "[redacted]",
+    "model_tag": "d16_cont",
+    "hf_repo_id": "harshad317/base_Medical_continuous",
+    "hf_repo_type": "model",
+    "hf_path_in_repo": "",
+    "hf_commit_message": "Uploading the base model",
+    "hf_private": false,
+    "hf_token": "[redacted]",
+    "hf_max_shard_size": "2gb",
+    "resume_from_checkpoint": "d16",
+    "resume_from_step": 22722,
+    "resume_load_optimizer": true,
+    "base_dataset_num_shards": 1250,
+    "train_dataset_mix": "",
+    "train_dataset_mix_path": "configs/train_dataset_mix.json",
+    "loader_dedup_window": 300000,
+    "loader_enable_length_bucketing": true,
+    "loader_bucket_bins": "512,1024,2048,4096,8192",
+    "loader_prefetch_batches": 2,
+    "loader_tokenizer_threads": "[redacted]",
+    "loader_tokenizer_batch_size": "[redacted]",
+    "train_single_pass": true,
+    "enable_length_curriculum": false,
+    "sequence_length_schedule": "0.25:512,0.50:1024,0.75:2048,1.0:4096",
+    "length_schedule_round_to": 32,
+    "enable_dataset_reweighting": true,
+    "dataset_reweight_interval": 2000,
+    "dataset_reweight_strength": 0.75,
+    "dataset_reweight_smoothing": 0.2,
+    "dataset_reweight_min_multiplier": 0.25,
+    "dataset_reweight_warmup_steps": 4000,
+    "memory_profile_interval": 500,
+    "dataset_mix_source": "/home/jupyter/nanochat/configs/train_dataset_mix.json"
+  },
+  "device_batch_size": 1,
+  "max_seq_len": 8192
+}

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a502f68c5bf94be29e88c56f45adce9cfe6e9894bc53ece50a9f76ce8f3416
+size 1845827576

model.safetensors.index.json

@@ -0,0 +1,228 @@
+{
+  "metadata": {
+    "total_size": 1845802880,
+    "num_shards": 1,
+    "step": 45444,
+    "format": "safetensors_state_dict"
+  },
+  "weight_map": {
+    "transformer.wte.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.2.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.3.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.4.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.5.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.6.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.7.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.8.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.9.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.10.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.11.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.12.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.13.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_bias": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.uniform_load": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.2.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.2.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.3.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.3.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.4.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.4.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.5.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.5.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.6.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.6.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.7.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.experts.7.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.0.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.0.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.1.w1.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.adjugate_experts.1.w2.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_scale_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.14.mlp.router_context_selection_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_q.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_k.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_v.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.attn.c_proj.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.mlp.c_fc.weight": "model-00001-of-00001.safetensors",
+    "transformer.h.15.mlp.c_proj.weight": "model-00001-of-00001.safetensors",
+    "lm_head.weight": "model-00001-of-00001.safetensors"
+  }
+}

token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657d33e4365804c866b1c9e8c065c0f870428a0ece1f3b0771533dd6a0dc6076
+size 525865

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d2e58be3e2ec81124dbf4941bbab6bd0715eca1a34e5a773953b60d6369e750
+size 1846082