Upload 5 files

config.json

@@ -0,0 +1,14 @@
+{
+  "vocab_size": 24576,
+  "d_model": 512,
+  "d_state": 256,
+  "num_scales": 4,
+  "scale_min": 32,
+  "scale_factor": 4,
+  "rope_base": 10000.0,
+  "dropout": 0.05,
+  "use_input_gate": true,
+  "tie_embeddings": false,
+  "mlp_hidden": 2048,
+  "max_seq_len": 4096
+}

metadata.json

@@ -0,0 +1,31 @@
+{
+  "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)",
+  "num_samples": 200000,
+  "epochs": 1,
+  "total_opt_steps": 3246,
+  "final_loss": 4.929484256505966,
+  "eval_loss": 4.898339782714844,
+  "perplexity": 134.0670144420039,
+  "batch_size": 4,
+  "grad_accum": 8,
+  "effective_batch": 32,
+  "max_seq_len": 2048,
+  "lr_max": 0.0005,
+  "lr_min": 5e-05,
+  "warmup_steps": 100,
+  "tokenizer": "BPE-24576-FineWeb",
+  "vocab_size": 24576,
+  "num_params": 66878464,
+  "d_model": 512,
+  "d_state": 256,
+  "num_scales": 4,
+  "scale_lengths": [
+    32,
+    128,
+    512,
+    2048
+  ],
+  "mlp_hidden": 2048,
+  "training_mode": "causal_lm_packing_fp16_grad_accum",
+  "training_time_s": 5868.622669219971
+}

model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b76176b55e9dba0d7866cb35b768e259ba5430f7ebdead2b04ac60702251a63
+size 267525795

tokenizer_info.json

@@ -0,0 +1,9 @@
+{
+  "type": "Byte-Level BPE",
+  "vocab_size": 24576,
+  "pad_id": 0,
+  "bos_id": 2,
+  "eos_id": 3,
+  "max_seq_len": 2048,
+  "trained_on": "FineWeb-Edu 60,000 muestras"
+}