FP8 DFlash checkpoint weights update

config.json

@@ -4,7 +4,7 @@
   "hidden_act": "silu",
   "hidden_size": 3072,
   "intermediate_size": 12288,
-  "max_position_embeddings": 1048576,
+  "max_position_embeddings": 262144,
   "model_type": "laguna",
   "num_attention_heads": 72,
   "num_hidden_layers": 6,
@@ -20,6 +20,18 @@
     "sliding_attention",
     "sliding_attention"
   ],
+  "rope_theta": 500000.0,
+  "partial_rotary_factor": 0.5,
+  "rope_parameters": {
+    "rope_type": "yarn",
+    "factor": 32.0,
+    "original_max_position_embeddings": 8192
+  },
+  "gating": "per-head",
+  "architectures": [
+    "DFlashLagunaForCausalLM"
+  ],
+  "num_experts": 0,
   "sliding_windows": [
     512,
     512,
@@ -28,12 +40,6 @@
     512,
     512
   ],
-  "rope_theta": 500000.0,
-  "gating": "per-head",
-  "architectures": [
-    "DFlashLagunaForCausalLM"
-  ],
-  "num_experts": 0,
   "draft_vocab_size": 100352,
   "torch_dtype": "bfloat16",
   "eagle_aux_hidden_state_layer_ids": [

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3026487dc6a6b22a78f15ca1433eae69e85b77b19629daf135681683de768f6e
+oid sha256:933dcd552289da3f7489f358964a97160c8175ee109d1b18543fdcd2156651b9
 size 2229962896