INT4 DFlash checkpoint weights update

config.json

@@ -4,7 +4,7 @@
   "hidden_act": "silu",
   "hidden_size": 3072,
   "intermediate_size": 12288,
-  "max_position_embeddings": 1048576,
+  "max_position_embeddings": 262144,
   "model_type": "laguna",
   "num_attention_heads": 72,
   "num_hidden_layers": 6,
@@ -21,6 +21,18 @@
     "sliding_attention"
   ],
   "rope_theta": 500000.0,
+  "partial_rotary_factor": 0.5,
+  "rope_parameters": {
+    "rope_type": "yarn",
+    "factor": 32.0,
+    "rope_theta": 500000.0,
+    "partial_rotary_factor": 0.5,
+    "original_max_position_embeddings": 8192,
+    "extrapolation_factor": 1.0,
+    "attn_factor": 1.0,
+    "beta_fast": 32.0,
+    "beta_slow": 1.0
+  },
   "gating": "per-head",
   "architectures": [
     "DFlashLagunaForCausalLM"

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9665e30bbced996011d1a3f8dcc392af4ea5463fc8a469cdc7019f2795a24b5
+oid sha256:0850e39b5c079a9f1a9bafed729a4545b088a91876541d010d871f6d6d8bf909
 size 2229962896