poolside
/

Laguna-XS.2-NVFP4

Text Generation

8-bit precision

compressed-tensors

Model card Files Files and versions

joerowell commited on 6 days ago

Commit

d2a5f9c

·

1 Parent(s): 514daf4

increase context length to 256k

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -14,7 +14,7 @@
   "num_attention_heads": 48,
   "num_key_value_heads": 8,
   "head_dim": 128,
-  "max_position_embeddings": 131072,
   "attention_bias": false,
   "attention_dropout": 0.0,
   "rms_norm_eps": 1e-06,
@@ -38,7 +38,7 @@
     "full_attention": {
       "rope_theta": 500000.0,
       "rope_type": "yarn",
-      "factor": 32.0,
       "original_max_position_embeddings": 4096,
       "beta_slow": 1.0,
       "beta_fast": 64.0,

   "num_attention_heads": 48,
   "num_key_value_heads": 8,
   "head_dim": 128,
+  "max_position_embeddings": 262144,
   "attention_bias": false,
   "attention_dropout": 0.0,
   "rms_norm_eps": 1e-06,
     "full_attention": {
       "rope_theta": 500000.0,
       "rope_type": "yarn",
+      "factor": 64.0,
       "original_max_position_embeddings": 4096,
       "beta_slow": 1.0,
       "beta_fast": 64.0,