Upload Gemma3ForCausalLM

config.json

@@ -7,7 +7,7 @@
   "attention_dropout": 0.0,
   "attn_logit_softcapping": null,
   "bos_token_id": 2,
-  "cache_implementation": "hybrid",
+  "cache_implementation": "sliding_window",
   "dtype": "float32",
   "eos_token_id": [
     1,
@@ -47,21 +47,29 @@
     "sliding_attention",
     "sliding_attention"
   ],
-  "max_position_embeddings": 32768,
+  "max_position_embeddings": 262144,
   "model_type": "gemma3_text",
   "num_attention_heads": 4,
   "num_hidden_layers": 26,
   "num_key_value_heads": 1,
   "pad_token_id": 0,
   "query_pre_attn_scalar": 256,
-  "rms_norm_eps": 1e-06,
+  "rms_norm_eps": 1e-05,
   "rope_local_base_freq": 10000,
-  "rope_scaling": null,
-  "rope_theta": 1000000,
+  "rope_scaling": {
+    "beta_fast": 1.0,
+    "beta_slow": 1.0,
+    "factor": 8.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "yarn"
+  },
+  "rope_theta": 50000.0,
   "sliding_window": 512,
   "sliding_window_pattern": 6,
   "transformers_version": "4.57.1",
   "use_bidirectional_attention": false,
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 262149
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ac75794a439d8d467c9c698046279ac8fcdf05affe544f7e98a3a802d9b140b
+oid sha256:54520fb0553b6c841d9cabe4859be156b7d9dbbadde4fc0baaa2fbffb7d17f53
 size 3999606000