Upload folder using huggingface_hub

config.json

@@ -1,19 +1,19 @@
 {
   "_sliding_window_pattern": 6,
   "architectures": [
-    "Gemma3EncoderForMaskedLM"
+    "Gemma3EncoderModel"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "attn_logit_softcapping": null,
   "auto_map": {
-    "AutoModel": "modeling_gemma3_biencoder.Gemma3EncoderForMaskedLM",
+    "AutoModel": "modeling_gemma3_biencoder.Gemma3EncoderModel",
     "AutoModelForMaskedLM": "modeling_gemma3_biencoder.Gemma3EncoderForMaskedLM",
     "AutoModelForSequenceClassification": "modeling_gemma3_biencoder.Gemma3EncoderForSequenceClassification",
     "AutoModelForTokenClassification": "modeling_gemma3_biencoder.Gemma3EncoderForTokenClassification"
   },
   "bos_token_id": 2,
-  "dtype": "bfloat16",
+  "dtype": "float32",
   "eos_token_id": 1,
   "final_logit_softcapping": null,
   "head_dim": 256,
@@ -41,7 +41,7 @@
     "sliding_attention",
     "full_attention"
   ],
-  "max_position_embeddings": 2048,
+  "max_position_embeddings": 32768,
   "model_type": "gemma3_text",
   "num_attention_heads": 4,
   "num_hidden_layers": 18,
@@ -56,5 +56,6 @@
   "transformers_version": "4.57.3",
   "use_bidirectional_attention": true,
   "use_cache": false,
-  "vocab_size": 262145
-}
+  "vocab_size": 262145,
+  "attn_implementation": null
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b85687659049cd09b6da529da4b4190ba7c00528c17d7065ce8e40ac850a33da
-size 536224808
+oid sha256:e9dc0cc7558bc128b11280fbdbacf630a260a637110ad69d3de2f03ca9650093
+size 1072422288

modeling_gemma3_biencoder.py

@@ -11,61 +11,41 @@ from transformers.models.gemma3.modeling_gemma3 import (
     Gemma3TextModel,
 )
 
+class Gemma3EncoderModel(Gemma3PreTrainedModel):
+    config_class = Gemma3TextConfig
+    base_model_prefix = "encoder"
+
+    def __init__(self, config):
+        cfg = copy.deepcopy(config)
+        if hasattr(cfg, "use_bidirectional_attention"):
+            cfg.use_bidirectional_attention = True
+        cfg.use_cache = False
+        super().__init__(cfg)
+        self.encoder = Gemma3TextModel(cfg)
+        self.post_init()
+
+    def forward(self, input_ids=None, attention_mask=None, position_ids=None,
+                inputs_embeds=None, output_attentions=None, output_hidden_states=None,
+                return_dict=True, **kwargs):
+        return self.encoder(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=False,
+            is_causal=False,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            **kwargs,
+        )
+
 class Gemma3EncoderForMaskedLM(Gemma3PreTrainedModel):
     config_class = Gemma3TextConfig
     base_model_prefix = "encoder"
     _tied_weights_keys = ["lm_head.weight"]
     _keys_to_ignore_on_load_missing = [r"lm_head\.weight"]
 
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
-        """Override to preserve sliding_window from config.json."""
-        import json
-        import os
-        
-        # Read original sliding_window from config.json before it gets modified
-        original_sliding_window = None
-        try:
-            # Use transformers utility to resolve config path (handles both local and Hub)
-            from transformers.utils import CONFIG_NAME
-            from transformers.utils.hub import cached_file
-            
-            config_path = None
-            if os.path.isdir(pretrained_model_name_or_path):
-                # Local path
-                config_path = os.path.join(pretrained_model_name_or_path, CONFIG_NAME)
-            else:
-                # Hub path - this will download/cache if needed
-                try:
-                    config_path = cached_file(
-                        pretrained_model_name_or_path,
-                        CONFIG_NAME,
-                        cache_dir=kwargs.get("cache_dir"),
-                        force_download=kwargs.get("force_download", False),
-                        resume_download=kwargs.get("resume_download", False),
-                    )
-                except Exception:
-                    pass
-            
-            if config_path and os.path.exists(config_path):
-                with open(config_path, "r", encoding="utf-8") as f:
-                    config_dict = json.load(f)
-                    original_sliding_window = config_dict.get("sliding_window")
-        except Exception:
-            # If we can't read the config, continue anyway
-            pass
-        
-        # Load model normally
-        model = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        
-        # Restore original sliding_window if it was modified by Gemma3TextModel
-        if original_sliding_window is not None:
-            current_sw = getattr(model.config, "sliding_window", None)
-            if current_sw != original_sliding_window:
-                model.config.sliding_window = original_sliding_window
-        
-        return model
-
     def __init__(self, config: Gemma3TextConfig):
         cfg = copy.deepcopy(config)
         if hasattr(cfg, "use_bidirectional_attention"):