trying the autoconfig

config.json

@@ -10,7 +10,7 @@
   ],
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoProcessor": "processing_img2html.Img2HTMLProcessor",
+    "AutoConfig": "confgiration_img2html.Img2HTMLConfig",
     "AutoModelForCausalLM": "modeling_img2html.Img2HTMLForVisionText2Text"
   },
   "bos_token_id": 1,

configuration_img2html.py

@@ -24,16 +24,15 @@ MISTRAL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 }
 
 
-class VMistralVisionConfig(PretrainedConfig):
+class Img2HTMLVisionConfig(PretrainedConfig):
     r"""
     """
-    model_type = "vmistral"
+    model_type = "img2html"
 
     def __init__(
         self,
         hidden_size=768,
         intermediate_size=3072,
-        projection_dim=512,
         num_hidden_layers=12,
         num_attention_heads=12,
         num_channels=3,
@@ -51,7 +50,6 @@ class VMistralVisionConfig(PretrainedConfig):
 
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
-        self.projection_dim = projection_dim
         self.num_hidden_layers = num_hidden_layers
         self.num_attention_heads = num_attention_heads
         self.num_channels = num_channels
@@ -65,7 +63,7 @@ class VMistralVisionConfig(PretrainedConfig):
         self._flash_attn_2_enabled = _flash_attn_2_enabled
 
 
-class VMistralPerceiverConfig(PretrainedConfig):
+class Img2HTMLPerceiverConfig(PretrainedConfig):
     r"""
     TThis is the configuration class to store the configuration of a [`MistralModel`]. It is used to instantiate an
     Mistral model according to the specified arguments, defining the model architecture. Instantiating a configuration
@@ -91,7 +89,7 @@ class VMistralPerceiverConfig(PretrainedConfig):
         qk_layer_norms_perceiver (`bool`, *optional*, defaults to `False`):
             Whether or not to use qk layer norms in perceiver
     """
-    model_type = "vmistral"
+    model_type = "img2html"
 
     def __init__(
         self,
@@ -111,7 +109,7 @@ class VMistralPerceiverConfig(PretrainedConfig):
         super().__init__(**kwargs)
 
 
-class VMistralConfig(PretrainedConfig):
+class Img2HTMLConfig(PretrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`MistralModel`]. It is used to instantiate an
     Mistral model according to the specified arguments, defining the model architecture. Instantiating a configuration
@@ -203,7 +201,7 @@ class VMistralConfig(PretrainedConfig):
     >>> # Accessing the model configuration
     >>> configuration = model.config
     ```"""
-    model_type = "vmistral"
+    model_type = "img2html"
     is_composition = False
 
     def __init__(
@@ -282,17 +280,17 @@ class VMistralConfig(PretrainedConfig):
         self.attention_dropout = attention_dropout
 
         if perceiver_config is None:
-            self.perceiver_config = VMistralPerceiverConfig()
+            self.perceiver_config = Img2HTMLPerceiverConfig()
         elif isinstance(perceiver_config, dict):
-            self.perceiver_config = VMistralPerceiverConfig(**perceiver_config)
-        elif isinstance(perceiver_config, VMistralPerceiverConfig):
+            self.perceiver_config = Img2HTMLPerceiverConfig(**perceiver_config)
+        elif isinstance(perceiver_config, Img2HTMLPerceiverConfig):
             self.perceiver_config = perceiver_config
 
         if vision_config is None:
-            self.vision_config = VMistralVisionConfig()
+            self.vision_config = Img2HTMLVisionConfig()
         elif isinstance(vision_config, dict):
-            self.vision_config = VMistralVisionConfig(**vision_config)
-        elif isinstance(vision_config, VMistralVisionConfig):
+            self.vision_config = Img2HTMLVisionConfig(**vision_config)
+        elif isinstance(vision_config, Img2HTMLVisionConfig):
             self.vision_config = vision_config
 
         super().__init__(

modeling_img2html.py

@@ -43,7 +43,7 @@ from transformers import PreTrainedModel
 from transformers.utils import logging
 from transformers.modeling_outputs import ModelOutput
 
-from .configuration_img2html import VMistralConfig
+from .configuration_img2html import Img2HTMLConfig
 from .vision import SiglipVisionModel
 
 
@@ -55,7 +55,7 @@ if is_flash_attn_2_available():
 
 logger = logging.get_logger(__name__)
 
-_CONFIG_FOR_DOC = "VMistralConfig"
+_CONFIG_FOR_DOC = "Img2HTMLConfig"
 
 IMG2HTML_PRETRAINED_MODEL_ARCHIVE_LIST = [
     "HuggingFaceM4/Img2HTML"
@@ -698,7 +698,7 @@ class MistralAttention(nn.Module):
     and "Generating Long Sequences with Sparse Transformers".
     """
 
-    def __init__(self, config: VMistralConfig, qk_layer_norms: bool = False):
+    def __init__(self, config: Img2HTMLConfig, qk_layer_norms: bool = False):
         super().__init__()
         self.config = config
         self.hidden_size = config.hidden_size
@@ -1093,7 +1093,7 @@ class MistralFlashAttention2(MistralAttention):
 
 
 class MistralDecoderLayer(nn.Module):
-    def __init__(self, config: VMistralConfig):
+    def __init__(self, config: Img2HTMLConfig):
         super().__init__()
         self.hidden_size = config.hidden_size
         self.self_attn = (
@@ -1176,7 +1176,7 @@ MISTRAL_START_DOCSTRING = r"""
     and behavior.
 
     Parameters:
-        config ([`VMistralConfig`]):
+        config ([`Img2HTMLConfig`]):
             Model configuration class with all the parameters of the model. Initializing with a config file does not
             load the weights associated with the model, only the configuration. Check out the
             [`~PreTrainedModel.from_pretrained`] method to load the model weights.
@@ -1188,7 +1188,7 @@ MISTRAL_START_DOCSTRING = r"""
     MISTRAL_START_DOCSTRING,
 )
 class VMistralPreTrainedModel(PreTrainedModel):
-    config_class = VMistralConfig
+    config_class = Img2HTMLConfig
     base_model_prefix = "model"
     supports_gradient_checkpointing = True
     _no_split_modules = ["MistralDecoderLayer"]
@@ -1290,10 +1290,10 @@ class VMistralModel(VMistralPreTrainedModel):
     Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MistralDecoderLayer`]
 
     Args:
-        config: VMistralConfig
+        config: Img2HTMLConfig
     """
 
-    def __init__(self, config: VMistralConfig, vision_model=None):
+    def __init__(self, config: Img2HTMLConfig, vision_model=None):
         super().__init__(config)
         self.config = config
         self.padding_idx = config.pad_token_id