Upload TFBilma

config.json

@@ -9,6 +9,7 @@
   },
   "hidden_dropout_prob": 0.1,
   "hidden_size": 512,
+  "include_top": true,
   "model_type": "bilma",
   "num_attention_heads": 4,
   "num_hidden_layers": 2,

configuration_bilma.py

@@ -6,6 +6,7 @@ class BilmaConfig(PretrainedConfig):
     def __init__(
         self,
         weights="spanish",
+        include_top=True,
         num_attention_heads: int = 4,
         num_hidden_layers: int = 2,
         seq_max_length: int = 280,
@@ -15,9 +16,10 @@ class BilmaConfig(PretrainedConfig):
         **kwargs,
     ):
         if weights not in ["spanish", ""]:
-            raise ValueError(f"`weights` must be 'spanish' or '', got {weights}.")
+            raise ValueError(f"`weights` must be 'spanish', got {weights}.")
         if weights == "spanish":
             self.weights = weights
+            self.include_top = include_top
             self.num_attention_heads = 4
             self.num_hidden_layers = 2
             self.seq_max_length = 280
@@ -28,6 +30,7 @@ class BilmaConfig(PretrainedConfig):
             return
 
         self.weights = weights
+        self.include_top = include_top
         self.num_attention_heads = num_attention_heads
         self.num_hidden_layers = num_hidden_layers
         self.seq_max_length = seq_max_length

modeling_bilma.py

@@ -9,7 +9,7 @@ from typing import Dict
 import re
 import unicodedata
 
-from .configuration_bilma import BilmaConfig
+from configuration_bilma import BilmaConfig
 
 # copied from preprocessing.py
 BLANK = ' '
@@ -49,7 +49,8 @@ class TFBilma(TFPreTrainedModel):
                            num_heads=config.num_attention_heads,
                            ff_dim=config.hidden_size,
                            vocab_size=config.vocab_size,
-                           rate=config.hidden_dropout_prob)
+                           rate=config.hidden_dropout_prob,
+                           include_top = config.include_top)
             
     @property
     def dummy_inputs(self) -> Dict[str, tf.Tensor]:
@@ -486,14 +487,17 @@ def accuracy_function(ignore_id=0):
         return tf.math.divide_no_nan(tf.reduce_sum(accuracies), tf.reduce_sum(mask))
     return acc_mlm
 
-def bilma(num_enc=6, embed_dim=300, max_length=50, num_heads=6, ff_dim=512, vocab_size=9739, rate=0.1):
+def bilma(num_enc=6, embed_dim=300, max_length=50, num_heads=6, ff_dim=512, vocab_size=9739, rate=0.1, include_top=True):
     capt_inputs_ids = Input(shape=(max_length, ), name='input_ids')
     capt_embedding = Embedding(vocab_size, embed_dim, mask_zero=False, name="bilma/embedding")
     capt_inputs = capt_embedding(capt_inputs_ids)
     
     enc = Encoder(num_enc, embed_dim, max_length, num_heads, ff_dim, rate=rate, name="bilma/encoder")
     enc_output = enc(capt_inputs)
-    fin_output = Dense(vocab_size, use_bias=True, name="bilma/dense_final")(enc_output)
+    if include_top:
+        fin_output = Dense(vocab_size, use_bias=True, name="bilma/dense_final")(enc_output)
+    else:
+        fin_output = enc_output
     
     caption_model = Model(inputs=capt_inputs_ids, outputs=[fin_output], name="bilma_model")
     return caption_model

tf_model.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae6a05836d990960fb2c4a9fa4616d18e0c396c2e075f85cc98d1ab8aecdc7a0
+oid sha256:6d31e357973be9bf86a3676237280b3ffe852ac994efd62d6eb67e06e36cd039
 size 156564220