Upload STLForCausalLM

config.json

@@ -12,14 +12,14 @@
   "bos_token_id": 2,
   "d_model": 32,
   "decoder_attention_heads": 16,
-  "decoder_ffn_dim": 512,
+  "decoder_ffn_dim": 4096,
   "decoder_layerdrop": 0.0,
   "decoder_layers": 12,
   "decoder_start_token_id": 3,
   "decoder_vocab_size": 35,
   "dropout": 0.1,
   "encoder_attention_heads": 16,
-  "encoder_ffn_dim": 512,
+  "encoder_ffn_dim": 4096,
   "encoder_layerdrop": 0.0,
   "encoder_layers": 12,
   "eos_token_id": 3,

configuration_stldec.py

@@ -12,10 +12,10 @@ class STLConfig(PretrainedConfig):
         decoder_vocab_size=None, # unused
         max_position_embeddings=512,
         encoder_layers=12,
-        encoder_ffn_dim=512,
+        encoder_ffn_dim=4096,
         encoder_attention_heads=16,
         decoder_layers=12,
-        decoder_ffn_dim=512,
+        decoder_ffn_dim=4096,
         decoder_attention_heads=16,
         encoder_layerdrop=0.0,
         decoder_layerdrop=0.0,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44cd4bccc39663b6347ee8be1dd926750bc659e20ac78edbbe696958411d3010
-size 2093488
+oid sha256:79370eb12d442094d4a253967bc9f9050d45c3f39f3cff5f73f54a8ae4dfb137
+size 13275880

modeling_stldec.py

@@ -2139,7 +2139,7 @@ class STLForCausalLM(STLModel, GenerationMixin):
         loss = None
         if labels is not None:
             labels = labels.to(logits.device)
-            loss_fct = nn.CrossEntropyLoss()
+            loss_fct = CrossEntropyLoss()
             loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1))
 
         if not return_dict: