Upload folder using huggingface_hub

README.md

@@ -0,0 +1,86 @@
+---
+library_name: transformers
+license: mit
+base_model: saracandu/stlenc-new-temp0.3
+tags:
+- generated_from_trainer
+model-index:
+- name: stlenc-new-temp0.3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# stlenc-new-temp0.3
+
+This model is a fine-tuned version of [saracandu/stlenc-new-temp0.3](https://huggingface.co/saracandu/stlenc-new-temp0.3) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3740
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 128
+- eval_batch_size: 128
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 512
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- num_epochs: 10
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 2.4093        | 0.3639 | 250  | 2.5770          |
+| 2.4426        | 0.7278 | 500  | 2.5822          |
+| 2.4486        | 1.0917 | 750  | 2.5639          |
+| 2.4435        | 1.4556 | 1000 | 2.5948          |
+| 2.4516        | 1.8195 | 1250 | 2.6232          |
+| 2.4462        | 2.1834 | 1500 | 2.5765          |
+| 2.4458        | 2.5473 | 1750 | 2.6115          |
+| 2.4496        | 2.9112 | 2000 | 2.6147          |
+| 2.4366        | 3.2751 | 2250 | 2.6214          |
+| 2.445         | 3.6390 | 2500 | 2.6100          |
+| 2.4477        | 4.0029 | 2750 | 2.6061          |
+| 2.4428        | 4.3668 | 3000 | 2.5868          |
+| 2.4442        | 4.7307 | 3250 | 2.6178          |
+| 2.3614        | 5.0946 | 3500 | 2.5041          |
+| 1.4114        | 5.4585 | 3750 | 1.9460          |
+| 1.1683        | 5.8224 | 4000 | 1.6975          |
+| 1.1236        | 6.1863 | 4250 | 1.5663          |
+| 1.1069        | 6.5502 | 4500 | 1.5173          |
+| 1.0834        | 6.9141 | 4750 | 1.5008          |
+| 1.0683        | 7.2780 | 5000 | 1.4697          |
+| 1.0608        | 7.6419 | 5250 | 1.4206          |
+| 1.0412        | 8.0058 | 5500 | 1.4282          |
+| 1.0346        | 8.3697 | 5750 | 1.3987          |
+| 1.0305        | 8.7336 | 6000 | 1.3890          |
+| 1.0282        | 9.0975 | 6250 | 1.3912          |
+| 1.0199        | 9.4614 | 6500 | 1.3683          |
+| 1.0209        | 9.8253 | 6750 | 1.3740          |
+
+
+### Framework versions
+
+- Transformers 4.57.3
+- Pytorch 2.9.1+cu128
+- Datasets 4.4.2
+- Tokenizers 0.22.1

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "STLEncoderModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_stlenc.STLEncoderConfig",
+    "AutoModel": "modeling_stlenc.STLEncoderModel",
+    "AutoTokenizer": [
+      "tokenizer_stlenc.STLTokenizer",
+      null
+    ]
+  },
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "embedding_dim_target": 1024,
+  "eos_token_id": 3,
+  "hidden_size": 1024,
+  "intermediate_size": 4096,
+  "max_position_embeddings": 512,
+  "model_type": "stl_encoder",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.3",
+  "vocab_size": 35
+}

configuration_stlenc.py

@@ -0,0 +1,23 @@
+from transformers import PretrainedConfig
+
+class STLEncoderConfig(PretrainedConfig):
+    model_type = "stl_encoder"
+    def __init__(
+        self,
+        vocab_size=35,
+        hidden_size=1024,
+        num_hidden_layers=12,
+        num_attention_heads=16,
+        intermediate_size=4096,
+        max_position_embeddings=512,
+        embedding_dim_target=1024,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.embedding_dim_target = embedding_dim_target

modeling_stlenc.py

@@ -0,0 +1,52 @@
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel
+from .configuration_stlenc import STLEncoderConfig
+
+class STLEncoderModel(PreTrainedModel):
+    config_class = STLEncoderConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.embeddings = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=config.hidden_size,
+            nhead=config.num_attention_heads,
+            dim_feedforward=config.intermediate_size,
+            activation="gelu", # GELU è standard per i Transformer moderni
+            batch_first=True
+        )
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=config.num_hidden_layers)
+        
+        # --- POTENZIAMENTO ARCHITETTURALE ---
+        # Creiamo una testa di proiezione profonda (MLP)
+        self.projector = nn.Sequential(
+            nn.Linear(config.hidden_size, config.hidden_size),
+            nn.GELU(),
+            nn.LayerNorm(config.hidden_size),
+            nn.Dropout(0.1), # Aiuta a non overfittare sulle costanti numeriche
+            nn.Linear(config.hidden_size, config.hidden_size // 2),
+            nn.GELU(),
+            nn.Linear(config.hidden_size // 2, config.embedding_dim_target)
+        )
+        # ------------------------------------
+        
+        self.post_init()
+
+    def forward(self, input_ids, attention_mask=None, **kwargs):
+        batch_size, seq_length = input_ids.size()
+        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)
+        position_ids = position_ids.unsqueeze(0).expand(batch_size, seq_length)
+        x = self.embeddings(input_ids) + self.position_embeddings(position_ids)
+        # Maschera per il padding (TransformerEncoder si aspetta True dove NON deve guardare)
+        padding_mask = (attention_mask == 0) if attention_mask is not None else None
+        # Encoding delle sequenze
+        sequence_output = self.encoder(x, src_key_padding_mask=padding_mask)
+        # Prendiamo il CLS (indice 0)
+        cls_token = sequence_output[:, 0, :]
+        # Passiamo per la testa di proiezione non-lineare
+        # Rimuoviamo la Tanh finale per lasciare che il kernel scalare respiri
+        pooled_output = self.projector(cls_token)
+        return pooled_output

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "/s",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "s",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "pad",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "unk",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,50 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "unk",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "pad",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "/s",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "s",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer_stlenc.STLTokenizer",
+      null
+    ]
+  },
+  "bos_token": "/s",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "s",
+  "extra_special_tokens": {},
+  "model_max_length": 512,
+  "pad_token": "pad",
+  "tokenizer_class": "STLTokenizer",
+  "unk_token": "unk"
+}

tokenizer_stlenc.py

@@ -0,0 +1,93 @@
+import json
+import os
+import torch
+from typing import Any, Dict, List, Optional, Tuple, Union
+from transformers import PreTrainedTokenizer, AutoTokenizer
+
+class STLTokenizer(PreTrainedTokenizer):
+    model_type = "stl_encoder"
+
+    def __init__(
+        self, 
+        vocab_file="vocab.json", 
+        unk_token="unk", 
+        pad_token="pad",
+        bos_token="/s", 
+        eos_token="s", 
+        model_max_length=512, 
+        **kwargs
+    ):
+        current_dir = os.path.dirname(__file__)
+        full_vocab_path = os.path.join(current_dir, vocab_file)
+        
+        if not os.path.exists(full_vocab_path):
+            from huggingface_hub import hf_hub_download
+            try:
+                full_vocab_path = hf_hub_download("saracandu/stlenc", vocab_file)
+            except:
+                full_vocab_path = vocab_file
+
+        with open(full_vocab_path, "r", encoding="utf-8") as f:
+            self.vocab = json.load(f)
+
+        self.id_to_token = {v: k for k, v in self.vocab.items()}
+        
+        super().__init__(
+            unk_token=unk_token, 
+            pad_token=pad_token, 
+            bos_token=bos_token, 
+            eos_token=eos_token, 
+            model_max_length=model_max_length, 
+            **kwargs
+        )
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self.vocab)
+
+    def get_vocab(self) -> Dict[str, int]:
+        return dict(self.vocab)
+
+    def _tokenize(self, text: str) -> List[str]:
+        text = f'{self.bos_token} {text} {self.eos_token}'.replace(' ', '@')
+        
+        tokens = []
+        i = 0
+        while i < len(text):
+            best_match = None
+            for j in range(min(i + 50, len(text)), i, -1):
+                subtoken = text[i:j]
+                if subtoken in self.vocab:
+                    best_match = subtoken
+                    break
+            
+            if best_match:
+                tokens.append(best_match)
+                i += len(best_match)
+            else:
+                tokens.append(self.unk_token)
+                i += 1
+        return tokens
+
+    def _convert_token_to_id(self, token: str) -> int:
+        return self.vocab.get(token, self.vocab.get(self.unk_token))
+
+    def _convert_id_to_token(self, index: int) -> str:
+        return self.id_to_token.get(index, self.unk_token)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            os.makedirs(save_directory)
+            
+        prefix = filename_prefix if filename_prefix is not None else ""
+        vocab_file = os.path.join(save_directory, prefix + "vocab.json")
+        
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            json.dump(self.vocab, f, indent=2, ensure_ascii=False)
+            
+        return (vocab_file,)
+
+try:
+    AutoTokenizer.register("stl_encoder", STLTokenizer)
+except Exception:
+    pass

vocab.json

@@ -0,0 +1,37 @@
+{
+  "unk": 0,
+  "pad": 1,
+  "/s": 2,
+  "s": 3,
+  "(": 4,
+  ")": 5,
+  "always": 6,
+  "eventually": 7,
+  "until": 8,
+  "and": 9,
+  "or": 10,
+  "not": 11,
+  ">=": 12,
+  "<=": 13,
+  ">": 14,
+  "<": 15,
+  "=": 16,
+  "x_": 17,
+  "[": 18,
+  "]": 19,
+  ",": 20,
+  "inf": 21,
+  "-": 22,
+  ".": 23,
+  "0": 24,
+  "1": 25,
+  "2": 26,
+  "3": 27,
+  "4": 28,
+  "5": 29,
+  "6": 30,
+  "7": 31,
+  "8": 32,
+  "9": 33,
+  "@": 34
+}