june 2026 update

README.md

@@ -91,18 +91,19 @@ EUBERT is a pretrained BERT model that leverages a substantial corpus of documen
 
 ## Training procedure
 
-Dedicated Word Piece tokenizer vocabulary size 2**16, 
+Trained using the EuroVoc dataset. 
 
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 32
-- eval_batch_size: 32
+- learning_rate: 2e-4
+- train_batch_size: 8
+- eval_batch_size: 8
+- warmup_steps: 200
+- lora_alpha: 16
+- lora_dropout: 0.1
 - seed: 42
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: linear
-- num_epochs: 1.85
+- num_epochs: 1
 
 ### Framework versions
 

adapter_config.json

@@ -0,0 +1,43 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "nlpaueb/legal-bert-base-uncased",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "query",
+    "value"
+  ],
+  "target_parameters": null,
+  "task_type": "FEATURE_EXTRACTION",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5bc8eb3b718153088f03cc5fd984198f8f0c143b9936ea6bee3efadf5e5d65b
+size 1186328

inference.py

@@ -1,21 +1,47 @@
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+#!/usr/bin/env python3
+"""EuroBERT Inference - Predict Masked Tokens"""
 
-# Path of fine-tuned model
-model_path = "./fine_tuned_model"
+import torch
+import argparse
+from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline
+from peft import PeftModel
 
-# Load tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained(model_path)
-model = AutoModelForCausalLM.from_pretrained(model_path)
+BASE_MODEL = "nlpaueb/legal-bert-base-uncased"
+ADAPTER_PATH = "./model"
 
-# Create chatbot pipeline
-chatbot = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
-    device=0 if torch.cuda.is_available() else -1  # Use GPU if available
-)
 
-# Example usage
-prompt = "Hello, can you tell me some fun facts about european legislation?"
-response = chatbot(prompt, max_length=100, do_sample=True, temperature=0.7)
-print(response[0]['generated_text'])
+def main():
+    parser = argparse.ArgumentParser(description="EuroBERT - Predict Masked Tokens")
+    parser.add_argument("--text", type=str, default="The European [MASK] regulates digital platforms.",
+                       help="Text with [MASK] tokens to predict")
+    parser.add_argument("--top-k", type=int, default=5, help="Number of top predictions")
+    parser.add_argument("--adapter-path", type=str, default=ADAPTER_PATH, help="Path to LoRA adapter")
+    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu")
+    
+    args = parser.parse_args()
+    
+    # Load model
+    print(f"📦 Loading {BASE_MODEL}")
+    model = AutoModelForMaskedLM.from_pretrained(BASE_MODEL)
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+    
+    print(f"⚡ Loading LoRA adapter from {args.adapter_path}")
+    model = PeftModel.from_pretrained(model, args.adapter_path)
+    model.to(args.device)
+    model.eval()
+    
+    print(f"🎮 Device: {args.device}\n")
+    
+    # Predict masked tokens
+    pipe = pipeline("fill-mask", model=model, tokenizer=tokenizer)
+    results = pipe(args.text, top_k=args.top_k)
+    
+    print(f"📝 Input: {args.text}")
+    print(f"🎯 Top {args.top_k} predictions:")
+    for i, result in enumerate(results, 1):
+        print(f"   {i}. '{result['token_str']}' (score: {result['score']:.4f})")
+        print(f"      → {result['sequence']}")
+
+
+if __name__ == "__main__":
+    main()

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d02bfa992da6e69864499fd31dcb3d704675d16e55a0c5ad7c329ad3798f24
+size 2401099

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b08ab4ae85fee7596463136852d5f70af6c20c7d3271c15beaff64a55bfa82
+size 14645

scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0ed87b5f1b332e484564368c4a705e3c051fee55e9e66e1633af48d12d24104
+size 1383

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd3190e85b69bf1e75bdbf9b2e683ac33113ee45837950be410af0b40d2c7f0b
+size 1465

tokenizer_config.json

@@ -1,58 +1,15 @@
 {
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50264": {
-      "content": "<mask>",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "<s>",
-  "eos_token": "</s>",
-  "errors": "replace",
-  "extra_special_tokens": {},
-  "mask_token": "<mask>",
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "[MASK]",
   "model_max_length": 512,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "tokenizer_class": "RobertaTokenizer",
-  "trim_offsets": true,
-  "unk_token": "<unk>"
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
 }

train.py

@@ -1,85 +1,227 @@
-from datasets import load_dataset
-from transformers import DataCollatorForLanguageModeling
-from transformers import Trainer, TrainingArguments
+#!/usr/bin/env python3
+"""
+EuroBERT Fine-tuning Script with Improvements
+- Multi-GPU support
+- Early stopping (loss stabilization detection)
+- Random data sampling (removed date sorting)
+- Reduced epochs for efficiency
+"""
+
 import os
+import argparse
+from datetime import datetime
 import torch
+from datasets import load_dataset
+from transformers import (
+    AutoTokenizer,
+    AutoModelForMaskedLM,
+    DataCollatorForLanguageModeling,
+    TrainingArguments,
+    Trainer,
+    EarlyStoppingCallback,
+)
+from peft import LoraConfig, get_peft_model, TaskType
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Train EuroBERT model with LoRA")
+    parser.add_argument(
+        "--gpu-ids",
+        type=str,
+        default="0,1,2",
+        help="GPU IDs to use (comma-separated). Default: all 3 RTX 3080s",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=16,
+        help="Per-device batch size. With 3 GPUs and LoRA: 16 per GPU = 48 total",
+    )
+    parser.add_argument(
+        "--epochs",
+        type=int,
+        default=1,
+        help="Number of epochs (default: 1, reduced from 3 due to large dataset)",
+    )
+    parser.add_argument(
+        "--data-path",
+        type=str,
+        default="/home/apapagiannis/EuroVoc/files/*",
+        help="Path to training data",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="./results",
+        help="Output directory for results",
+    )
+    parser.add_argument(
+        "--model-name",
+        type=str,
+        default="nlpaueb/legal-bert-base-uncased",
+        help="Base model to fine-tune",
+    )
+    return parser.parse_args()
+
+
+def load_and_prepare_data(data_path, tokenizer, model_name):
+    """Load dataset and prepare for training."""
+    print("📦 Loading dataset...")
+    dataset = load_dataset("json", data_files=data_path, split="train")
+    print(f"Total samples: {len(dataset):,}")
+
+    # Split into train/eval
+    print("Splitting data (90% train, 10% eval)...")
+    split_dataset = dataset.train_test_split(test_size=0.1, seed=42)
+    train_dataset = split_dataset["train"]
+    eval_dataset = split_dataset["test"]
+
+    print(f"  Train: {len(train_dataset):,} samples")
+    print(f"  Eval:  {len(eval_dataset):,} samples")
+
+    # Tokenization function
+    def tokenize_function(batch):
+        texts = []
+        for t in batch.get("text", batch.get("content", [])):
+            texts.append(str(t) if t is not None else "")
+
+        return tokenizer(
+            texts,
+            truncation=True,
+            max_length=512,
+            padding="max_length",
+        )
+
+    print("Tokenizing datasets...")
+    tokenized_train = train_dataset.map(
+        tokenize_function,
+        batched=True,
+        remove_columns=train_dataset.column_names,
+        desc="Tokenizing train",
+    )
+    tokenized_eval = eval_dataset.map(
+        tokenize_function,
+        batched=True,
+        remove_columns=eval_dataset.column_names,
+        desc="Tokenizing eval",
+    )
+
+    return tokenized_train, tokenized_eval
+
+
+def setup_model_with_lora(model_name):
+    """Load model and apply LoRA configuration."""
+    print(f"🤖 Loading model: {model_name}")
+    model = AutoModelForMaskedLM.from_pretrained(model_name)
+
+    print("⚡ Setting up LoRA configuration...")
+    # IMPORTANT: For BERT MLM training, use FEATURE_EXTRACTION
+    # NOTE: If you switch to causal LM (GPT-style), change to TaskType.CAUSAL_LM
+    lora_config = LoraConfig(
+        task_type=TaskType.FEATURE_EXTRACTION,  # MLM pre-training task
+        r=8,
+        lora_alpha=16,
+        lora_dropout=0.1,
+        target_modules=["query", "value"],  # BERT attention layers
+        bias="none",
+    )
 
+    model = get_peft_model(model, lora_config)
+    model.print_trainable_parameters()
+    return model
 
 
 def main():
-      
-    local_rank = int(os.environ['LOCAL_RANK'])    
-    rank = int(os.environ['RANK'])    
-    world_size = int(os.environ['WORLD_SIZE']) 
-
-    torch.distributed.init_process_group("nccl")    
-    print(f"Local Rank = {local_rank}/{world_size}")    
-
-
-
-    # Load your JSONL file
-    dataset = load_dataset('json', data_files='../../data/m500_clean.jsonl', split='train')
-    
-    # Load a Model
-    from transformers import AutoTokenizer, AutoModelForCausalLM
-    
-    model_name = "FacebookAI/roberta-base"
-    
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(model_name)
-    
-    # Set pad token if not set
+    args = parse_arguments()
+
+    # Set GPU devices
+    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
+    num_gpus = len(args.gpu_ids.split(","))
+    print(f"🎮 Using {num_gpus} GPU(s): {args.gpu_ids}")
+    print(f"📊 GPU Info:")
+    if torch.cuda.is_available():
+        for i in range(torch.cuda.device_count()):
+            print(f"   GPU {i}: {torch.cuda.get_device_name(i)}")
+
+    # Load tokenizer
+    print(f"📝 Loading tokenizer from {args.model_name}")
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
     if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    
-    # Tokenize the dataset
-    def tokenize_function(examples):
-        return tokenizer(examples["text"], truncation=True, max_length=512)
-    
-    tokenized_dataset = dataset.map(tokenize_function, batched=True)
-    
-    # Split the dataset into training and validation sets
-    split_dataset = tokenized_dataset.train_test_split(test_size=0.1)
-    
-    # Data collator, pad the inputs to the maximum length in the batch
+        tokenizer.pad_token = tokenizer.cls_token
+
+    # Load and prepare data
+    tokenized_train, tokenized_eval = load_and_prepare_data(
+        args.data_path, tokenizer, args.model_name
+    )
+
+    # Setup model with LoRA
+    model = setup_model_with_lora(args.model_name)
+
+    # Data collator
     data_collator = DataCollatorForLanguageModeling(
-        tokenizer=tokenizer, mlm=False  # mlm=False: causal language modeling
+        tokenizer=tokenizer,
+        mlm=True,
+        mlm_probability=0.15,
     )
-    
-    # Training
+
+    # Calculate steps for context
+    train_steps_per_epoch = len(tokenized_train) // (args.batch_size * num_gpus)
+    print(f"\n📈 Training Configuration:")
+    print(f"   Batch size (per GPU): {args.batch_size}")
+    print(f"   Total batch size: {args.batch_size * num_gpus}")
+    print(f"   Steps per epoch: {train_steps_per_epoch:,}")
+    print(f"   Epochs: {args.epochs}")
+    print(f"   Total steps: {train_steps_per_epoch * args.epochs:,}")
+    print(f"   Early stopping: YES (monitor loss stability around 60k steps)")
+
+    # Training arguments with early stopping
     training_args = TrainingArguments(
-        output_dir="./results",
-        overwrite_output_dir=True,
-        num_train_epochs=3,
-        per_device_train_batch_size=4,
-        per_device_eval_batch_size=4,
-        dataloader_num_workers=8,  
+        output_dir=args.output_dir,
+        num_train_epochs=args.epochs,
+        per_device_train_batch_size=args.batch_size,
+        per_device_eval_batch_size=args.batch_size,
+        eval_strategy="steps",
         eval_steps=500,
         save_steps=1000,
-        warmup_steps=500,
-        prediction_loss_only=True,
-        logging_dir="./logs",
         logging_steps=100,
-        learning_rate=5e-5,
-        fp16=True,  # true for GPU
+        learning_rate=2e-4,
+        warmup_steps=200,
+        fp16=True,  # RTX 3080 supports FP16
+        report_to="none",
+        save_total_limit=3,  # Keep only last 3 checkpoints
+        load_best_model_at_end=True,
+        metric_for_best_model="eval_loss",
+        greater_is_better=False,
     )
-    
+
+    # Early stopping callback
+    # Monitors eval loss - stops if no improvement for 3 evaluations (~1500 steps)
+    early_stopping = EarlyStoppingCallback(
+        early_stopping_patience=3,  # Stop if no improvement for 3 evals
+        early_stopping_threshold=0.001,  # Minimum improvement threshold
+    )
+
+    # Create trainer
+    print("\n🚀 Initializing trainer...")
     trainer = Trainer(
         model=model,
         args=training_args,
-        train_dataset=split_dataset["train"],
-        eval_dataset=split_dataset["test"],
+        train_dataset=tokenized_train,
+        eval_dataset=tokenized_eval,
         data_collator=data_collator,
+        callbacks=[early_stopping],
     )
-    
-    # Start training
+
+    # Train
+    print("\n🔥 Starting training...\n")
     trainer.train()
 
-    torch.distributed.destroy_process_group()
-    
-    # Save the model and tokenizer
-    model.save_pretrained("./fine_tuned_model")
-    tokenizer.save_pretrained("./fine_tuned_model")
+    # Save model
+    print("\n💾 Saving fine-tuned model...")
+    model.save_pretrained(os.path.join(args.output_dir, "fine_tuned_model"))
+    tokenizer.save_pretrained(os.path.join(args.output_dir, "fine_tuned_model"))
+    print(f"✓ Model saved to {os.path.join(args.output_dir, 'fine_tuned_model')}")
+
 
 if __name__ == "__main__":
     main()

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40a4825312a77639f789ba54a89fc281e36a6ca718b6259b972c5a5db048463b
+size 5201