update 2026

.gitignore

@@ -0,0 +1 @@
+.DS_Store

adapter_config.json

@@ -1,43 +0,0 @@
-{
-  "alora_invocation_tokens": null,
-  "alpha_pattern": {},
-  "arrow_config": null,
-  "auto_mapping": null,
-  "base_model_name_or_path": "nlpaueb/legal-bert-base-uncased",
-  "bias": "none",
-  "corda_config": null,
-  "ensure_weight_tying": false,
-  "eva_config": null,
-  "exclude_modules": null,
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layer_replication": null,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "loftq_config": {},
-  "lora_alpha": 16,
-  "lora_bias": false,
-  "lora_dropout": 0.1,
-  "lora_ga_config": null,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "peft_version": "0.19.1",
-  "qalora_group_size": 16,
-  "r": 8,
-  "rank_pattern": {},
-  "revision": null,
-  "target_modules": [
-    "query",
-    "value"
-  ],
-  "target_parameters": null,
-  "task_type": "FEATURE_EXTRACTION",
-  "trainable_token_indices": null,
-  "use_bdlora": null,
-  "use_dora": false,
-  "use_qalora": false,
-  "use_rslora": false
-}

adapter_model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b5bc8eb3b718153088f03cc5fd984198f8f0c143b9936ea6bee3efadf5e5d65b
-size 1186328

config.json

@@ -1,47 +1,34 @@
 {
+  "add_cross_attention": false,
   "architectures": [
-    "BertForPreTraining"
+    "BertForMaskedLM"
   ],
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
-  "do_sample": false,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": null,
   "eos_token_ids": 0,
   "finetuning_task": null,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
-  "id2label": {
-    "0": "LABEL_0",
-    "1": "LABEL_1"
-  },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
   "is_decoder": false,
-  "label2id": {
-    "LABEL_0": 0,
-    "LABEL_1": 1
-  },
   "layer_norm_eps": 1e-12,
-  "length_penalty": 1.0,
-  "max_length": 20,
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_beams": 1,
   "num_hidden_layers": 12,
-  "num_labels": 2,
-  "num_return_sequences": 1,
-  "output_attentions": false,
-  "output_hidden_states": false,
   "output_past": true,
   "pad_token_id": 0,
   "pruned_heads": {},
-  "repetition_penalty": 1.0,
-  "temperature": 1.0,
-  "top_k": 50,
-  "top_p": 1.0,
+  "tie_word_embeddings": true,
   "torchscript": false,
+  "transformers_version": "5.9.0",
   "type_vocab_size": 2,
   "use_bfloat16": false,
+  "use_cache": true,
   "vocab_size": 30522
 }

inference.ipynb

@@ -0,0 +1,98 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "73e2d6cb-82b5-42cc-919d-dceaaf1f09f7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/apapagiannis/update_2025/eurovoc_training/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "Loading weights: 100%|██████████| 197/197 [00:00<00:00, 8783.65it/s]\n",
+      "[transformers] \u001b[1mBertModel LOAD REPORT\u001b[0m from: ./merged_model\n",
+      "Key                                        | Status     | \n",
+      "-------------------------------------------+------------+-\n",
+      "cls.predictions.transform.dense.bias       | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.bias   | UNEXPECTED | \n",
+      "cls.predictions.transform.LayerNorm.weight | UNEXPECTED | \n",
+      "cls.predictions.transform.dense.weight     | UNEXPECTED | \n",
+      "cls.predictions.bias                       | UNEXPECTED | \n",
+      "pooler.dense.bias                          | MISSING    | \n",
+      "pooler.dense.weight                        | MISSING    | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n",
+      "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "energy vs energy:   0.7808\n",
+      "energy vs criminal: 0.5920\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import AutoModel, AutoTokenizer\n",
+    "\n",
+    "model_dir = \"./merged_model\"\n",
+    "\n",
+    "# What you actually want — embedding quality\n",
+    "model = AutoModel.from_pretrained(model_dir)  # NOT AutoModelForMaskedLM\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_dir)\n",
+    "\n",
+    "def embed(text):\n",
+    "    inputs = tokenizer(text, return_tensors=\"pt\", truncation=True, max_length=512)\n",
+    "    with torch.no_grad():\n",
+    "        out = model(**inputs)\n",
+    "    return out.last_hidden_state[:, 0, :]  # CLS token\n",
+    "\n",
+    "# Test: similar legal concepts should be closer than unrelated ones\n",
+    "from torch.nn.functional import cosine_similarity\n",
+    "\n",
+    "e1 = embed(\"renewable energy regulation\")\n",
+    "e2 = embed(\"solar power and wind policy\")  # should be close\n",
+    "e3 = embed(\"criminal sentencing guidelines\")       # should be far\n",
+    "\n",
+    "print(f\"energy vs energy:   {cosine_similarity(e1, e2).item():.4f}\")  # expect ~high\n",
+    "print(f\"energy vs criminal: {cosine_similarity(e1, e3).item():.4f}\")  # expect ~lower"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94556c9a-bd65-489f-b2ab-d44b9b0812c7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "eurovoc_training",
+   "language": "python",
+   "name": "my-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

inference.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-"""EuroBERT Inference - Predict Masked Tokens"""
-
-import torch
-import argparse
-from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline
-from peft import PeftModel
-
-BASE_MODEL = "nlpaueb/legal-bert-base-uncased"
-ADAPTER_PATH = "./model"
-
-
-def main():
-    parser = argparse.ArgumentParser(description="EuroBERT - Predict Masked Tokens")
-    parser.add_argument("--text", type=str, default="The European [MASK] regulates digital platforms.",
-                       help="Text with [MASK] tokens to predict")
-    parser.add_argument("--top-k", type=int, default=5, help="Number of top predictions")
-    parser.add_argument("--adapter-path", type=str, default=ADAPTER_PATH, help="Path to LoRA adapter")
-    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu")
-    
-    args = parser.parse_args()
-    
-    # Load model
-    print(f"📦 Loading {BASE_MODEL}")
-    model = AutoModelForMaskedLM.from_pretrained(BASE_MODEL)
-    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
-    
-    print(f"⚡ Loading LoRA adapter from {args.adapter_path}")
-    model = PeftModel.from_pretrained(model, args.adapter_path)
-    model.to(args.device)
-    model.eval()
-    
-    print(f"🎮 Device: {args.device}\n")
-    
-    # Predict masked tokens
-    pipe = pipeline("fill-mask", model=model, tokenizer=tokenizer)
-    results = pipe(args.text, top_k=args.top_k)
-    
-    print(f"📝 Input: {args.text}")
-    print(f"🎯 Top {args.top_k} predictions:")
-    for i, result in enumerate(results, 1):
-        print(f"   {i}. '{result['token_str']}' (score: {result['score']:.4f})")
-        print(f"      → {result['sequence']}")
-
-
-if __name__ == "__main__":
-    main()

merge_with_base.ipynb

@@ -0,0 +1,72 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36a93a8b-1c82-4b47-b34d-cf6ce5dafc2e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForMaskedLM, AutoTokenizer\n",
+    "from peft import PeftModel\n",
+    "\n",
+    "base_model_name = \"nlpaueb/legal-bert-base-uncased\"\n",
+    "adapter_model_dir = \"results/checkpoint-xxxx\" # update with your actual checkpoint path\n",
+    "outdir = \"./merged_model\"\n",
+    "\n",
+    "# Load base + adapter\n",
+    "base_model = AutoModelForMaskedLM.from_pretrained(base_model_name)\n",
+    "peft_model = PeftModel.from_pretrained(base_model, adapter_model_dir)\n",
+    "\n",
+    "# Verify LoRA weights exist before merging\n",
+    "lora_keys = [k for k in peft_model.state_dict() if \"lora_\" in k]\n",
+    "print(f\"Found {len(lora_keys)} LoRA keys — should be 48 (2 per layer × 2 matrices × 12 layers)\")\n",
+    "assert len(lora_keys) > 0, \"No LoRA keys found — wrong checkpoint?\"\n",
+    "\n",
+    "# Merge and unload\n",
+    "merged = peft_model.merge_and_unload()\n",
+    "\n",
+    "# Verify no LoRA keys remain\n",
+    "merged_keys = [k for k in merged.state_dict() if \"lora_\" in k]\n",
+    "print(f\"LoRA keys after merge: {len(merged_keys)} — should be 0\")\n",
+    "\n",
+    "# Save\n",
+    "merged.save_pretrained(outdir, safe_serialization=True)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(base_model_name)\n",
+    "tokenizer.save_pretrained(outdir)\n",
+    "print(\"Saved cleanly.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43cddd93-7c0b-4bd7-b010-4657d7f09951",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "eubert",
+   "language": "python",
+   "name": "eurovoc-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

rng_state.pth → model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16b08ab4ae85fee7596463136852d5f70af6c20c7d3271c15beaff64a55bfa82
-size 14645
+oid sha256:be53e87dd08b424e9b1a4ba3cb24005f702e69f1b9a04ac26cbaee79e035f69e
+size 438080896

optimizer.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a3d02bfa992da6e69864499fd31dcb3d704675d16e55a0c5ad7c329ad3798f24
-size 2401099

scaler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d0ed87b5f1b332e484564368c4a705e3c051fee55e9e66e1633af48d12d24104
-size 1383

scheduler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd3190e85b69bf1e75bdbf9b2e683ac33113ee45837950be410af0b40d2c7f0b
-size 1465

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

train.ipynb

@@ -0,0 +1,289 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cd96b784",
+   "metadata": {},
+   "source": [
+    "# EuroBERT training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95edf3f9-59b6-4af3-8c14-2463256d4ec1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d0f5cc51",
+   "metadata": {},
+   "source": [
+    "## Load dataset from huggingface"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4b536d72",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\n",
+    "    \"json\",\n",
+    "    data_files=\"/home/apapagiannis/EuroVoc/files/*\",\n",
+    "    split=\"train\"\n",
+    ")\n",
+    "\n",
+    "print(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54d215a0",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "split_dataset = dataset.train_test_split(test_size=0.1, seed=42)\n",
+    "\n",
+    "train_dataset = split_dataset[\"train\"]\n",
+    "eval_dataset = split_dataset[\"test\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eefec7f8",
+   "metadata": {},
+   "source": [
+    "## Tokenize text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f2236486",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoTokenizer, AutoModelForMaskedLM\n",
+    "\n",
+    "BERT_MODEL_NAME = \"nlpaueb/legal-bert-base-uncased\"\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(BERT_MODEL_NAME)\n",
+    "model = AutoModelForMaskedLM.from_pretrained(BERT_MODEL_NAME)\n",
+    "\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.cls_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "825d7a3d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def tokenize_function(batch):\n",
+    "    texts = []\n",
+    "\n",
+    "    for t in batch[\"text\"]:\n",
+    "        if t is None:\n",
+    "            texts.append(\"\")\n",
+    "        else:\n",
+    "            texts.append(str(t))\n",
+    "\n",
+    "    return tokenizer(\n",
+    "        texts,\n",
+    "        truncation=True,\n",
+    "        max_length=512\n",
+    "    )\n",
+    "\n",
+    "tokenized_train = train_dataset.map(tokenize_function, batched=True, remove_columns=train_dataset.column_names)\n",
+    "tokenized_eval = eval_dataset.map(tokenize_function, batched=True, remove_columns=eval_dataset.column_names)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9fc6309a",
+   "metadata": {},
+   "source": [
+    "## Load model and set up training arguments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ccd9ea70",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForMaskedLM\n",
+    "\n",
+    "model = AutoModelForMaskedLM.from_pretrained(BERT_MODEL_NAME)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ce5b33a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import DataCollatorForLanguageModeling\n",
+    "\n",
+    "data_collator = DataCollatorForLanguageModeling(\n",
+    "    tokenizer=tokenizer,\n",
+    "    mlm=True,\n",
+    "    mlm_probability=0.15\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "645f2ac6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Define LoRA configuration and apply it to the model\n",
+    "from peft import LoraConfig, get_peft_model, TaskType\n",
+    "\n",
+    "lora_config = LoraConfig(\n",
+    "    task_type=TaskType.FEATURE_EXTRACTION,\n",
+    "    r=8,\n",
+    "    lora_alpha=16,\n",
+    "    lora_dropout=0.1,\n",
+    "    target_modules=[\"query\", \"value\"]  # BERT attention layers\n",
+    ")\n",
+    "\n",
+    "model = get_peft_model(model, lora_config)\n",
+    "\n",
+    "model.print_trainable_parameters()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9dd7dfbb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import TrainingArguments, EarlyStoppingCallback\n",
+    "\n",
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"./results\",\n",
+    "\n",
+    "    num_train_epochs=3,\n",
+    "    per_device_train_batch_size=8,   # INCREASE this with LoRA\n",
+    "    per_device_eval_batch_size=8,\n",
+    "\n",
+    "    eval_strategy=\"steps\",\n",
+    "    eval_steps=500,\n",
+    "\n",
+    "    save_steps=1000,\n",
+    "    logging_steps=100,\n",
+    "\n",
+    "    learning_rate=2e-4,\n",
+    "    warmup_steps=200,\n",
+    "\n",
+    "    fp16=True,  # or bf16=True if supported\n",
+    "    report_to=\"none\",\n",
+    "    \n",
+    "    save_total_limit=3,  # Keep only last 3 checkpoints\n",
+    "    load_best_model_at_end=True,\n",
+    "    metric_for_best_model=\"eval_loss\",\n",
+    "    greater_is_better=False,\n",
+    "\n",
+    ")\n",
+    "\n",
+    "# Early stopping callback\n",
+    "# Monitors eval loss - stops if no improvement for 3 evaluations (~1500 steps)\n",
+    "early_stopping = EarlyStoppingCallback(\n",
+    "    early_stopping_patience=3,  # Stop if no improvement for 3 evals\n",
+    "    early_stopping_threshold=0.001,  # Minimum improvement threshold\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "531b1862",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import Trainer\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=tokenized_train,\n",
+    "    eval_dataset=tokenized_eval,\n",
+    "    data_collator=data_collator,\n",
+    "    callbacks=[early_stopping],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d41560e-66d7-406f-8be4-3ccbe7950879",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer.train()\n",
+    "\n",
+    "model.save_pretrained(\"./fine_tuned_model\")\n",
+    "tokenizer.save_pretrained(\"./fine_tuned_model\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "eubert",
+   "language": "python",
+   "name": "eurovoc-venv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

train.py

@@ -1,227 +0,0 @@
-#!/usr/bin/env python3
-"""
-EuroBERT Fine-tuning Script with Improvements
-- Multi-GPU support
-- Early stopping (loss stabilization detection)
-- Random data sampling (removed date sorting)
-- Reduced epochs for efficiency
-"""
-
-import os
-import argparse
-from datetime import datetime
-import torch
-from datasets import load_dataset
-from transformers import (
-    AutoTokenizer,
-    AutoModelForMaskedLM,
-    DataCollatorForLanguageModeling,
-    TrainingArguments,
-    Trainer,
-    EarlyStoppingCallback,
-)
-from peft import LoraConfig, get_peft_model, TaskType
-
-
-def parse_arguments():
-    parser = argparse.ArgumentParser(description="Train EuroBERT model with LoRA")
-    parser.add_argument(
-        "--gpu-ids",
-        type=str,
-        default="0,1,2",
-        help="GPU IDs to use (comma-separated). Default: all 3 RTX 3080s",
-    )
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=16,
-        help="Per-device batch size. With 3 GPUs and LoRA: 16 per GPU = 48 total",
-    )
-    parser.add_argument(
-        "--epochs",
-        type=int,
-        default=1,
-        help="Number of epochs (default: 1, reduced from 3 due to large dataset)",
-    )
-    parser.add_argument(
-        "--data-path",
-        type=str,
-        default="/home/apapagiannis/EuroVoc/files/*",
-        help="Path to training data",
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        default="./results",
-        help="Output directory for results",
-    )
-    parser.add_argument(
-        "--model-name",
-        type=str,
-        default="nlpaueb/legal-bert-base-uncased",
-        help="Base model to fine-tune",
-    )
-    return parser.parse_args()
-
-
-def load_and_prepare_data(data_path, tokenizer, model_name):
-    """Load dataset and prepare for training."""
-    print("📦 Loading dataset...")
-    dataset = load_dataset("json", data_files=data_path, split="train")
-    print(f"Total samples: {len(dataset):,}")
-
-    # Split into train/eval
-    print("Splitting data (90% train, 10% eval)...")
-    split_dataset = dataset.train_test_split(test_size=0.1, seed=42)
-    train_dataset = split_dataset["train"]
-    eval_dataset = split_dataset["test"]
-
-    print(f"  Train: {len(train_dataset):,} samples")
-    print(f"  Eval:  {len(eval_dataset):,} samples")
-
-    # Tokenization function
-    def tokenize_function(batch):
-        texts = []
-        for t in batch.get("text", batch.get("content", [])):
-            texts.append(str(t) if t is not None else "")
-
-        return tokenizer(
-            texts,
-            truncation=True,
-            max_length=512,
-            padding="max_length",
-        )
-
-    print("Tokenizing datasets...")
-    tokenized_train = train_dataset.map(
-        tokenize_function,
-        batched=True,
-        remove_columns=train_dataset.column_names,
-        desc="Tokenizing train",
-    )
-    tokenized_eval = eval_dataset.map(
-        tokenize_function,
-        batched=True,
-        remove_columns=eval_dataset.column_names,
-        desc="Tokenizing eval",
-    )
-
-    return tokenized_train, tokenized_eval
-
-
-def setup_model_with_lora(model_name):
-    """Load model and apply LoRA configuration."""
-    print(f"🤖 Loading model: {model_name}")
-    model = AutoModelForMaskedLM.from_pretrained(model_name)
-
-    print("⚡ Setting up LoRA configuration...")
-    # IMPORTANT: For BERT MLM training, use FEATURE_EXTRACTION
-    # NOTE: If you switch to causal LM (GPT-style), change to TaskType.CAUSAL_LM
-    lora_config = LoraConfig(
-        task_type=TaskType.FEATURE_EXTRACTION,  # MLM pre-training task
-        r=8,
-        lora_alpha=16,
-        lora_dropout=0.1,
-        target_modules=["query", "value"],  # BERT attention layers
-        bias="none",
-    )
-
-    model = get_peft_model(model, lora_config)
-    model.print_trainable_parameters()
-    return model
-
-
-def main():
-    args = parse_arguments()
-
-    # Set GPU devices
-    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
-    num_gpus = len(args.gpu_ids.split(","))
-    print(f"🎮 Using {num_gpus} GPU(s): {args.gpu_ids}")
-    print(f"📊 GPU Info:")
-    if torch.cuda.is_available():
-        for i in range(torch.cuda.device_count()):
-            print(f"   GPU {i}: {torch.cuda.get_device_name(i)}")
-
-    # Load tokenizer
-    print(f"📝 Loading tokenizer from {args.model_name}")
-    tokenizer = AutoTokenizer.from_pretrained(args.model_name)
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.cls_token
-
-    # Load and prepare data
-    tokenized_train, tokenized_eval = load_and_prepare_data(
-        args.data_path, tokenizer, args.model_name
-    )
-
-    # Setup model with LoRA
-    model = setup_model_with_lora(args.model_name)
-
-    # Data collator
-    data_collator = DataCollatorForLanguageModeling(
-        tokenizer=tokenizer,
-        mlm=True,
-        mlm_probability=0.15,
-    )
-
-    # Calculate steps for context
-    train_steps_per_epoch = len(tokenized_train) // (args.batch_size * num_gpus)
-    print(f"\n📈 Training Configuration:")
-    print(f"   Batch size (per GPU): {args.batch_size}")
-    print(f"   Total batch size: {args.batch_size * num_gpus}")
-    print(f"   Steps per epoch: {train_steps_per_epoch:,}")
-    print(f"   Epochs: {args.epochs}")
-    print(f"   Total steps: {train_steps_per_epoch * args.epochs:,}")
-    print(f"   Early stopping: YES (monitor loss stability around 60k steps)")
-
-    # Training arguments with early stopping
-    training_args = TrainingArguments(
-        output_dir=args.output_dir,
-        num_train_epochs=args.epochs,
-        per_device_train_batch_size=args.batch_size,
-        per_device_eval_batch_size=args.batch_size,
-        eval_strategy="steps",
-        eval_steps=500,
-        save_steps=1000,
-        logging_steps=100,
-        learning_rate=2e-4,
-        warmup_steps=200,
-        fp16=True,  # RTX 3080 supports FP16
-        report_to="none",
-        save_total_limit=3,  # Keep only last 3 checkpoints
-        load_best_model_at_end=True,
-        metric_for_best_model="eval_loss",
-        greater_is_better=False,
-    )
-
-    # Early stopping callback
-    # Monitors eval loss - stops if no improvement for 3 evaluations (~1500 steps)
-    early_stopping = EarlyStoppingCallback(
-        early_stopping_patience=3,  # Stop if no improvement for 3 evals
-        early_stopping_threshold=0.001,  # Minimum improvement threshold
-    )
-
-    # Create trainer
-    print("\n🚀 Initializing trainer...")
-    trainer = Trainer(
-        model=model,
-        args=training_args,
-        train_dataset=tokenized_train,
-        eval_dataset=tokenized_eval,
-        data_collator=data_collator,
-        callbacks=[early_stopping],
-    )
-
-    # Train
-    print("\n🔥 Starting training...\n")
-    trainer.train()
-
-    # Save model
-    print("\n💾 Saving fine-tuned model...")
-    model.save_pretrained(os.path.join(args.output_dir, "fine_tuned_model"))
-    tokenizer.save_pretrained(os.path.join(args.output_dir, "fine_tuned_model"))
-    print(f"✓ Model saved to {os.path.join(args.output_dir, 'fine_tuned_model')}")
-
-
-if __name__ == "__main__":
-    main()

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:40a4825312a77639f789ba54a89fc281e36a6ca718b6259b972c5a5db048463b
-size 5201