Upload folder using huggingface_hub

Code/cpt_mlm_then_finetune.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+CPT (continued pretraining) + finetune pipeline for Any model .
+
+1) Use the text from the test CSV (test_file.csv) to do masked language model (MLM) training
+   on any model (CPT step). This adapts the LM to the test-domain text.
+
+2) Load the CPT weights and fine-tune a classification head (same task as `traim.py`):
+   - Train on the training CSV (train.csv) that has `Commentaire client` and `Class` (1..9)
+   - Keep classes as 1..9 but the model uses 0..8 internally (label mapping saved to config)
+
+Usage:
+    python cpt_mlm_then_finetune.py
+
+Notes:
+- This script uses the Hugging Face Trainer API for both phases.
+- Adjust the TRAIN/MLM epochs and batch sizes if you have a different GPU memory budget.
+"""
+
+import os
+import json
+import numpy as np
+import torch
+from datasets import load_dataset
+from transformers import (
+    AutoTokenizer,
+    AutoConfig,
+    AutoModelForMaskedLM,
+    AutoModelForSequenceClassification,
+    DataCollatorForLanguageModeling,
+    TrainingArguments,
+    Trainer,
+)
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+from inspect import signature
+
+# ---------------------------
+# Paths & basic config
+# ---------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--labelds/snapshots/48f016fd5987875b0e9f79d0689cef2ec3b2ce0b/test_file.csv"
+TRAIN_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--labelds/snapshots/48f016fd5987875b0e9f79d0689cef2ec3b2ce0b/train.csv"
+BASE_MODEL = "alger-ia/dziribert_sentiment"
+
+CPT_OUTPUT_DIR = "./dziribert" # UBC-NLP/MARBERTv2 , aubmindlab/bert-large-arabertv2 , alger-ia/dziribert , CAMeL-Lab/bert-base-arabic-camelbert-msa
+FT_OUTPUT_DIR = "./telecom_dziribert_cpt_ft" 
+
+MAX_LENGTH = 512
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Device: {DEVICE}")
+
+# Label mapping (keep user-facing classes 1..9 but model indices 0..8)
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {v: k for k, v in LABEL2ID.items()}
+NUM_LABELS = len(LABEL2ID)
+
+# ---------------------------
+# 1) CPT: Masked LM training on test text
+# ---------------------------
+print("\n=== CPT (MLM) phase: load test texts and continue pretraining the LM ===\n")
+
+# Load test CSV (contains column 'Commentaire client')
+print(f"Loading test CSV from: {TEST_FILE}")
+test_ds = load_dataset("csv", data_files=TEST_FILE, split="train")
+print(f"Test samples: {len(test_ds)} | Columns: {test_ds.column_names}")
+
+# Extract texts column name heuristics
+text_col = None
+for c in test_ds.column_names:
+    if c.lower() in {"commentaire client", "commentaire_client", "comment", "text", "commentaire"}:
+        text_col = c
+        break
+if text_col is None:
+    # fallback to the last column
+    text_col = test_ds.column_names[-1]
+print(f"Using text column for MLM: {text_col}")
+
+# Load tokenizer and model for MLM
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+
+# Ensure tokenizer has a mask token
+if tokenizer.mask_token is None:
+    # BERT-style tokenizers typically have [MASK]
+    tokenizer.add_special_tokens({"mask_token": "[MASK]"})
+    print("Added [MASK] token to tokenizer")
+
+print("Loading base model for MLM:", BASE_MODEL)
+mlm_model = AutoModelForMaskedLM.from_pretrained(BASE_MODEL)
+mlm_model.resize_token_embeddings(len(tokenizer))
+mlm_model = mlm_model.to(DEVICE)
+
+# Tokenize for MLM using simple line-by-line tokenization
+def tokenize_mlm(examples):
+    texts = examples[text_col]
+    return tokenizer(
+        texts,
+        truncation=True,
+        padding="max_length",
+        max_length=MAX_LENGTH,
+    )
+
+print("Tokenizing test dataset for MLM...")
+mlm_tokenized = test_ds.map(tokenize_mlm, batched=True, remove_columns=test_ds.column_names)
+
+# Data collator for dynamic masking
+data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=True, mlm_probability=0.15)
+
+# Training args for CPT (adjust epochs/batch as needed)
+cpt_train_args = TrainingArguments(
+    output_dir=CPT_OUTPUT_DIR,
+    overwrite_output_dir=True,
+    num_train_epochs=38,
+    per_device_train_batch_size=32,
+    save_total_limit=2,
+    save_strategy="epoch",
+    logging_steps=200,
+    fp16=torch.cuda.is_available(),
+)
+
+# Trainer for MLM
+cpt_trainer = Trainer(
+    model=mlm_model,
+    args=cpt_train_args,
+    train_dataset=mlm_tokenized,
+    data_collator=data_collator,
+    tokenizer=tokenizer,
+)
+
+print("Starting CPT (MLM) training...")
+cpt_trainer.train()
+print("CPT finished. Saving CPT model to:", CPT_OUTPUT_DIR)
+
+# Save model & tokenizer
+cpt_trainer.save_model(CPT_OUTPUT_DIR)
+tokenizer.save_pretrained(CPT_OUTPUT_DIR)
+
+# Save an informative config with label mappings too (so downstream scripts can read them)
+config_path = os.path.join(CPT_OUTPUT_DIR, "config.json")
+if os.path.exists(config_path):
+    with open(config_path, "r", encoding="utf-8") as f:
+        config_data = json.load(f)
+else:
+    config_data = {}
+
+# id2label as strings mapping index->original class (1..9), label2id as str -> int
+config_data["id2label"] = {str(k): str(v) for k, v in ID2LABEL.items()}  # e.g. "0":"1"
+config_data["label2id"] = {str(v): k for k, v in LABEL2ID.items()}  # e.g. "1":0
+config_data["num_labels"] = NUM_LABELS
+config_data["problem_type"] = "single_label_classification"
+
+with open(config_path, "w", encoding="utf-8") as f:
+    json.dump(config_data, f, ensure_ascii=False, indent=2)
+
+print("Saved config with label mappings to CPT output dir")
+
+# ---------------------------
+# 2) Finetune classification using CPT weights
+# ---------------------------
+print("\n=== Finetuning phase: load CPT weights and fine-tune classifier ===\n")
+
+# Load train dataset with labels
+print(f"Loading training CSV from: {TRAIN_FILE}")
+train_ds = load_dataset("csv", data_files=TRAIN_FILE, split="train")
+print(f"Train samples: {len(train_ds)} | Columns: {train_ds.column_names}")
+
+# Prepare label mapping on train file (ensure handling of string/int)
+def encode_train_labels(example):
+    c = example.get("Class")
+    if isinstance(c, str):
+        try:
+            c = int(c)
+        except Exception:
+            # Attempt to strip and convert
+            c = int(c.strip())
+    if c not in LABEL2ID:
+        raise ValueError(f"Unexpected class value in training data: {c}")
+    example["labels"] = LABEL2ID[c]
+    return example
+
+train_ds = train_ds.map(encode_train_labels)
+
+# Train/validation split
+split = train_ds.train_test_split(test_size=0.1, seed=42)
+train_split = split["train"]
+eval_split = split["test"]
+print("Train split size:", len(train_split), "Eval split size:", len(eval_split))
+
+# Load tokenizer and model from CPT output
+print("Loading tokenizer and model from CPT output for finetuning...")
+ft_tokenizer = AutoTokenizer.from_pretrained(CPT_OUTPUT_DIR)
+
+# Load sequence classification model initialized from the CPT weights
+print("Loading AutoModelForSequenceClassification from CPT weights")
+ft_model = AutoModelForSequenceClassification.from_pretrained(
+    CPT_OUTPUT_DIR,
+    num_labels=NUM_LABELS,
+    id2label={str(k): str(v) for k, v in ID2LABEL.items()},
+    label2id={str(v): k for k, v in LABEL2ID.items()},
+)
+
+ft_model = ft_model.to(DEVICE)
+
+# Tokenization function matches traim.py
+def preprocess_classification(examples):
+    return ft_tokenizer(
+        examples["Commentaire client"],
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH,
+    )
+
+train_split = train_split.map(preprocess_classification, batched=True, num_proc=4)
+eval_split = eval_split.map(preprocess_classification, batched=True, num_proc=4)
+
+train_split.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+eval_split.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+# Metrics same as traim.py
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    preds = np.argmax(logits, axis=-1)
+    acc = accuracy_score(labels, preds)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(labels, preds, average='weighted', zero_division=0)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(labels, preds, average='macro', zero_division=0)
+    precision_mi, recall_mi, f1_mi, _ = precision_recall_fscore_support(labels, preds, average='micro', zero_division=0)
+    metrics = {
+        'accuracy': acc,
+        'f1_weighted': f1_w,
+        'f1_macro': f1_m,
+        'f1_micro': f1_mi,
+        'precision_weighted': precision_w,
+        'recall_weighted': recall_w,
+        'precision_macro': precision_m,
+        'recall_macro': recall_m,
+    }
+    # per-class f1
+    per_class_f1 = f1_score(labels, preds, average=None, zero_division=0)
+    for idx, class_name in ID2LABEL.items():
+        if idx < len(per_class_f1):
+            metrics[f'f1_class_{class_name}'] = float(per_class_f1[idx])
+    return metrics
+
+# Training arguments for finetuning
+# Reuse dynamic check for transformers TrainingArguments signature
+ta_sig = signature(TrainingArguments.__init__)
+ta_params = set(ta_sig.parameters.keys())
+
+ft_base_kwargs = {
+    'output_dir': FT_OUTPUT_DIR,
+    'num_train_epochs': 70,
+    'per_device_train_batch_size': 32,
+    'per_device_eval_batch_size': 64,
+    'learning_rate': 1e-5,
+    'weight_decay': 0.01,
+    'warmup_ratio': 0.1,
+    'logging_steps': 50,
+    'save_total_limit': 2,
+}
+
+if 'bf16' in ta_params and torch.cuda.is_available() and hasattr(torch.cuda, 'is_bf16_supported') and torch.cuda.is_bf16_supported():
+    ft_base_kwargs['bf16'] = True
+elif 'fp16' in ta_params and torch.cuda.is_available():
+    ft_base_kwargs['fp16'] = True
+
+# Add evaluation_strategy if supported
+if 'evaluation_strategy' in ta_params:
+    ft_base_kwargs['evaluation_strategy'] = 'epoch'
+    ft_base_kwargs['save_strategy'] = 'epoch'
+    ft_base_kwargs['load_best_model_at_end'] = True
+    ft_base_kwargs['metric_for_best_model'] = 'f1_weighted'
+
+# Filter supported args
+ft_filtered = {k: v for k, v in ft_base_kwargs.items() if k in ta_params}
+
+ft_training_args = TrainingArguments(**ft_filtered)
+
+# Trainer for finetuning
+ft_trainer = Trainer(
+    model=ft_model,
+    args=ft_training_args,
+    train_dataset=train_split,
+    eval_dataset=eval_split,
+    tokenizer=ft_tokenizer,
+    compute_metrics=compute_metrics,
+)
+
+print("Starting finetuning on classification task...")
+ft_trainer.train()
+
+print("Finetuning finished. Saving finetuned model to:", FT_OUTPUT_DIR)
+ft_trainer.save_model(FT_OUTPUT_DIR)
+ft_tokenizer.save_pretrained(FT_OUTPUT_DIR)
+
+# Update config with label mappings (so inference scripts can read cleanly)
+config_path = os.path.join(FT_OUTPUT_DIR, 'config.json')
+if os.path.exists(config_path):
+    with open(config_path, 'r', encoding='utf-8') as f:
+        cfg = json.load(f)
+else:
+    cfg = {}
+
+cfg['id2label'] = {str(k): str(v) for k, v in ID2LABEL.items()}
+cfg['label2id'] = {str(v): k for k, v in LABEL2ID.items()}
+cfg['num_labels'] = NUM_LABELS
+cfg['problem_type'] = 'single_label_classification'
+
+with open(config_path, 'w', encoding='utf-8') as f:
+    json.dump(cfg, f, ensure_ascii=False, indent=2)
+
+print("Saved label mappings to finetuned model config")
+
+print('\nAll done. CPT and finetuning completed.')
+print('CPT model saved to:', CPT_OUTPUT_DIR)
+print('Finetuned classifier saved to:', FT_OUTPUT_DIR)
+

Code/finetune_from_cpt.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Fine-tuning pipeline from saved CPT model for any BERT-style model.
+
+Loads the CPT weights and fine-tunes a classification head:
+- Train on the training CSV (train.csv) that has `Commentaire client` and `Class` (1..9)
+- Keep classes as 1..9 but the model uses 0..8 internally (label mapping saved to config)
+
+Usage:
+    python finetune_from_cpt.py
+
+Notes:
+- This script uses the Hugging Face Trainer API.
+- Adjust the epochs and batch sizes if you have a different GPU memory budget.
+"""
+
+import os
+import json
+import numpy as np
+import torch
+from datasets import load_dataset
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    TrainingArguments,
+    Trainer,
+)
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+from inspect import signature
+
+# ---------------------------
+# Paths & basic config
+# ---------------------------
+TRAIN_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--labelds/snapshots/48f016fd5987875b0e9f79d0689cef2ec3b2ce0b/train.csv"
+
+# Path to your saved CPT model
+CPT_MODEL_PATH = "" # Change this to your CPT model path
+
+# Output directory for fine-tuned model
+FT_OUTPUT_DIR = "./telecom_arabert_large_full_pipeline"
+
+MAX_LENGTH = 512
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Device: {DEVICE}")
+
+# Label mapping (keep user-facing classes 1..9 but model indices 0..8)
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {v: k for k, v in LABEL2ID.items()}
+NUM_LABELS = len(LABEL2ID)
+
+# ---------------------------
+# Finetune classification using CPT weights
+# ---------------------------
+print("\n=== Finetuning phase: load CPT weights and fine-tune classifier ===\n")
+
+# Check if CPT model exists
+if not os.path.exists(CPT_MODEL_PATH):
+    raise FileNotFoundError(f"CPT model not found at: {CPT_MODEL_PATH}")
+
+print(f"Loading CPT model from: {CPT_MODEL_PATH}")
+
+# Load train dataset with labels
+print(f"Loading training CSV from: {TRAIN_FILE}")
+train_ds = load_dataset("csv", data_files=TRAIN_FILE, split="train")
+print(f"Train samples: {len(train_ds)} | Columns: {train_ds.column_names}")
+
+# Prepare label mapping on train file (ensure handling of string/int)
+def encode_train_labels(example):
+    c = example.get("Class")
+    if isinstance(c, str):
+        try:
+            c = int(c)
+        except Exception:
+            # Attempt to strip and convert
+            c = int(c.strip())
+    if c not in LABEL2ID:
+        raise ValueError(f"Unexpected class value in training data: {c}")
+    example["labels"] = LABEL2ID[c]
+    return example
+
+train_ds = train_ds.map(encode_train_labels)
+
+# Train/validation split
+split = train_ds.train_test_split(test_size=0.1, seed=42)
+train_split = split["train"]
+eval_split = split["test"]
+print("Train split size:", len(train_split), "Eval split size:", len(eval_split))
+
+# Load tokenizer and model from CPT output
+print("Loading tokenizer and model from CPT output for finetuning...")
+ft_tokenizer = AutoTokenizer.from_pretrained(CPT_MODEL_PATH)
+
+# Load sequence classification model initialized from the CPT weights
+print("Loading AutoModelForSequenceClassification from CPT weights")
+ft_model = AutoModelForSequenceClassification.from_pretrained(
+    CPT_MODEL_PATH,
+    num_labels=NUM_LABELS,
+    id2label={str(k): str(v) for k, v in ID2LABEL.items()},
+    label2id={str(v): k for k, v in LABEL2ID.items()},
+)
+
+ft_model = ft_model.to(DEVICE)
+
+# Print model info
+total_params = sum(p.numel() for p in ft_model.parameters())
+trainable_params = sum(p.numel() for p in ft_model.parameters() if p.requires_grad)
+print(f"Total parameters: {total_params:,}")
+print(f"Trainable parameters: {trainable_params:,}")
+
+# Tokenization function
+def preprocess_classification(examples):
+    return ft_tokenizer(
+        examples["Commentaire client"],
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH,
+    )
+
+train_split = train_split.map(preprocess_classification, batched=True, num_proc=4)
+eval_split = eval_split.map(preprocess_classification, batched=True, num_proc=4)
+
+train_split.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+eval_split.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+# Metrics
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    preds = np.argmax(logits, axis=-1)
+    acc = accuracy_score(labels, preds)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(labels, preds, average='weighted', zero_division=0)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(labels, preds, average='macro', zero_division=0)
+    precision_mi, recall_mi, f1_mi, _ = precision_recall_fscore_support(labels, preds, average='micro', zero_division=0)
+    metrics = {
+        'accuracy': acc,
+        'f1_weighted': f1_w,
+        'f1_macro': f1_m,
+        'f1_micro': f1_mi,
+        'precision_weighted': precision_w,
+        'recall_weighted': recall_w,
+        'precision_macro': precision_m,
+        'recall_macro': recall_m,
+    }
+    # per-class f1
+    per_class_f1 = f1_score(labels, preds, average=None, zero_division=0)
+    for idx, class_name in ID2LABEL.items():
+        if idx < len(per_class_f1):
+            metrics[f'f1_class_{class_name}'] = float(per_class_f1[idx])
+    return metrics
+
+# Training arguments for finetuning
+# Reuse dynamic check for transformers TrainingArguments signature
+ta_sig = signature(TrainingArguments.__init__)
+ta_params = set(ta_sig.parameters.keys())
+
+ft_base_kwargs = {
+    'output_dir': FT_OUTPUT_DIR,
+    'num_train_epochs': 100,
+    'per_device_train_batch_size': 32,
+    'per_device_eval_batch_size': 64,
+    'learning_rate': 1e-5,
+    'weight_decay': 0.01,
+    'warmup_ratio': 0.1,
+    'logging_steps': 50,
+    'save_total_limit': 2,
+}
+
+if 'bf16' in ta_params and torch.cuda.is_available() and hasattr(torch.cuda, 'is_bf16_supported') and torch.cuda.is_bf16_supported():
+    ft_base_kwargs['bf16'] = True
+elif 'fp16' in ta_params and torch.cuda.is_available():
+    ft_base_kwargs['fp16'] = True
+
+# Add evaluation_strategy if supported
+if 'evaluation_strategy' in ta_params:
+    ft_base_kwargs['evaluation_strategy'] = 'epoch'
+    ft_base_kwargs['save_strategy'] = 'epoch'
+    ft_base_kwargs['load_best_model_at_end'] = True
+    ft_base_kwargs['metric_for_best_model'] = 'f1_weighted'
+
+# Filter supported args
+ft_filtered = {k: v for k, v in ft_base_kwargs.items() if k in ta_params}
+
+ft_training_args = TrainingArguments(**ft_filtered)
+
+# Trainer for finetuning
+ft_trainer = Trainer(
+    model=ft_model,
+    args=ft_training_args,
+    train_dataset=train_split,
+    eval_dataset=eval_split,
+    tokenizer=ft_tokenizer,
+    compute_metrics=compute_metrics,
+)
+
+print("Starting finetuning on classification task...")
+ft_trainer.train()
+
+print("Finetuning finished. Saving finetuned model to:", FT_OUTPUT_DIR)
+ft_trainer.save_model(FT_OUTPUT_DIR)
+ft_tokenizer.save_pretrained(FT_OUTPUT_DIR)
+
+# Update config with label mappings (so inference scripts can read cleanly)
+config_path = os.path.join(FT_OUTPUT_DIR, 'config.json')
+if os.path.exists(config_path):
+    with open(config_path, 'r', encoding='utf-8') as f:
+        cfg = json.load(f)
+else:
+    cfg = {}
+
+cfg['id2label'] = {str(k): str(v) for k, v in ID2LABEL.items()}
+cfg['label2id'] = {str(v): k for k, v in LABEL2ID.items()}
+cfg['num_labels'] = NUM_LABELS
+cfg['problem_type'] = 'single_label_classification'
+
+with open(config_path, 'w', encoding='utf-8') as f:
+    json.dump(cfg, f, ensure_ascii=False, indent=2)
+
+print("Saved label mappings to finetuned model config")
+
+print('\nAll done. Finetuning completed.')
+print('Finetuned classifier saved to:', FT_OUTPUT_DIR)

Code/finetune_gemma3_classification.py

@@ -0,0 +1,686 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Gemma 3 4B - Instruction Fine-Tuning for Classification
+
+Fine-tuning Gemma 3 4B with instruction format (QA style) for 9-class classification.
+Uses RS-LoRA (Rank-Stabilized LoRA) to avoid overfitting.
+
+Features:
+- Text preprocessing (remove names, tatweel, emojis)
+- Instruction tuning format with few-shot examples
+- RS-LoRA for efficient training
+- BF16 training on A100
+
+Usage:
+    python finetune_gemma3_classification.py
+"""
+
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Suppress tokenizer fork warning
+
+import re
+import json
+import numpy as np
+import torch
+from datasets import load_dataset, Dataset
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    TrainingArguments,
+    Trainer,
+    DataCollatorForSeq2Seq,
+    BitsAndBytesConfig,
+)
+from peft import (
+    LoraConfig,
+    get_peft_model,
+    TaskType,
+    prepare_model_for_kbit_training,
+)
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+import warnings
+warnings.filterwarnings("ignore")
+
+# ---------------------------
+# Paths & Config
+# ---------------------------
+TRAIN_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/train.csv"
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+BASE_MODEL = "google/gemma-3-4b-it"
+
+FT_OUTPUT_DIR = "./gemma3_classification_ft"
+
+MAX_LENGTH = 2048  # Longer for instruction format with few-shot
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Device: {DEVICE}")
+
+# Enable TF32 for A100
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+# Label mapping
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {v: k for k, v in LABEL2ID.items()}
+NUM_LABELS = len(LABEL2ID)
+
+text_col = "Commentaire client"
+
+# ===========================================================================
+# System Prompt and Few-Shot Examples
+# ===========================================================================
+SYSTEM_PROMPT = """You are an expert Algerian linguist and data labeler. Your task is to classify customer comments from Algérie Télécom's social media into one of 9 specific categories.
+
+## CLASSES (DETAILED DESCRIPTIONS)
+- **Class 1 (Wish/Positive Anticipation):** Comments expressing a wish, a hopeful anticipation, or general positive feedback/appreciation for future services or offers.
+- **Class 2 (Complaint: Equipment/Supply):** Comments complaining about the lack, unavailability, or delay in the supply of necessary equipment (e.g., modems, fiber optics devices).
+- **Class 3 (Complaint: Marketing/Advertising):** Comments criticizing advertisements, marketing campaigns, or their lack of realism/meaning.
+- **Class 4 (Complaint: Installation/Deployment):** Comments about delays, stoppages, or failure in service installation, network expansion, or fiber optics deployment (e.g., digging issues).
+- **Class 5 (Inquiry/Request for Information):** Comments asking for eligibility, connection dates, service status, coverage details, or specific contact information.
+- **Class 6 (Complaint: Technical Support/Intervention):** Comments regarding delays in repair interventions, issues with technical staff competence, or unsatisfactory customer service agency visits.
+- **Class 7 (Pricing/Service Enhancement):** Comments focused on pricing, requests for cost reduction, or suggestions for general service/app functionality enhancements.
+- **Class 8 (Complaint: Total Service Outage/Disconnection):** Comments indicating a complete, sustained loss of service (e.g., no phone, no internet, total disconnection).
+- **Class 9 (Complaint: Service Performance/Quality):** Comments about technical issues impacting performance (e.g., slow speed, high latency, broken website/portal, coverage claims).
+
+Respond with ONLY the class number (1-9). Do not include any explanation."""
+
+# Few-shot examples (2-3 per class for diversity)
+FEW_SHOT_EXAMPLES = [
+    # Class 1
+    {"comment": "إن شاء الله يكون عرض صحاب 300 و 500 ميجا فيبر ياربي", "class": "1"},
+    {"comment": "اتمنى لكم مزيد من التألق", "class": "1"},
+    # Class 2
+    {"comment": "زعما جابو المودام ؟", "class": "2"},
+    {"comment": "وفرو أجهزة مودام الباقي ساهل !", "class": "2"},
+    # Class 3
+    {"comment": "إشهار بدون معنه", "class": "3"},
+    # Class 4
+    {"comment": "المشروع متوقف منذ اشهر", "class": "4"},
+    {"comment": "نتمنى تكملو في ايسطو وهران في اقرب وقت رانا نعانو مع ADSL", "class": "4"},
+    # Class 5
+    {"comment": "modem", "class": "5"},
+    {"comment": "يعني كي نطلعها ثلاثون ميغا كارطة تاع مائة الف ��داه تحكملي؟", "class": "5"},
+    # Class 6
+    {"comment": "عرض 20 ميجا نحيوه مدام مش قادرين تعطيونا حقنا", "class": "6"},
+    # Class 7
+    {"comment": "نقصوا الاسعار بزااااف غالية", "class": "7"},
+    {"comment": "علاه ماديروش في التطبيق خاصية التوقيف المؤقت للانترانات", "class": "7"},
+    # Class 8
+    {"comment": "رانا بلا تلفون ولا انترنت", "class": "8"},
+    {"comment": "ثلاثة اشهر بلا انترنت", "class": "8"},
+    # Class 9
+    {"comment": "فضاء الزبون علاه منقدروش نسجلو فيه", "class": "9"},
+    {"comment": "هل موقع فضاء الزبون متوقف", "class": "9"},
+]
+
+# ===========================================================================
+# Text Preprocessing
+# ===========================================================================
+def preprocess_text(text):
+    """
+    Preprocess text:
+    - Remove Arabic tatweel (ـ)
+    - Remove emojis
+    - Remove user mentions/names
+    - Normalize whitespace
+    - Remove phone numbers
+    - Remove URLs
+    """
+    if not isinstance(text, str):
+        return ""
+    
+    # Remove URLs
+    text = re.sub(r'https?://\S+|www\.\S+', '', text)
+    
+    # Remove email addresses
+    text = re.sub(r'\S+@\S+', '', text)
+    
+    # Remove phone numbers (various formats)
+    text = re.sub(r'[\+]?[(]?[0-9]{1,4}[)]?[-\s\./0-9]{6,}', '', text)
+    text = re.sub(r'\b0[567]\d{8}\b', '', text)  # Algerian mobile
+    text = re.sub(r'\b0[23]\d{7,8}\b', '', text)  # Algerian landline
+    
+    # Remove mentions (@username)
+    text = re.sub(r'@\w+', '', text)
+    
+    # Remove Arabic tatweel (kashida)
+    text = re.sub(r'ـ+', '', text)
+    
+    # Remove emojis and other symbols
+    emoji_pattern = re.compile("["
+        u"\U0001F600-\U0001F64F"  # emoticons
+        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+        u"\U0001F680-\U0001F6FF"  # transport & map symbols
+        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+        u"\U00002702-\U000027B0"
+        u"\U000024C2-\U0001F251"
+        u"\U0001f926-\U0001f937"
+        u"\U00010000-\U0010ffff"
+        u"\u2640-\u2642"
+        u"\u2600-\u2B55"
+        u"\u200d"
+        u"\u23cf"
+        u"\u23e9"
+        u"\u231a"
+        u"\ufe0f"
+        u"\u3030"
+        "]+", flags=re.UNICODE)
+    text = emoji_pattern.sub('', text)
+    
+    # Remove common platform names that might be mentioned
+    text = re.sub(r'Algérie Télécom - إتصالات الجزائر', '', text, flags=re.IGNORECASE)
+    text = re.sub(r'Algérie Télécom', '', text, flags=re.IGNORECASE)
+    text = re.sub(r'إتصالات الجزائر', '', text)
+    
+    # Remove repeated characters (more than 3)
+    text = re.sub(r'(.)\1{3,}', r'\1\1\1', text)
+    
+    # Normalize whitespace
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    return text
+
+# ===========================================================================
+# Format Data for Instruction Tuning
+# ===========================================================================
+# Hardcoded few-shot examples string (not from data)
+FEW_SHOT_STRING = """
+Comment: إن شاء الله يكون عرض صحاب 300 و 500 ميجا فيبر ياربي
+Class: 1
+
+Comment: الف مبروووك.. 
+Class: 1
+
+Comment:  - إتصالات الجزائر شكرا اتمنى لكم دوام الصحة والعافية 
+Class: 1
+
+Comment: C une fierté de faire partie de cette grande entreprise Algérienne de haute technologie et haute qualité
+Class: 1
+
+Comment: اتمنى لكم مزيد من التألق
+Class: 1
+
+Comment: زعما جابو المودام ؟
+Class: 2
+
+Comment: وفرو أجهزة مودام الباقي ساهل !
+Class: 2
+
+Comment: واش الفايدة تع العرض هذا هو اصلا لي مودام مهوش متوفر رنا قريب عام وحنا ستناو في جد موام هذا
+Class: 2
+
+Comment: Depuis un an et demi qu'on a installé w ma kan walou
+Class: 2
+
+Comment: قتلتونا بلكذب المودام غير متوفر عندي 4 أشهر ملي حطيت الطلب في ولاية خنشلة و مزال ماجابوش المودام
+Class: 2
+
+Comment: عندكم احساس و لا شريوه كما قالو خوتنا لمصريين
+Class: 3
+
+Comment: Kamel Dahmane الفايبر؟ مستحيل كامل عاجبتهم
+Class: 3
+
+Comment: ههههه نخلص مليون عادي كون يركبونا الفيبر 😂😂😂😂😂 كرهنا من 144p
+Class: 3
+
+Comment: إشهار بدون معنه
+Class: 3
+
+Comment: المشروع متوقف منذ اشهر
+Class: 4
+
+Comment: نتمنى تكملو في ايسطو وهران في اقرب وقت رانا نعانو مع ADSL
+Class: 4
+
+Comment: Fibre كاش واحد وصلوله الفيبر؟
+Class: 4
+
+Comment: ما هو الجديد وانا مزال ماعنديش الفيبر رغم الطلب ولالحاح
+Class: 4
+
+Comment: علبة الفيبر راكبة في الحي و لكن لا يوجد توصيل للمنزل للان
+Class: 4
+
+Comment: modem
+Class: 5
+
+Comment: يعني كي نطلعها ثلاثون ميغا كارطة تاع مائة الف قداه تحكملي؟
+Class: 5
+
+Comment: سآل الأماكن لي ما فيهاش الألياف البصرية إذا جابولنا الألياف السرعة تكون محدودة كيما ف ADSL؟
+Class: 5
+
+Comment: ماعرف كاش خبر على ايدوم 4G ماعرف تبقى قرد العش
+Class: 5
+
+Comment: هل متوفرة في حي عدل 1046 مسكن دويرة
+Class: 5
+
+Comment: عرض 20 ميجا نحيوه مدام مش قادرين تعطيونا حقنا
+Class: 6
+
+Comment: 4 سنوات وحنا نخلصو فالدار ماشفنا حتى bonus
+Class: 6
+
+Comment: لماذا التغيير في الرقم بدون تغيير سرعة التدفق هل من أجل الإشهار وفقط انا غير من 50 ميغا إلا 200 ميغا نظريا تغيرت وفي الواقع بقت قياس أقل من 50 ميغا
+Class: 6
+
+Comment: انا طلعت تدفق انترنات من 15 الى 20 عبر تطبيق my idoom لاكن سرعة لم تتغير
+Class: 6
+
+Comment: نقصوا الاسعار بزااااف غالية
+Class: 7
+
+Comment: علاه ماديروش في التطبيق خاصية التوقيف المؤقت للانترانات
+Class: 7
+
+Comment: وفرونا من بعد اي ساهلة
+Class: 7
+
+Comment: لازم ترجعو اتصال بتطبيقات الدفع بلا انترنت و مجاني ريقلوها يا اتصالات الجزائر
+Class: 7
+
+Comment: Promotion fin d'année ADSL idoom
+Class: 7
+
+Comment: رانا بلا تلفون ولا انترنت
+Class: 8
+
+Comment: ثلاثة اشهر بلا انترنت
+Class: 8
+
+Comment: votre site espace client ne fonctionne pas pourquoi?
+Class: 8
+
+Comment: ما عندنا الانترنيت ما نخلصوها من الدار
+Class: 8
+
+Comment: مشكل في 1.200جيق فيبر مدام نوكيا مخرج الانترنت 1جيق فقط كفاش راح تحلو هذا مشكل ومشكل ثاني فضاء الزبون ميمشيش مندو شهر
+Class: 8
+
+Comment: فضاء الزبون علاه منقدروش نسجلو فيه
+Class: 9
+
+Comment: هل موقع فضاء الزبون متوقف
+Class: 9
+
+Comment: ماراهيش توصل الفاتورة لا عن طريق الإيميل ولا عن طريق فضاء الزبون 
+Class: 9
+
+Comment: فضاء الزبون قرابة 20 يوم متوقف!!!!!!؟؟؟؟؟
+Class: 9
+
+Comment: برج الكيفان اظنها من العاصمة خارج تغطيتكم....احشموا بركاو بلا كذب....طلعنا الصواريخ للفضاء....بصح بالكذب....
+Class: 9"""
+
+def format_few_shot_prompt(comment):
+    """Format prompt with few-shot examples for classification."""
+    # Build the user prompt with hardcoded few-shot examples
+    user_prompt = f"""Here are some examples of how to classify comments:
+
+{FEW_SHOT_STRING}
+
+Now classify this comment:
+Comment: {comment}
+Class:"""
+    
+    return user_prompt
+
+def create_instruction_format(example, tokenizer, is_train=True):
+    """
+    Create instruction format for Gemma 3.
+    
+    For training: includes the answer
+    For inference: no answer
+    """
+    comment = preprocess_text(example.get(text_col, ""))
+    
+    # Build conversation
+    messages = [
+        {"role": "user", "content": SYSTEM_PROMPT + "\n\n" + format_few_shot_prompt(comment)}
+    ]
+    
+    if is_train:
+        label = example.get("Class", example.get("labels", 1))
+        if isinstance(label, str):
+            label = int(label.strip())
+        # Add assistant response (just the class number)
+        messages.append({"role": "assistant", "content": str(label)})
+    
+    # Apply chat template
+    if is_train:
+        # For training, we need the full conversation
+        text = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=False
+        )
+    else:
+        # For inference, add generation prompt
+        text = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+    
+    return text
+
+def prepare_train_dataset(dataset, tokenizer):
+    """Prepare training dataset with instruction format.
+    
+    Only compute loss on the assistant's response (class number),
+    not on the prompt. This is done by setting labels to -100 for prompt tokens.
+    """
+    
+    def process_example(example):
+        # Get the full text with answer
+        full_text = create_instruction_format(example, tokenizer, is_train=True)
+        
+        # Get the prompt only (without answer) to find where response starts
+        prompt_text = create_instruction_format(example, tokenizer, is_train=False)
+        
+        # Tokenize both
+        full_tokenized = tokenizer(
+            full_text,
+            truncation=True,
+            max_length=MAX_LENGTH,
+            padding=False,
+        )
+        
+        prompt_tokenized = tokenizer(
+            prompt_text,
+            truncation=True,
+            max_length=MAX_LENGTH,
+            padding=False,
+        )
+        
+        # Create labels: -100 for prompt tokens (ignored in loss), actual ids for response
+        prompt_len = len(prompt_tokenized["input_ids"])
+        labels = [-100] * prompt_len + full_tokenized["input_ids"][prompt_len:]
+        
+        # Ensure labels has same length as input_ids
+        if len(labels) < len(full_tokenized["input_ids"]):
+            labels = labels + full_tokenized["input_ids"][len(labels):]
+        elif len(labels) > len(full_tokenized["input_ids"]):
+            labels = labels[:len(full_tokenized["input_ids"])]
+        
+        full_tokenized["labels"] = labels
+        
+        return full_tokenized
+    
+    return dataset.map(process_example, remove_columns=dataset.column_names)
+
+# ===========================================================================
+# RS-LoRA Configuration (Rank-Stabilized LoRA)
+# ===========================================================================
+RS_LORA_CONFIG = {
+    "r": 64,                    # LoRA rank
+    "lora_alpha": 64,           # For RS-LoRA, alpha = r (rank-stabilized)
+    "lora_dropout": 0.05,       # Dropout for regularization
+    "target_modules": [         # Gemma attention/MLP modules
+        "q_proj", "k_proj", "v_proj", "o_proj",
+        "gate_proj", "up_proj", "down_proj",
+    ],
+    "use_rslora": True,         # Enable RS-LoRA
+}
+
+# Fine-Tuning Config  
+FT_CONFIG = {
+    "num_epochs": 3,
+    "batch_size": 4,
+    "gradient_accumulation_steps": 8,  # Effective batch = 32
+    "learning_rate": 2e-4,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.1,
+    "max_grad_norm": 1.0,
+}
+
+# ===========================================================================
+# Main Training
+# ===========================================================================
+print("\n" + "="*70)
+print("Gemma 3 4B - Instruction Fine-Tuning for Classification")
+print("="*70 + "\n")
+
+# Load tokenizer
+print(f"Loading tokenizer from: {BASE_MODEL}")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+
+# Set padding token
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token_id = tokenizer.eos_token_id
+
+# Set padding side for causal LM
+tokenizer.padding_side = "right"
+
+# Load model
+print(f"Loading model from: {BASE_MODEL}")
+model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    torch_dtype=torch.bfloat16,
+    trust_remote_code=True,
+    device_map="auto",
+    attn_implementation="eager",  # Use eager attention for compatibility
+)
+
+# Apply RS-LoRA
+print("\nApplying RS-LoRA configuration...")
+lora_config = LoraConfig(
+    task_type=TaskType.CAUSAL_LM,
+    r=RS_LORA_CONFIG["r"],
+    lora_alpha=RS_LORA_CONFIG["lora_alpha"],
+    lora_dropout=RS_LORA_CONFIG["lora_dropout"],
+    target_modules=RS_LORA_CONFIG["target_modules"],
+    bias="none",
+    use_rslora=RS_LORA_CONFIG["use_rslora"],  # RS-LoRA for better stability
+)
+
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+# Load training data
+print(f"\nLoading training data from: {TRAIN_FILE}")
+train_ds = load_dataset("csv", data_files=TRAIN_FILE, split="train")
+print(f"Total training samples: {len(train_ds)}")
+
+# Preprocess and check data
+print("\nPreprocessing text data...")
+def preprocess_dataset(example):
+    example["clean_text"] = preprocess_text(example.get(text_col, ""))
+    return example
+
+train_ds = train_ds.map(preprocess_dataset)
+
+# Show preprocessing examples
+print("\nPreprocessing examples:")
+for i in range(min(3, len(train_ds))):
+    original = train_ds[i].get(text_col, "")[:80]
+    cleaned = train_ds[i].get("clean_text", "")[:80]
+    print(f"  Original: {original}...")
+    print(f"  Cleaned:  {cleaned}...")
+    print()
+
+# Train/val split
+split = train_ds.train_test_split(test_size=0.01, seed=42)
+train_split = split["train"]
+eval_split = split["test"]
+print(f"Train split: {len(train_split)} | Eval split: {len(eval_split)}")
+
+# Prepare datasets
+print("\nPreparing instruction-formatted datasets...")
+train_dataset = prepare_train_dataset(train_split, tokenizer)
+eval_dataset = prepare_train_dataset(eval_split, tokenizer)
+
+# Show example formatted input
+print("\nExample formatted input (truncated):")
+example_text = create_instruction_format(train_split[0], tokenizer, is_train=True)
+print(example_text[:500] + "..." if len(example_text) > 500 else example_text)
+
+# Data collator
+data_collator = DataCollatorForSeq2Seq(
+    tokenizer=tokenizer,
+    padding=True,
+    return_tensors="pt",
+)
+
+# Training arguments
+print("\n--- Fine-Tuning Hyperparameters ---")
+for k, v in FT_CONFIG.items():
+    print(f"  {k}: {v}")
+print(f"\n--- RS-LoRA Configuration ---")
+print(f"  rank: {RS_LORA_CONFIG['r']}")
+print(f"  alpha: {RS_LORA_CONFIG['lora_alpha']}")
+print(f"  dropout: {RS_LORA_CONFIG['lora_dropout']}")
+print(f"  use_rslora: {RS_LORA_CONFIG['use_rslora']}")
+
+training_args = TrainingArguments(
+    output_dir=FT_OUTPUT_DIR,
+    num_train_epochs=FT_CONFIG["num_epochs"],
+    per_device_train_batch_size=FT_CONFIG["batch_size"],
+    per_device_eval_batch_size=FT_CONFIG["batch_size"],
+    gradient_accumulation_steps=FT_CONFIG["gradient_accumulation_steps"],
+    learning_rate=FT_CONFIG["learning_rate"],
+    weight_decay=FT_CONFIG["weight_decay"],
+    warmup_ratio=FT_CONFIG["warmup_ratio"],
+    max_grad_norm=FT_CONFIG["max_grad_norm"],
+    bf16=True,
+    logging_steps=10,
+    eval_strategy="epoch",
+    save_strategy="epoch",
+    save_total_limit=2,
+    load_best_model_at_end=True,
+    metric_for_best_model="eval_loss",
+    greater_is_better=False,
+    dataloader_num_workers=4,
+    report_to="none",
+    gradient_checkpointing=True,
+    gradient_checkpointing_kwargs={"use_reentrant": False},
+)
+
+# Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    tokenizer=tokenizer,
+    data_collator=data_collator,
+)
+
+print("\nStarting fine-tuning...")
+trainer.train()
+
+print(f"\nSaving model to: {FT_OUTPUT_DIR}")
+trainer.save_model(FT_OUTPUT_DIR)
+tokenizer.save_pretrained(FT_OUTPUT_DIR)
+
+# Save config
+config = {
+    "base_model": BASE_MODEL,
+    "num_labels": NUM_LABELS,
+    "id2label": ID2LABEL,
+    "label2id": LABEL2ID,
+    "rs_lora_config": RS_LORA_CONFIG,
+    "ft_config": FT_CONFIG,
+}
+with open(os.path.join(FT_OUTPUT_DIR, "training_config.json"), "w") as f:
+    json.dump(config, f, indent=2)
+
+# ===========================================================================
+# Inference on Test Set
+# ===========================================================================
+print("\n" + "="*70)
+print("Inference on Test Set")
+print("="*70 + "\n")
+
+# Load test data
+test_ds = load_dataset("csv", data_files=TEST_FILE, split="train")
+print(f"Test samples: {len(test_ds)}")
+
+# Preprocess test data
+test_ds = test_ds.map(preprocess_dataset)
+
+# Run inference
+print("Running inference...")
+model.eval()
+
+all_preds = []
+batch_size = 1  # Process one at a time for generation
+
+from tqdm import tqdm
+
+for i in tqdm(range(len(test_ds)), desc="Predicting"):
+    example = test_ds[i]
+    
+    # Create prompt (without answer)
+    prompt = create_instruction_format(example, tokenizer, is_train=False)
+    
+    # Tokenize
+    inputs = tokenizer(prompt, return_tensors="pt", truncation=True, max_length=MAX_LENGTH)
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    
+    # Generate
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=5,
+            do_sample=False,
+            pad_token_id=tokenizer.pad_token_id,
+            eos_token_id=tokenizer.eos_token_id,
+        )
+    
+    # Decode only the new tokens
+    generated_tokens = outputs[0][inputs["input_ids"].shape[1]:]
+    generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()
+    
+    # Extract class number
+    try:
+        # Try to extract first number found
+        match = re.search(r'\b([1-9])\b', generated_text)
+        if match:
+            pred_class = int(match.group(1))
+        else:
+            pred_class = 1  # Default
+    except:
+        pred_class = 1  # Default
+    
+    all_preds.append(pred_class)
+
+# Save predictions
+import pandas as pd
+test_df = pd.read_csv(TEST_FILE)
+test_df["Predicted_Class"] = all_preds
+
+output_file = "test_predictions_gemma3.csv"
+test_df.to_csv(output_file, index=False)
+print(f"\nPredictions saved to: {output_file}")
+
+# Show sample predictions
+print("\nSample predictions:")
+for i in range(min(10, len(test_df))):
+    text = str(test_df.iloc[i][text_col])[:60] + "..." if len(str(test_df.iloc[i][text_col])) > 60 else str(test_df.iloc[i][text_col])
+    pred = test_df.iloc[i]["Predicted_Class"]
+    print(f"  [{i+1}] Class {pred}: {text}")
+
+# Class distribution
+print("\nPrediction distribution:")
+pred_counts = test_df["Predicted_Class"].value_counts().sort_index()
+for class_label, count in pred_counts.items():
+    print(f"  Class {class_label}: {count} samples ({count/len(test_df)*100:.1f}%)")
+
+# ===========================================================================
+# Summary
+# ===========================================================================
+print("\n" + "="*70)
+print("TRAINING COMPLETE!")
+print("="*70)
+print(f"\nBase Model:                {BASE_MODEL}")
+print(f"Fine-tuned model saved to: {FT_OUTPUT_DIR}")
+print(f"Predictions saved to:      {output_file}")
+print(f"\nTraining samples:          {len(train_split)}")
+print(f"Validation samples:        {len(eval_split)}")
+print(f"Test samples:              {len(test_df)}")
+print(f"RS-LoRA rank:              {RS_LORA_CONFIG['r']}")
+print(f"Use RS-LoRA:               {RS_LORA_CONFIG['use_rslora']}")

Code/inference_camelbert.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Inference script for CPT+finetuned MARBERTv2 telecom classification model.
+
+Loads the model from ./telecom_marbertv2_cpt_ft and runs predictions on test.csv
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import torch
+
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    AutoConfig,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    precision_recall_fscore_support,
+    classification_report,
+    confusion_matrix,
+)
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+MODEL_DIR = "./telecom_camelbert_cpt_ft"
+OUTPUT_FILE = "./test_predictions_camelbert_cpt_ft.csv"
+
+MAX_LENGTH = 256
+BATCH_SIZE = 64
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+# -------------------------------------------------------------------
+# 2. Load test data
+# -------------------------------------------------------------------
+print(f"Loading test data from: {TEST_FILE}")
+test_df = pd.read_csv(TEST_FILE)
+print(f"Test samples: {len(test_df)}")
+print(f"Columns: {test_df.columns.tolist()}")
+
+# Check if test data has labels
+has_labels = "Class" in test_df.columns
+if has_labels:
+    print("Test data contains labels - will compute metrics")
+else:
+    print("Test data has no labels - will only generate predictions")
+
+# -------------------------------------------------------------------
+# 3. Load model and tokenizer
+# -------------------------------------------------------------------
+print(f"\nLoading model from: {MODEL_DIR}")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
+
+# Load config to get label mappings
+config_path = os.path.join(MODEL_DIR, "config.json")
+if os.path.exists(config_path):
+    import json
+    with open(config_path, 'r') as f:
+        config_data = json.load(f)
+    
+    if 'id2label' in config_data:
+        id2label = {int(k): int(v) for k, v in config_data['id2label'].items()}
+        # Create label2id with both string and int keys for robustness
+        label2id = {}
+        for k, v in id2label.items():
+            label2id[v] = k  # int key -> int value
+            label2id[str(v)] = k  # string key -> int value
+        num_labels = len(id2label)
+    else:
+        # Fallback: infer from test data if available
+        if has_labels:
+            unique_classes = sorted(test_df["Class"].unique())
+            label2id = {label: idx for idx, label in enumerate(unique_classes)}
+            id2label = {idx: label for label, idx in label2id.items()}
+            num_labels = len(unique_classes)
+        else:
+            raise ValueError("Cannot determine number of labels without config or test labels")
+else:
+    # Fallback: infer from test data if available
+    if has_labels:
+        unique_classes = sorted(test_df["Class"].unique())
+        label2id = {label: idx for idx, label in enumerate(unique_classes)}
+        id2label = {idx: label for label, idx in label2id.items()}
+        num_labels = len(unique_classes)
+    else:
+        raise ValueError("Cannot find config.json and test data has no labels")
+
+print(f"Number of classes: {num_labels}")
+print(f"Label mapping: {id2label}")
+
+# Load model
+model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR)
+model = model.to(device)
+model.eval()
+print("Model loaded successfully!")
+
+# -------------------------------------------------------------------
+# 4. Run inference
+# -------------------------------------------------------------------
+print("\nRunning inference...")
+
+all_predictions = []
+all_probabilities = []
+
+# Process in batches for efficiency
+for i in range(0, len(test_df), BATCH_SIZE):
+    batch_texts = test_df["Commentaire client"].iloc[i:i+BATCH_SIZE].tolist()
+    
+    # Tokenize
+    inputs = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH,
+        return_tensors="pt",
+    ).to(device)
+    
+    # Predict
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probs = torch.softmax(logits, dim=-1)
+        predictions = torch.argmax(logits, dim=-1)
+    
+    all_predictions.extend(predictions.cpu().numpy())
+    all_probabilities.extend(probs.cpu().numpy())
+    
+    if (i // BATCH_SIZE + 1) % 10 == 0:
+        print(f"Processed {i + len(batch_texts)}/{len(test_df)} samples...")
+
+print(f"Inference complete! Processed {len(all_predictions)} samples")
+
+# -------------------------------------------------------------------
+# 5. Save predictions
+# -------------------------------------------------------------------
+# Convert predictions to class names (1-9)
+predicted_classes = [id2label[pred] for pred in all_predictions]
+
+# Add predictions to dataframe
+test_df["Predicted_Class"] = predicted_classes
+test_df["Predicted_Label_ID"] = all_predictions
+
+# Add probability for each class
+for idx, class_name in id2label.items():
+    test_df[f"Prob_{class_name}"] = [probs[idx] for probs in all_probabilities]
+
+# Add confidence (max probability)
+test_df["Confidence"] = [max(probs) for probs in all_probabilities]
+
+# Save results
+test_df.to_csv(OUTPUT_FILE, index=False)
+print(f"\nPredictions saved to: {OUTPUT_FILE}")
+
+# -------------------------------------------------------------------
+# 6. Compute metrics (if labels available)
+# -------------------------------------------------------------------
+if has_labels:
+    print("\n" + "="*80)
+    print("EVALUATION METRICS")
+    print("="*80)
+    
+    # Convert true labels to indices
+    true_labels = test_df["Class"].map(label2id).values
+    pred_labels = np.array(all_predictions)
+    
+    # Overall metrics
+    accuracy = accuracy_score(true_labels, pred_labels)
+    print(f"\nAccuracy: {accuracy:.4f}")
+    
+    # Weighted metrics (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='weighted', zero_division=0
+    )
+    print(f"\nWeighted Metrics:")
+    print(f"  Precision: {precision_w:.4f}")
+    print(f"  Recall:    {recall_w:.4f}")
+    print(f"  F1 Score:  {f1_w:.4f}")
+    
+    # Macro metrics (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='macro', zero_division=0
+    )
+    print(f"\nMacro Metrics:")
+    print(f"  Precision: {precision_m:.4f}")
+    print(f"  Recall:    {recall_m:.4f}")
+    print(f"  F1 Score:  {f1_m:.4f}")
+    
+    # Per-class metrics
+    print(f"\nPer-Class Metrics:")
+    per_class_f1 = f1_score(true_labels, pred_labels, average=None, zero_division=0)
+    per_class_precision, per_class_recall, _, support = precision_recall_fscore_support(
+        true_labels, pred_labels, average=None, zero_division=0
+    )
+    
+    for idx in range(num_labels):
+        class_name = id2label[idx]
+        print(f"\n  Class {class_name}:")
+        print(f"    Precision: {per_class_precision[idx]:.4f}")
+        print(f"    Recall:    {per_class_recall[idx]:.4f}")
+        print(f"    F1 Score:  {per_class_f1[idx]:.4f}")
+        print(f"    Support:   {int(support[idx])}")
+    
+    # Classification report
+    print("\n" + "="*80)
+    print("DETAILED CLASSIFICATION REPORT")
+    print("="*80)
+    target_names = [str(id2label[i]) for i in range(num_labels)]
+    print(classification_report(true_labels, pred_labels, target_names=target_names, zero_division=0))
+    
+    # Confusion matrix
+    print("\n" + "="*80)
+    print("CONFUSION MATRIX")
+    print("="*80)
+    cm = confusion_matrix(true_labels, pred_labels)
+    
+    # Print confusion matrix with labels
+    print("\nTrue \\ Predicted", end="")
+    for i in range(num_labels):
+        print(f"\t{id2label[i]}", end="")
+    print()
+    
+    for i in range(num_labels):
+        print(f"{id2label[i]:<15}", end="")
+        for j in range(num_labels):
+            print(f"\t{cm[i][j]}", end="")
+        print()
+    
+    # Save confusion matrix to CSV
+    cm_df = pd.DataFrame(
+        cm,
+        index=[str(id2label[i]) for i in range(num_labels)],
+        columns=[str(id2label[i]) for i in range(num_labels)]
+    )
+    cm_df.to_csv("./confusion_matrix_marbertv2_cpt_ft.csv")
+    print("\nConfusion matrix saved to: ./confusion_matrix_marbertv2_cpt_ft.csv")
+
+# -------------------------------------------------------------------
+# 7. Show sample predictions
+# -------------------------------------------------------------------
+print("\n" + "="*80)
+print("SAMPLE PREDICTIONS (CPT+Finetuned MARBERTv2)")
+print("="*80)
+
+# Show first 5 predictions
+num_samples = min(5, len(test_df))
+for i in range(num_samples):
+    print(f"\nSample {i+1}:")
+    print(f"Text: {test_df['Commentaire client'].iloc[i]}")
+    if has_labels:
+        print(f"True Class: {test_df['Class'].iloc[i]}")
+    print(f"Predicted Class: {predicted_classes[i]}")
+    print(f"Confidence: {test_df['Confidence'].iloc[i]:.4f}")
+    print(f"Probabilities:")
+    for idx, class_name in id2label.items():
+        print(f"  Class {class_name}: {all_probabilities[i][idx]:.4f}")
+
+print("\n" + "="*80)
+print("Inference completed successfully!")
+print(f"Model used: CPT+Finetuned MARBERTv2 from {MODEL_DIR}")
+print("="*80)

Code/inference_dziribert.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Inference script for trained DziriBERT telecom classification model.
+
+Loads the trained model and runs predictions on test.csv
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import torch
+
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    AutoConfig,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    precision_recall_fscore_support,
+    classification_report,
+    confusion_matrix,
+)
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+MODEL_DIR = "./telecom_dziribert_final"
+OUTPUT_FILE = "./test_predictions_dziribert.csv"
+
+MAX_LENGTH = 512
+BATCH_SIZE = 64
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+# -------------------------------------------------------------------
+# 2. Load test data
+# -------------------------------------------------------------------
+print(f"Loading test data from: {TEST_FILE}")
+test_df = pd.read_csv(TEST_FILE)
+print(f"Test samples: {len(test_df)}")
+print(f"Columns: {test_df.columns.tolist()}")
+
+# Check if test data has labels
+has_labels = "Class" in test_df.columns
+if has_labels:
+    print("Test data contains labels - will compute metrics")
+else:
+    print("Test data has no labels - will only generate predictions")
+
+# -------------------------------------------------------------------
+# 3. Load model and tokenizer
+# -------------------------------------------------------------------
+print(f"\nLoading model from: {MODEL_DIR}")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
+
+# Load config to get label mappings
+config_path = os.path.join(MODEL_DIR, "config.json")
+if os.path.exists(config_path):
+    import json
+    with open(config_path, 'r') as f:
+        config_data = json.load(f)
+    
+    if 'id2label' in config_data:
+        id2label = {int(k): int(v) for k, v in config_data['id2label'].items()}
+        # Create label2id with both string and int keys for robustness
+        label2id = {}
+        for k, v in id2label.items():
+            label2id[v] = k  # int key -> int value
+            label2id[str(v)] = k  # string key -> int value
+        num_labels = len(id2label)
+    else:
+        # Fallback: infer from test data if available
+        if has_labels:
+            unique_classes = sorted(test_df["Class"].unique())
+            label2id = {label: idx for idx, label in enumerate(unique_classes)}
+            id2label = {idx: label for label, idx in label2id.items()}
+            num_labels = len(unique_classes)
+        else:
+            raise ValueError("Cannot determine number of labels without config or test labels")
+else:
+    # Fallback: infer from test data if available
+    if has_labels:
+        unique_classes = sorted(test_df["Class"].unique())
+        label2id = {label: idx for idx, label in enumerate(unique_classes)}
+        id2label = {idx: label for label, idx in label2id.items()}
+        num_labels = len(unique_classes)
+    else:
+        raise ValueError("Cannot find config.json and test data has no labels")
+
+print(f"Number of classes: {num_labels}")
+print(f"Label mapping: {id2label}")
+
+# Load model
+model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR)
+model = model.to(device)
+model.eval()
+print("Model loaded successfully!")
+
+# -------------------------------------------------------------------
+# 4. Run inference
+# -------------------------------------------------------------------
+print("\nRunning inference...")
+
+all_predictions = []
+all_probabilities = []
+
+# Process in batches for efficiency
+for i in range(0, len(test_df), BATCH_SIZE):
+    batch_texts = test_df["Commentaire client"].iloc[i:i+BATCH_SIZE].tolist()
+    
+    # Tokenize
+    inputs = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH,
+        return_tensors="pt",
+    ).to(device)
+    
+    # Predict
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probs = torch.softmax(logits, dim=-1)
+        predictions = torch.argmax(logits, dim=-1)
+    
+    all_predictions.extend(predictions.cpu().numpy())
+    all_probabilities.extend(probs.cpu().numpy())
+    
+    if (i // BATCH_SIZE + 1) % 10 == 0:
+        print(f"Processed {i + len(batch_texts)}/{len(test_df)} samples...")
+
+print(f"Inference complete! Processed {len(all_predictions)} samples")
+
+# -------------------------------------------------------------------
+# 5. Save predictions
+# -------------------------------------------------------------------
+# Convert predictions to class names (1-9)
+predicted_classes = [id2label[pred] for pred in all_predictions]
+
+# Add predictions to dataframe
+test_df["Predicted_Class"] = predicted_classes
+test_df["Predicted_Label_ID"] = all_predictions
+
+# Add probability for each class
+for idx, class_name in id2label.items():
+    test_df[f"Prob_{class_name}"] = [probs[idx] for probs in all_probabilities]
+
+# Add confidence (max probability)
+test_df["Confidence"] = [max(probs) for probs in all_probabilities]
+
+# Save results
+test_df.to_csv(OUTPUT_FILE, index=False)
+print(f"\nPredictions saved to: {OUTPUT_FILE}")
+
+# -------------------------------------------------------------------
+# 6. Compute metrics (if labels available)
+# -------------------------------------------------------------------
+if has_labels:
+    print("\n" + "="*80)
+    print("EVALUATION METRICS")
+    print("="*80)
+    
+    # Convert true labels to indices
+    true_labels = test_df["Class"].map(label2id).values
+    pred_labels = np.array(all_predictions)
+    
+    # Overall metrics
+    accuracy = accuracy_score(true_labels, pred_labels)
+    print(f"\nAccuracy: {accuracy:.4f}")
+    
+    # Weighted metrics (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='weighted', zero_division=0
+    )
+    print(f"\nWeighted Metrics:")
+    print(f"  Precision: {precision_w:.4f}")
+    print(f"  Recall:    {recall_w:.4f}")
+    print(f"  F1 Score:  {f1_w:.4f}")
+    
+    # Macro metrics (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='macro', zero_division=0
+    )
+    print(f"\nMacro Metrics:")
+    print(f"  Precision: {precision_m:.4f}")
+    print(f"  Recall:    {recall_m:.4f}")
+    print(f"  F1 Score:  {f1_m:.4f}")
+    
+    # Per-class metrics
+    print(f"\nPer-Class Metrics:")
+    per_class_f1 = f1_score(true_labels, pred_labels, average=None, zero_division=0)
+    per_class_precision, per_class_recall, _, support = precision_recall_fscore_support(
+        true_labels, pred_labels, average=None, zero_division=0
+    )
+    
+    for idx in range(num_labels):
+        class_name = id2label[idx]
+        print(f"\n  Class {class_name}:")
+        print(f"    Precision: {per_class_precision[idx]:.4f}")
+        print(f"    Recall:    {per_class_recall[idx]:.4f}")
+        print(f"    F1 Score:  {per_class_f1[idx]:.4f}")
+        print(f"    Support:   {int(support[idx])}")
+    
+    # Classification report
+    print("\n" + "="*80)
+    print("DETAILED CLASSIFICATION REPORT")
+    print("="*80)
+    target_names = [str(id2label[i]) for i in range(num_labels)]
+    print(classification_report(true_labels, pred_labels, target_names=target_names, zero_division=0))
+    
+    # Confusion matrix
+    print("\n" + "="*80)
+    print("CONFUSION MATRIX")
+    print("="*80)
+    cm = confusion_matrix(true_labels, pred_labels)
+    
+    # Print confusion matrix with labels
+    print("\nTrue \\ Predicted", end="")
+    for i in range(num_labels):
+        print(f"\t{id2label[i]}", end="")
+    print()
+    
+    for i in range(num_labels):
+        print(f"{id2label[i]:<15}", end="")
+        for j in range(num_labels):
+            print(f"\t{cm[i][j]}", end="")
+        print()
+    
+    # Save confusion matrix to CSV
+    cm_df = pd.DataFrame(
+        cm,
+        index=[str(id2label[i]) for i in range(num_labels)],
+        columns=[str(id2label[i]) for i in range(num_labels)]
+    )
+    cm_df.to_csv("./confusion_matrix_dziribert.csv")
+    print("\nConfusion matrix saved to: ./confusion_matrix_dziribert.csv")
+
+# -------------------------------------------------------------------
+# 7. Show sample predictions
+# -------------------------------------------------------------------
+print("\n" + "="*80)
+print("SAMPLE PREDICTIONS")
+print("="*80)
+
+# Show first 5 predictions
+num_samples = min(5, len(test_df))
+for i in range(num_samples):
+    print(f"\nSample {i+1}:")
+    print(f"Text: {test_df['Commentaire client'].iloc[i]}")
+    if has_labels:
+        print(f"True Class: {test_df['Class'].iloc[i]}")
+    print(f"Predicted Class: {predicted_classes[i]}")
+    print(f"Confidence: {test_df['Confidence'].iloc[i]:.4f}")
+    print(f"Probabilities:")
+    for idx, class_name in id2label.items():
+        print(f"  Class {class_name}: {all_probabilities[i][idx]:.4f}")
+
+print("\n" + "="*80)
+print("Inference completed successfully!")
+print("="*80)

Code/inference_gemma3.py

@@ -0,0 +1,409 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Gemma 3 12B - Fast Inference for Classification
+
+Load the fine-tuned Gemma 3 model and run inference on test set.
+Uses batch processing for faster inference.
+
+Usage:
+    python inference_gemma3.py
+"""
+
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+import re
+import torch
+import pandas as pd
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+
+# ---------------------------
+# Paths & Config 
+# ---------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+BASE_MODEL = "google/gemma-3-4b-it"  # Must match training base model
+ADAPTER_PATH = "./gemma3_classification_ft"  # LoRA adapter path
+
+MAX_LENGTH = 2048
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Device: {DEVICE}")
+
+# Enable TF32 for A100
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+text_col = "Commentaire client"
+
+# ===========================================================================
+# System Prompt and Few-Shot Examples (same as training)
+# ===========================================================================
+SYSTEM_PROMPT = """You are an expert Algerian linguist and data labeler. Your task is to classify customer comments from Algérie Télécom's social media into one of 9 specific categories.
+
+## CLASSES (DETAILED DESCRIPTIONS)
+- **Class 1 (Wish/Positive Anticipation):** Comments expressing a wish, a hopeful anticipation, or general positive feedback/appreciation for future services or offers.
+- **Class 2 (Complaint: Equipment/Supply):** Comments complaining about the lack, unavailability, or delay in the supply of necessary equipment (e.g., modems, fiber optics devices).
+- **Class 3 (Complaint: Marketing/Advertising):** Comments criticizing advertisements, marketing campaigns, or their lack of realism/meaning.
+- **Class 4 (Complaint: Installation/Deployment):** Comments about delays, stoppages, or failure in service installation, network expansion, or fiber optics deployment (e.g., digging issues).
+- **Class 5 (Inquiry/Request for Information):** Comments asking for eligibility, connection dates, service status, coverage details, or specific contact information.
+- **Class 6 (Complaint: Technical Support/Intervention):** Comments regarding delays in repair interventions, issues with technical staff competence, or unsatisfactory customer service agency visits.
+- **Class 7 (Pricing/Service Enhancement):** Comments focused on pricing, requests for cost reduction, or suggestions for general service/app functionality enhancements.
+- **Class 8 (Complaint: Total Service Outage/Disconnection):** Comments indicating a complete, sustained loss of service (e.g., no phone, no internet, total disconnection).
+- **Class 9 (Complaint: Service Performance/Quality):** Comments about technical issues impacting performance (e.g., slow speed, high latency, broken website/portal, coverage claims).
+
+Respond with ONLY the class number (1-9). Do not include any explanation."""
+
+FEW_SHOT_STRING = """
+Comment: إن شاء الله يكون عرض صحاب 300 و 500 ميجا فيبر ياربي
+Class: 1
+
+Comment: الف مبروووك.. 
+Class: 1
+
+Comment:  - إتصالات الجزائر شكرا اتمنى لكم دوام الصحة والعافية 
+Class: 1
+
+Comment: C une fierté de faire partie de cette grande entreprise Algérienne de haute technologie et haute qualité
+Class: 1
+
+Comment: اتمنى لكم مزيد من التألق
+Class: 1
+
+Comment: زعما جابو المودام ؟
+Class: 2
+
+Comment: وفرو أجهزة مودام الباقي ساهل !
+Class: 2
+
+Comment: واش الفايدة تع العرض هذا هو اصلا لي مودام مهوش متوفر رنا قريب عام وحنا ستناو في جد موام هذا
+Class: 2
+
+Comment: Depuis un an et demi qu'on a installé w ma kan walou
+Class: 2
+
+Comment: قتلتونا بلكذب المودام غير متوفر عندي 4 أشهر ملي حطيت الطلب في ولاية خنشلة و مزال ماجابوش المودام
+Class: 2
+
+Comment: عندكم احساس و لا شريوه كما قالو خوتنا لمصريين
+Class: 3
+
+Comment: Kamel Dahmane الفايبر؟ مستحيل كامل عاجبتهم
+Class: 3
+
+Comment: ههههه نخلص مليون عادي كون يركبونا الفيبر 😂😂😂😂😂 كرهنا من 144p
+Class: 3
+
+Comment: إشهار بدون معنه
+Class: 3
+
+Comment: المشروع متوقف منذ اشهر
+Class: 4
+
+Comment: نتمنى تكملو في ايسطو وهران في اقرب وقت رانا نعانو مع ADSL
+Class: 4
+
+Comment: Fibre كاش واحد وصلوله الفيبر؟
+Class: 4
+
+Comment: ما هو الجديد وانا مزال ماعنديش الفيبر رغم الطلب ولالحاح
+Class: 4
+
+Comment: علبة الفيبر راكبة في الحي و لكن لا يوجد توصيل للمنزل للان
+Class: 4
+
+Comment: modem
+Class: 5
+
+Comment: يعني كي نطلعها ثلاثون ميغا كارطة تاع مائة الف قداه تحكملي؟
+Class: 5
+
+Comment: سآل الأماكن لي ما فيهاش الألياف البصرية إذا جابولنا الألياف السرعة تكون محدودة كيما ف ADSL؟
+Class: 5
+
+Comment: ماعرف كاش خبر على ايدوم 4G ماعرف تبقى قرد العش
+Class: 5
+
+Comment: هل متوفرة في حي عدل 1046 مسكن دويرة
+Class: 5
+
+Comment: عرض 20 ميجا نحيوه مدام مش قادرين تعطيونا حقنا
+Class: 6
+
+Comment: 4 سنوات وحنا نخلصو فالدار ماشفنا حتى bonus
+Class: 6
+
+Comment: لماذا التغيير في الرقم بدون تغيير سرعة التدفق هل من أجل الإشهار وفقط انا غير من 50 ميغا إلا 200 ميغا نظريا تغيرت وفي الواقع بقت قياس أقل من 50 ميغا
+Class: 6
+
+Comment: انا طلعت تدفق انترنات من 15 الى 20 عبر تطبيق my idoom لاكن سرعة لم تتغير
+Class: 6
+
+Comment: نقصوا الاسعار بزااااف غالية
+Class: 7
+
+Comment: علاه ماديروش في التطبيق خاصية التوقيف المؤقت للانترانات
+Class: 7
+
+Comment: وفرونا من بعد اي ساهلة
+Class: 7
+
+Comment: لازم ترجعو اتصال بتطبيقات الدفع بلا انترنت و مجاني ريقلوها يا اتصالات الجزائر
+Class: 7
+
+Comment: Promotion fin d'année ADSL idoom
+Class: 7
+
+Comment: رانا بلا تلفون ولا انترنت
+Class: 8
+
+Comment: ثلاثة اشهر بلا انترنت
+Class: 8
+
+Comment: votre site espace client ne fonctionne pas pourquoi?
+Class: 8
+
+Comment: ما عندنا الانترنيت ما نخلصوها من الدار
+Class: 8
+
+Comment: مشكل في 1.200جيق فيبر مدام نوكيا مخرج الانترنت 1جيق فقط كفاش راح تحلو هذا مشكل ومشكل ثاني فضاء الزبون ميمشيش مندو شهر
+Class: 8
+
+Comment: فضاء الزبون علاه منقدروش نسجلو فيه
+Class: 9
+
+Comment: هل موقع فضاء الزبون متوقف
+Class: 9
+
+Comment: ماراهيش توصل الفاتورة لا عن طريق الإيميل ولا عن طريق فضاء الزبون 
+Class: 9
+
+Comment: فضاء الزبون قرابة 20 يوم متوقف!!!!!!؟؟؟؟؟
+Class: 9
+
+Comment: برج الكيفان اظنها من العاصمة خارج تغطيتكم....احشموا بركاو بلا كذب....طلعنا الصواريخ للفضاء....بصح بالكذب....
+Class: 9"""
+
+# ===========================================================================
+# Text Preprocessing
+# ===========================================================================
+def preprocess_text(text):
+    """Preprocess text: remove tatweel, emojis, URLs, phone numbers."""
+    if not isinstance(text, str):
+        return ""
+    
+    # Remove URLs
+    text = re.sub(r'https?://\S+|www\.\S+', '', text)
+    
+    # Remove email addresses
+    text = re.sub(r'\S+@\S+', '', text)
+    
+    # Remove phone numbers
+    text = re.sub(r'[\+]?[(]?[0-9]{1,4}[)]?[-\s\./0-9]{6,}', '', text)
+    text = re.sub(r'\b0[567]\d{8}\b', '', text)
+    text = re.sub(r'\b0[23]\d{7,8}\b', '', text)
+    
+    # Remove mentions
+    text = re.sub(r'@\w+', '', text)
+    
+    # Remove Arabic tatweel
+    text = re.sub(r'ـ+', '', text)
+    
+    # Remove emojis
+    emoji_pattern = re.compile("["
+        u"\U0001F600-\U0001F64F"
+        u"\U0001F300-\U0001F5FF"
+        u"\U0001F680-\U0001F6FF"
+        u"\U0001F1E0-\U0001F1FF"
+        u"\U00002702-\U000027B0"
+        u"\U000024C2-\U0001F251"
+        u"\U0001f926-\U0001f937"
+        u"\U00010000-\U0010ffff"
+        u"\u2640-\u2642"
+        u"\u2600-\u2B55"
+        u"\u200d"
+        u"\u23cf"
+        u"\u23e9"
+        u"\u231a"
+        u"\ufe0f"
+        u"\u3030"
+        "]+", flags=re.UNICODE)
+    text = emoji_pattern.sub('', text)
+    
+    # Remove platform names
+    text = re.sub(r'Algérie Télécom - إتصالات الجزائر', '', text, flags=re.IGNORECASE)
+    text = re.sub(r'Algérie Télécom', '', text, flags=re.IGNORECASE)
+    text = re.sub(r'إتصالات الجزائر', '', text)
+    
+    # Remove repeated characters
+    text = re.sub(r'(.)\1{3,}', r'\1\1\1', text)
+    
+    # Normalize whitespace
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    return text
+
+def format_prompt(comment):
+    """Format prompt for inference."""
+    user_prompt = f"""Here are some examples of how to classify comments:
+
+{FEW_SHOT_STRING}
+
+Now classify this comment:
+Comment: {comment}
+Class:"""
+    return user_prompt
+
+def create_inference_prompt(comment, tokenizer):
+    """Create full prompt for inference."""
+    clean_comment = preprocess_text(comment)
+    
+    messages = [
+        {"role": "user", "content": SYSTEM_PROMPT + "\n\n" + format_prompt(clean_comment)}
+    ]
+    
+    # Apply chat template with generation prompt
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    
+    return text
+
+def extract_class(generated_text):
+    """Extract class number from generated text."""
+    try:
+        match = re.search(r'\b([1-9])\b', generated_text)
+        if match:
+            return int(match.group(1))
+        return 1  # Default
+    except:
+        return 1
+
+# ===========================================================================
+# Main Inference
+# ===========================================================================
+print("\n" + "="*70)
+print("Gemma 3 4B - Fast Batch Inference")
+print("="*70 + "\n")
+
+# Load tokenizer
+print(f"Loading tokenizer from: {BASE_MODEL}")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token_id = tokenizer.eos_token_id
+
+tokenizer.padding_side = "left"  # Left padding for batch generation
+
+# Load base model
+print(f"Loading base model from: {BASE_MODEL}")
+model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    torch_dtype=torch.bfloat16,
+    trust_remote_code=True,
+    device_map="auto",
+    attn_implementation="eager",  # Flash attention not available
+)
+
+# Load LoRA adapter
+print(f"Loading LoRA adapter from: {ADAPTER_PATH}")
+model = PeftModel.from_pretrained(model, ADAPTER_PATH)
+model.eval()
+
+print(f"\nModel loaded successfully!")
+
+# Load test data
+print(f"\nLoading test data from: {TEST_FILE}")
+test_df = pd.read_csv(TEST_FILE)
+print(f"Test samples: {len(test_df)}")
+
+# Batch inference settings - use smaller batch for long prompts
+BATCH_SIZE = 8  # Small batch to avoid OOM with long few-shot prompts
+
+# Prepare all prompts first
+print("\nPreparing prompts...")
+all_prompts = []
+for i in tqdm(range(len(test_df)), desc="Preparing"):
+    comment = str(test_df.iloc[i][text_col])
+    prompt = create_inference_prompt(comment, tokenizer)
+    all_prompts.append(prompt)
+
+# Run batch inference
+print(f"\nRunning batch inference (batch_size={BATCH_SIZE})...")
+all_preds = []
+num_batches = (len(all_prompts) + BATCH_SIZE - 1) // BATCH_SIZE
+
+with torch.no_grad():
+    torch.cuda.empty_cache()  # Clear cache before inference
+    
+    for batch_idx in tqdm(range(num_batches), desc="Predicting"):
+        start_idx = batch_idx * BATCH_SIZE
+        end_idx = min(start_idx + BATCH_SIZE, len(all_prompts))
+        batch_prompts = all_prompts[start_idx:end_idx]
+        
+        # Tokenize batch with padding
+        inputs = tokenizer(
+            batch_prompts,
+            return_tensors="pt",
+            truncation=True,
+            max_length=MAX_LENGTH,
+            padding=True,  # Pad to longest in batch
+        )
+        input_lengths = [len(ids) for ids in inputs["input_ids"]]
+        inputs = {k: v.to(model.device) for k, v in inputs.items()}
+        
+        # Generate batch (greedy decoding)
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=3,  # Just need 1 digit
+            do_sample=False,
+            pad_token_id=tokenizer.pad_token_id,
+            eos_token_id=tokenizer.eos_token_id,
+        )
+        
+        # Decode each sequence in the batch
+        for seq_idx, output_ids in enumerate(outputs):
+            # Get only the new tokens (after the input)
+            input_len = inputs["input_ids"].shape[1]  # All padded to same length
+            generated_tokens = output_ids[input_len:]
+            generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()
+            
+            # Extract class
+            pred_class = extract_class(generated_text)
+            all_preds.append(pred_class)
+        
+        # Clear cache periodically
+        if batch_idx % 50 == 0:
+            torch.cuda.empty_cache()
+
+# Save predictions
+test_df["Predicted_Class"] = all_preds
+output_file = "test_predictions_gemma3.csv"
+test_df.to_csv(output_file, index=False)
+print(f"\nPredictions saved to: {output_file}")
+
+# Show sample predictions
+print("\nSample predictions:")
+for i in range(min(10, len(test_df))):
+    text = str(test_df.iloc[i][text_col])
+    text_display = text[:60] + "..." if len(text) > 60 else text
+    pred = test_df.iloc[i]["Predicted_Class"]
+    print(f"  [{i+1}] Class {pred}: {text_display}")
+
+# Class distribution
+print("\nPrediction distribution:")
+pred_counts = test_df["Predicted_Class"].value_counts().sort_index()
+for class_label, count in pred_counts.items():
+    pct = count / len(test_df) * 100
+    print(f"  Class {class_label}: {count:>5} samples ({pct:>5.1f}%)")
+
+print("\n" + "="*70)
+print("INFERENCE COMPLETE!")
+print("="*70)
+print(f"\nAdapter path: {ADAPTER_PATH}")
+print(f"Test samples: {len(test_df)}")
+print(f"Output file:  {output_file}")

Code/inference_marbertv2.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Inference script for trained MARBERTv2 telecom classification model.
+
+Loads the trained model and runs predictions on test.csv
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import torch
+
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    precision_recall_fscore_support,
+    classification_report,
+    confusion_matrix,
+)
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+MODEL_DIR = "./telecom_marbertv2_final"
+OUTPUT_FILE = "./test_predictions.csv"
+
+MAX_LENGTH = 256
+BATCH_SIZE = 64
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+
+# -------------------------------------------------------------------
+# 3. Load test data
+# -------------------------------------------------------------------
+print(f"Loading test data from: {TEST_FILE}")
+test_df = pd.read_csv(TEST_FILE)
+print(f"Test samples: {len(test_df)}")
+print(f"Columns: {test_df.columns.tolist()}")
+
+# Check if test data has labels
+has_labels = "Class" in test_df.columns
+if has_labels:
+    print("Test data contains labels - will compute metrics")
+else:
+    print("Test data has no labels - will only generate predictions")
+
+# -------------------------------------------------------------------
+# 4. Load model and tokenizer
+# -------------------------------------------------------------------
+print(f"\nLoading model from: {MODEL_DIR}")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
+
+# Load config to get label mappings
+config_path = os.path.join(MODEL_DIR, "config.json")
+if os.path.exists(config_path):
+    import json
+    with open(config_path, 'r') as f:
+        config_data = json.load(f)
+    
+    if 'id2label' in config_data:
+        id2label = {int(k): v for k, v in config_data['id2label'].items()}
+        # Create label2id with both string and int keys for robustness
+        label2id = {}
+        for k, v in id2label.items():
+            label2id[v] = k  # string key -> int value
+            try:
+                label2id[int(v)] = k  # int key -> int value (if label is numeric)
+            except (ValueError, TypeError):
+                pass
+        num_labels = len(id2label)
+    else:
+        # Fallback: infer from test data if available
+        if has_labels:
+            unique_classes = sorted(test_df["Class"].unique())
+            label2id = {label: idx for idx, label in enumerate(unique_classes)}
+            id2label = {idx: label for label, idx in label2id.items()}
+            num_labels = len(unique_classes)
+        else:
+            raise ValueError("Cannot determine number of labels without config or test labels")
+else:
+    # Fallback: infer from test data if available
+    if has_labels:
+        unique_classes = sorted(test_df["Class"].unique())
+        label2id = {label: idx for idx, label in enumerate(unique_classes)}
+        id2label = {idx: label for label, idx in label2id.items()}
+        num_labels = len(unique_classes)
+    else:
+        raise ValueError("Cannot find config.json and test data has no labels")
+
+print(f"Number of classes: {num_labels}")
+print(f"Label mapping: {id2label}")
+
+# Load model using AutoModelForSequenceClassification
+model = AutoModelForSequenceClassification.from_pretrained(
+    MODEL_DIR,
+    num_labels=num_labels,
+    id2label=id2label,
+    label2id=label2id,
+)
+model = model.to(device)
+model.eval()
+print("Model loaded successfully!")
+
+# -------------------------------------------------------------------
+# 5. Run inference
+# -------------------------------------------------------------------
+print("\nRunning inference...")
+
+all_predictions = []
+all_probabilities = []
+
+# Process in batches for efficiency
+for i in range(0, len(test_df), BATCH_SIZE):
+    batch_texts = test_df["Commentaire client"].iloc[i:i+BATCH_SIZE].tolist()
+    
+    # Tokenize
+    inputs = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH,
+        return_tensors="pt",
+    ).to(device)
+    
+    # Predict
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probs = torch.softmax(logits, dim=-1)
+        predictions = torch.argmax(logits, dim=-1)
+    
+    all_predictions.extend(predictions.cpu().numpy())
+    all_probabilities.extend(probs.cpu().numpy())
+    
+    if (i // BATCH_SIZE + 1) % 10 == 0:
+        print(f"Processed {i + len(batch_texts)}/{len(test_df)} samples...")
+
+print(f"Inference complete! Processed {len(all_predictions)} samples")
+
+# -------------------------------------------------------------------
+# 6. Save predictions
+# -------------------------------------------------------------------
+# Convert predictions to class names
+predicted_classes = [id2label[pred] for pred in all_predictions]
+
+# Add predictions to dataframe
+test_df["Predicted_Class"] = predicted_classes
+test_df["Predicted_Label_ID"] = all_predictions
+
+# Add probability for each class
+for idx, class_name in id2label.items():
+    test_df[f"Prob_{class_name}"] = [probs[idx] for probs in all_probabilities]
+
+# Add confidence (max probability)
+test_df["Confidence"] = [max(probs) for probs in all_probabilities]
+
+# Save results
+test_df.to_csv(OUTPUT_FILE, index=False)
+print(f"\nPredictions saved to: {OUTPUT_FILE}")
+
+# -------------------------------------------------------------------
+# 7. Compute metrics (if labels available)
+# -------------------------------------------------------------------
+if has_labels:
+    print("\n" + "="*80)
+    print("EVALUATION METRICS")
+    print("="*80)
+    
+    # Convert true labels to indices
+    true_labels = test_df["Class"].map(label2id).values
+    pred_labels = np.array(all_predictions)
+    
+    # Overall metrics
+    accuracy = accuracy_score(true_labels, pred_labels)
+    print(f"\nAccuracy: {accuracy:.4f}")
+    
+    # Weighted metrics (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='weighted', zero_division=0
+    )
+    print(f"\nWeighted Metrics:")
+    print(f"  Precision: {precision_w:.4f}")
+    print(f"  Recall:    {recall_w:.4f}")
+    print(f"  F1 Score:  {f1_w:.4f}")
+    
+    # Macro metrics (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='macro', zero_division=0
+    )
+    print(f"\nMacro Metrics:")
+    print(f"  Precision: {precision_m:.4f}")
+    print(f"  Recall:    {recall_m:.4f}")
+    print(f"  F1 Score:  {f1_m:.4f}")
+    
+    # Per-class metrics
+    print(f"\nPer-Class Metrics:")
+    per_class_f1 = f1_score(true_labels, pred_labels, average=None, zero_division=0)
+    per_class_precision, per_class_recall, _, support = precision_recall_fscore_support(
+        true_labels, pred_labels, average=None, zero_division=0
+    )
+    
+    for idx in range(num_labels):
+        class_name = id2label[idx]
+        print(f"\n  {class_name}:")
+        print(f"    Precision: {per_class_precision[idx]:.4f}")
+        print(f"    Recall:    {per_class_recall[idx]:.4f}")
+        print(f"    F1 Score:  {per_class_f1[idx]:.4f}")
+        print(f"    Support:   {int(support[idx])}")
+    
+    # Classification report
+    print("\n" + "="*80)
+    print("DETAILED CLASSIFICATION REPORT")
+    print("="*80)
+    target_names = [id2label[i] for i in range(num_labels)]
+    print(classification_report(true_labels, pred_labels, target_names=target_names, zero_division=0))
+    
+    # Confusion matrix
+    print("\n" + "="*80)
+    print("CONFUSION MATRIX")
+    print("="*80)
+    cm = confusion_matrix(true_labels, pred_labels)
+    
+    # Print confusion matrix with labels
+    print("\nTrue \\ Predicted", end="")
+    for i in range(num_labels):
+        print(f"\t{id2label[i][:8]}", end="")
+    print()
+    
+    for i in range(num_labels):
+        print(f"{id2label[i][:15]:<15}", end="")
+        for j in range(num_labels):
+            print(f"\t{cm[i][j]}", end="")
+        print()
+    
+    # Save confusion matrix to CSV
+    cm_df = pd.DataFrame(
+        cm,
+        index=[id2label[i] for i in range(num_labels)],
+        columns=[id2label[i] for i in range(num_labels)]
+    )
+    cm_df.to_csv("./confusion_matrix.csv")
+    print("\nConfusion matrix saved to: ./confusion_matrix.csv")
+
+# -------------------------------------------------------------------
+# 8. Show sample predictions
+# -------------------------------------------------------------------
+print("\n" + "="*80)
+print("SAMPLE PREDICTIONS")
+print("="*80)
+
+# Show first 5 predictions
+num_samples = min(5, len(test_df))
+for i in range(num_samples):
+    print(f"\nSample {i+1}:")
+    print(f"Text: {test_df['Commentaire client'].iloc[i]}")
+    if has_labels:
+        print(f"True Class: {test_df['Class'].iloc[i]}")
+    print(f"Predicted Class: {predicted_classes[i]}")
+    print(f"Confidence: {test_df['Confidence'].iloc[i]:.4f}")
+    print(f"Probabilities:")
+    for idx, class_name in id2label.items():
+        print(f"  {class_name}: {all_probabilities[i][idx]:.4f}")
+
+print("\n" + "="*80)
+print("Inference completed successfully!")
+print("="*80)

Code/inference_marbertv2_cpt_ft.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Inference script for CPT+finetuned MARBERTv2 telecom classification model.
+
+Loads the model from ./telecom_marbertv2_cpt_ft and runs predictions on test.csv
+"""
+
+import os
+import numpy as np
+import pandas as pd
+import torch
+
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    AutoConfig,
+)
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    precision_recall_fscore_support,
+    classification_report,
+    confusion_matrix,
+)
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+TEST_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/test_file.csv"
+MODEL_DIR = "./telecom_marbertv2_cpt_ft"
+OUTPUT_FILE = "./test_predictions_telecom_marbertv2_cpt_ft.csv"
+
+MAX_LENGTH = 256
+BATCH_SIZE = 64
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+# -------------------------------------------------------------------
+# 2. Load test data
+# -------------------------------------------------------------------
+print(f"Loading test data from: {TEST_FILE}")
+test_df = pd.read_csv(TEST_FILE)
+print(f"Test samples: {len(test_df)}")
+print(f"Columns: {test_df.columns.tolist()}")
+
+# Check if test data has labels
+has_labels = "Class" in test_df.columns
+if has_labels:
+    print("Test data contains labels - will compute metrics")
+else:
+    print("Test data has no labels - will only generate predictions")
+
+# -------------------------------------------------------------------
+# 3. Load model and tokenizer
+# -------------------------------------------------------------------
+print(f"\nLoading model from: {MODEL_DIR}")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
+
+# Load config to get label mappings
+config_path = os.path.join(MODEL_DIR, "config.json")
+if os.path.exists(config_path):
+    import json
+    with open(config_path, 'r') as f:
+        config_data = json.load(f)
+    
+    if 'id2label' in config_data:
+        id2label = {int(k): int(v) for k, v in config_data['id2label'].items()}
+        # Create label2id with both string and int keys for robustness
+        label2id = {}
+        for k, v in id2label.items():
+            label2id[v] = k  # int key -> int value
+            label2id[str(v)] = k  # string key -> int value
+        num_labels = len(id2label)
+    else:
+        # Fallback: infer from test data if available
+        if has_labels:
+            unique_classes = sorted(test_df["Class"].unique())
+            label2id = {label: idx for idx, label in enumerate(unique_classes)}
+            id2label = {idx: label for label, idx in label2id.items()}
+            num_labels = len(unique_classes)
+        else:
+            raise ValueError("Cannot determine number of labels without config or test labels")
+else:
+    # Fallback: infer from test data if available
+    if has_labels:
+        unique_classes = sorted(test_df["Class"].unique())
+        label2id = {label: idx for idx, label in enumerate(unique_classes)}
+        id2label = {idx: label for label, idx in label2id.items()}
+        num_labels = len(unique_classes)
+    else:
+        raise ValueError("Cannot find config.json and test data has no labels")
+
+print(f"Number of classes: {num_labels}")
+print(f"Label mapping: {id2label}")
+
+# Load model
+model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR)
+model = model.to(device)
+model.eval()
+print("Model loaded successfully!")
+
+# -------------------------------------------------------------------
+# 4. Run inference
+# -------------------------------------------------------------------
+print("\nRunning inference...")
+
+all_predictions = []
+all_probabilities = []
+
+# Process in batches for efficiency
+for i in range(0, len(test_df), BATCH_SIZE):
+    batch_texts = test_df["Commentaire client"].iloc[i:i+BATCH_SIZE].tolist()
+    
+    # Tokenize
+    inputs = tokenizer(
+        batch_texts,
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH,
+        return_tensors="pt",
+    ).to(device)
+    
+    # Predict
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probs = torch.softmax(logits, dim=-1)
+        predictions = torch.argmax(logits, dim=-1)
+    
+    all_predictions.extend(predictions.cpu().numpy())
+    all_probabilities.extend(probs.cpu().numpy())
+    
+    if (i // BATCH_SIZE + 1) % 10 == 0:
+        print(f"Processed {i + len(batch_texts)}/{len(test_df)} samples...")
+
+print(f"Inference complete! Processed {len(all_predictions)} samples")
+
+# -------------------------------------------------------------------
+# 5. Save predictions
+# -------------------------------------------------------------------
+# Convert predictions to class names (1-9)
+predicted_classes = [id2label[pred] for pred in all_predictions]
+
+# Add predictions to dataframe
+test_df["Predicted_Class"] = predicted_classes
+test_df["Predicted_Label_ID"] = all_predictions
+
+# Add probability for each class
+for idx, class_name in id2label.items():
+    test_df[f"Prob_{class_name}"] = [probs[idx] for probs in all_probabilities]
+
+# Add confidence (max probability)
+test_df["Confidence"] = [max(probs) for probs in all_probabilities]
+
+# Save results
+test_df.to_csv(OUTPUT_FILE, index=False)
+print(f"\nPredictions saved to: {OUTPUT_FILE}")
+
+# -------------------------------------------------------------------
+# 6. Compute metrics (if labels available)
+# -------------------------------------------------------------------
+if has_labels:
+    print("\n" + "="*80)
+    print("EVALUATION METRICS")
+    print("="*80)
+    
+    # Convert true labels to indices
+    true_labels = test_df["Class"].map(label2id).values
+    pred_labels = np.array(all_predictions)
+    
+    # Overall metrics
+    accuracy = accuracy_score(true_labels, pred_labels)
+    print(f"\nAccuracy: {accuracy:.4f}")
+    
+    # Weighted metrics (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='weighted', zero_division=0
+    )
+    print(f"\nWeighted Metrics:")
+    print(f"  Precision: {precision_w:.4f}")
+    print(f"  Recall:    {recall_w:.4f}")
+    print(f"  F1 Score:  {f1_w:.4f}")
+    
+    # Macro metrics (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        true_labels, pred_labels, average='macro', zero_division=0
+    )
+    print(f"\nMacro Metrics:")
+    print(f"  Precision: {precision_m:.4f}")
+    print(f"  Recall:    {recall_m:.4f}")
+    print(f"  F1 Score:  {f1_m:.4f}")
+    
+    # Per-class metrics
+    print(f"\nPer-Class Metrics:")
+    per_class_f1 = f1_score(true_labels, pred_labels, average=None, zero_division=0)
+    per_class_precision, per_class_recall, _, support = precision_recall_fscore_support(
+        true_labels, pred_labels, average=None, zero_division=0
+    )
+    
+    for idx in range(num_labels):
+        class_name = id2label[idx]
+        print(f"\n  Class {class_name}:")
+        print(f"    Precision: {per_class_precision[idx]:.4f}")
+        print(f"    Recall:    {per_class_recall[idx]:.4f}")
+        print(f"    F1 Score:  {per_class_f1[idx]:.4f}")
+        print(f"    Support:   {int(support[idx])}")
+    
+    # Classification report
+    print("\n" + "="*80)
+    print("DETAILED CLASSIFICATION REPORT")
+    print("="*80)
+    target_names = [str(id2label[i]) for i in range(num_labels)]
+    print(classification_report(true_labels, pred_labels, target_names=target_names, zero_division=0))
+    
+    # Confusion matrix
+    print("\n" + "="*80)
+    print("CONFUSION MATRIX")
+    print("="*80)
+    cm = confusion_matrix(true_labels, pred_labels)
+    
+    # Print confusion matrix with labels
+    print("\nTrue \\ Predicted", end="")
+    for i in range(num_labels):
+        print(f"\t{id2label[i]}", end="")
+    print()
+    
+    for i in range(num_labels):
+        print(f"{id2label[i]:<15}", end="")
+        for j in range(num_labels):
+            print(f"\t{cm[i][j]}", end="")
+        print()
+    
+    # Save confusion matrix to CSV
+    cm_df = pd.DataFrame(
+        cm,
+        index=[str(id2label[i]) for i in range(num_labels)],
+        columns=[str(id2label[i]) for i in range(num_labels)]
+    )
+    cm_df.to_csv("./confusion_matrix_marbertv2_cpt_ft.csv")
+    print("\nConfusion matrix saved to: ./confusion_matrix_marbertv2_cpt_ft.csv")
+
+# -------------------------------------------------------------------
+# 7. Show sample predictions
+# -------------------------------------------------------------------
+print("\n" + "="*80)
+print("SAMPLE PREDICTIONS (CPT+Finetuned MARBERTv2)")
+print("="*80)
+
+# Show first 5 predictions
+num_samples = min(5, len(test_df))
+for i in range(num_samples):
+    print(f"\nSample {i+1}:")
+    print(f"Text: {test_df['Commentaire client'].iloc[i]}")
+    if has_labels:
+        print(f"True Class: {test_df['Class'].iloc[i]}")
+    print(f"Predicted Class: {predicted_classes[i]}")
+    print(f"Confidence: {test_df['Confidence'].iloc[i]:.4f}")
+    print(f"Probabilities:")
+    for idx, class_name in id2label.items():
+        print(f"  Class {class_name}: {all_probabilities[i][idx]:.4f}")
+
+print("\n" + "="*80)
+print("Inference completed successfully!")
+print(f"Model used: CPT+Finetuned MARBERTv2 from {MODEL_DIR}")
+print("="*80)

Code/train_arabert.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Fine-tune aubmindlab/bert-large-arabertv2 for Arabic telecom customer comment classification.
+
+Dataset (CSV):
+  /home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/train.csv
+
+Columns:
+  Commentaire client: str (text)
+  Class: int (label - values 1 through 9)
+
+Model:
+  - AraBERTv2 Large encoder
+  - Classification head for multi-class prediction (9 classes)
+"""
+
+import os
+import numpy as np
+import torch
+
+from inspect import signature
+from datasets import load_dataset
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    TrainingArguments,
+    Trainer,
+)
+
+# Slight speed boost on Ampere GPUs
+if hasattr(torch, "set_float32_matmul_precision"):
+    torch.set_float32_matmul_precision("high")
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+DATA_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/train.csv"
+
+MODEL_NAME = "aubmindlab/bert-large-arabertv2"
+OUTPUT_DIR = "./telecom_arabert_final"
+
+MAX_LENGTH = 512
+
+# Define label mapping - classes are 1-9
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9}
+NUM_LABELS = 9
+
+# -------------------------------------------------------------------
+# 2. Dataset loading
+# -------------------------------------------------------------------
+print("Loading telecom dataset from CSV...")
+dataset = load_dataset(
+    "csv",
+    data_files=DATA_FILE,
+    split="train",
+)
+
+print("Sample example:", dataset[0])
+print(f"Total examples: {len(dataset)}")
+
+print(f"Number of classes: {NUM_LABELS}")
+print("Label mapping (class -> model index):", LABEL2ID)
+print("Inverse mapping (model index -> class):", ID2LABEL)
+
+
+def encode_labels(example):
+    """Convert class (1-9) to model label index (0-8)."""
+    class_val = example["Class"]
+    
+    # Handle both int and string types
+    if isinstance(class_val, str):
+        class_val = int(class_val)
+    
+    if class_val not in LABEL2ID:
+        raise ValueError(f"Unknown class: {class_val}. Expected 1-9.")
+    
+    example["labels"] = LABEL2ID[class_val]
+    return example
+
+
+dataset = dataset.map(encode_labels)
+
+# Train/val split (90/10)
+dataset = dataset.train_test_split(test_size=0.1, seed=42)
+train_dataset = dataset["train"]
+eval_dataset = dataset["test"]
+
+print("Train size:", len(train_dataset))
+print("Eval size:", len(eval_dataset))
+
+# -------------------------------------------------------------------
+# 3. Tokenization
+# -------------------------------------------------------------------
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+
+def preprocess_function(examples):
+    return tokenizer(
+        examples["Commentaire client"],
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH,
+    )
+
+
+train_dataset = train_dataset.map(preprocess_function, batched=True, num_proc=4)
+eval_dataset = eval_dataset.map(preprocess_function, batched=True, num_proc=4)
+
+train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+eval_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+# -------------------------------------------------------------------
+# 4. Model - Using AutoModelForSequenceClassification
+# -------------------------------------------------------------------
+model = AutoModelForSequenceClassification.from_pretrained(
+    MODEL_NAME,
+    num_labels=NUM_LABELS,
+    id2label=ID2LABEL,
+    label2id=LABEL2ID,
+)
+
+print("Model initialized with classification head")
+print(f"Number of labels: {NUM_LABELS}")
+print(f"Classes: {list(ID2LABEL.values())}")
+
+# -------------------------------------------------------------------
+# 5. Metrics
+# -------------------------------------------------------------------
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    
+    # Overall metrics
+    accuracy = accuracy_score(labels, predictions)
+    
+    # Weighted average (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        labels, predictions, average='weighted', zero_division=0
+    )
+    
+    # Macro average (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        labels, predictions, average='macro', zero_division=0
+    )
+    
+    metrics = {
+        'accuracy': accuracy,
+        'f1_weighted': f1_w,
+        'f1_macro': f1_m,
+        'precision_weighted': precision_w,
+        'recall_weighted': recall_w,
+        'precision_macro': precision_m,
+        'recall_macro': recall_m,
+    }
+    
+    # Per-class F1 scores
+    per_class_f1 = f1_score(labels, predictions, average=None, zero_division=0)
+    for idx in range(NUM_LABELS):
+        class_name = ID2LABEL[idx]
+        if idx < len(per_class_f1):
+            metrics[f'f1_class_{class_name}'] = per_class_f1[idx]
+    
+    return metrics
+
+
+# -------------------------------------------------------------------
+# 6. TrainingArguments (old/new transformers compatible)
+# -------------------------------------------------------------------
+ta_sig = signature(TrainingArguments.__init__)
+ta_params = set(ta_sig.parameters.keys())
+
+is_bf16_supported = (
+    torch.cuda.is_available()
+    and hasattr(torch.cuda, "is_bf16_supported")
+    and torch.cuda.is_bf16_supported()
+)
+use_bf16 = bool(is_bf16_supported)
+use_fp16 = not use_bf16
+
+print(f"bf16 supported: {is_bf16_supported} -> using bf16={use_bf16}, fp16={use_fp16}")
+
+base_kwargs = {
+    "output_dir": OUTPUT_DIR,
+    "num_train_epochs": 10,
+    "per_device_train_batch_size": 32,
+    "per_device_eval_batch_size": 64,
+    "learning_rate": 1e-3,
+    "weight_decay": 0.03,
+    "warmup_ratio": 0.1,
+    "logging_steps": 50,
+    "save_total_limit": 2,
+    "dataloader_num_workers": 4,
+}
+
+# Mixed precision flags if supported
+if "bf16" in ta_params:
+    base_kwargs["bf16"] = use_bf16
+if "fp16" in ta_params:
+    base_kwargs["fp16"] = use_fp16
+
+# Handle evaluation_strategy compatibility
+if "evaluation_strategy" in ta_params:
+    base_kwargs["evaluation_strategy"] = "epoch"
+    if "save_strategy" in ta_params:
+        base_kwargs["save_strategy"] = "epoch"
+    if "logging_strategy" in ta_params:
+        base_kwargs["logging_strategy"] = "steps"
+    if "load_best_model_at_end" in ta_params:
+        base_kwargs["load_best_model_at_end"] = True
+    if "metric_for_best_model" in ta_params:
+        base_kwargs["metric_for_best_model"] = "f1_weighted"
+    if "greater_is_better" in ta_params:
+        base_kwargs["greater_is_better"] = True
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+else:
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+    print("[TrainingArguments] Old transformers version: no evaluation_strategy argument. Using simple setup.")
+
+filtered_kwargs = {}
+for k, v in base_kwargs.items():
+    if k in ta_params:
+        filtered_kwargs[k] = v
+    else:
+        print(f"[TrainingArguments] Skipping unsupported arg: {k}={v}")
+
+training_args = TrainingArguments(**filtered_kwargs)
+
+# -------------------------------------------------------------------
+# 7. Trainer
+# -------------------------------------------------------------------
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    tokenizer=tokenizer,
+    compute_metrics=compute_metrics,
+)
+
+# -------------------------------------------------------------------
+# 8. Train & eval
+# -------------------------------------------------------------------
+if __name__ == "__main__":
+    print("Starting telecom classification training with AraBERT...")
+    trainer.train()
+
+    print("Evaluating on validation split...")
+    metrics = trainer.evaluate()
+    print("Validation metrics:", metrics)
+
+    print("Saving final model & tokenizer...")
+    trainer.save_model(OUTPUT_DIR)
+    tokenizer.save_pretrained(OUTPUT_DIR)
+    
+    print(f"Label mappings saved in config:")
+    print(f"  ID to Label: {ID2LABEL}")
+    print(f"  Label to ID: {LABEL2ID}")
+
+    # Quick sanity-check inference
+    example_texts = [
+        "الخدمة ممتازة جدا وسريعة",
+        "سيء للغاية ولا يستجيبون",
+        "متوسط الجودة"
+    ]
+    
+    inputs = tokenizer(
+        example_texts,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH
+    ).to(model.device)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+    
+    logits = outputs.logits.cpu().numpy()
+    predictions = np.argmax(logits, axis=-1)
+
+    print("\nSanity-check predictions:")
+    for text, pred_idx in zip(example_texts, predictions):
+        pred_class = ID2LABEL[pred_idx]
+        print(f"Text: {text}")
+        print(f"  -> Predicted Class: {pred_class}")
+        print()
+
+    print("Training complete and model saved to:", OUTPUT_DIR)

Code/train_dziribert.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Fine-tune alger-ia/dziribert for Arabic telecom customer comment classification.
+
+Dataset (CSV):
+  /home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/train.csv
+
+Columns:
+  Commentaire client: str (text)
+  Class: int (label - values 1 through 9)
+
+Model:
+  - DziriBERT encoder (alger-ia/dziribert)
+  - Classification head for multi-class prediction (9 classes)
+"""
+
+import os
+import numpy as np
+import torch
+
+from inspect import signature
+from datasets import load_dataset
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    TrainingArguments,
+    Trainer,
+)
+
+# Slight speed boost on Ampere GPUs
+if hasattr(torch, "set_float32_matmul_precision"):
+    torch.set_float32_matmul_precision("high")
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+DATA_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--labelds/snapshots/48f016fd5987875b0e9f79d0689cef2ec3b2ce0b/train.csv"
+
+MODEL_NAME = "alger-ia/dziribert"
+OUTPUT_DIR = "./telecom_dziribert_final"
+
+MAX_LENGTH = 512
+
+# Define label mapping - classes are 1-9
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9}
+NUM_LABELS = 9
+
+# -------------------------------------------------------------------
+# 2. Dataset loading
+# -------------------------------------------------------------------
+print("Loading telecom dataset from CSV...")
+dataset = load_dataset(
+    "csv",
+    data_files=DATA_FILE,
+    split="train",
+)
+
+print("Sample example:", dataset[0])
+print(f"Total examples: {len(dataset)}")
+
+print(f"Number of classes: {NUM_LABELS}")
+print("Label mapping (class -> model index):", LABEL2ID)
+print("Inverse mapping (model index -> class):", ID2LABEL)
+
+
+def encode_labels(example):
+    """Convert class (1-9) to model label index (0-8)."""
+    class_val = example["Class"]
+    
+    # Handle both int and string types
+    if isinstance(class_val, str):
+        class_val = int(class_val)
+    
+    if class_val not in LABEL2ID:
+        raise ValueError(f"Unknown class: {class_val}. Expected 1-9.")
+    
+    example["labels"] = LABEL2ID[class_val]
+    return example
+
+
+dataset = dataset.map(encode_labels)
+
+# Train/val split (90/10)
+dataset = dataset.train_test_split(test_size=0.1, seed=42)
+train_dataset = dataset["train"]
+eval_dataset = dataset["test"]
+
+print("Train size:", len(train_dataset))
+print("Eval size:", len(eval_dataset))
+
+# -------------------------------------------------------------------
+# 3. Tokenization
+# -------------------------------------------------------------------
+print(f"\nLoading tokenizer for: {MODEL_NAME}")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+
+def preprocess_function(examples):
+    return tokenizer(
+        examples["Commentaire client"],
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH,
+    )
+
+
+train_dataset = train_dataset.map(preprocess_function, batched=True, num_proc=4)
+eval_dataset = eval_dataset.map(preprocess_function, batched=True, num_proc=4)
+
+train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+eval_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+# -------------------------------------------------------------------
+# 4. Model - Using AutoModelForSequenceClassification
+# -------------------------------------------------------------------
+print(f"\nLoading model: {MODEL_NAME}")
+model = AutoModelForSequenceClassification.from_pretrained(
+    MODEL_NAME,
+    num_labels=NUM_LABELS,
+    id2label=ID2LABEL,
+    label2id=LABEL2ID,
+)
+
+print("Model initialized with classification head")
+print(f"Number of labels: {NUM_LABELS}")
+print(f"Classes: {list(ID2LABEL.values())}")
+
+# -------------------------------------------------------------------
+# 5. Metrics
+# -------------------------------------------------------------------
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    
+    # Overall metrics
+    accuracy = accuracy_score(labels, predictions)
+    
+    # Weighted average (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        labels, predictions, average='weighted', zero_division=0
+    )
+    
+    # Macro average (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        labels, predictions, average='macro', zero_division=0
+    )
+    
+    metrics = {
+        'accuracy': accuracy,
+        'f1_weighted': f1_w,
+        'f1_macro': f1_m,
+        'precision_weighted': precision_w,
+        'recall_weighted': recall_w,
+        'precision_macro': precision_m,
+        'recall_macro': recall_m,
+    }
+    
+    # Per-class F1 scores
+    per_class_f1 = f1_score(labels, predictions, average=None, zero_division=0)
+    for idx in range(NUM_LABELS):
+        class_name = ID2LABEL[idx]
+        if idx < len(per_class_f1):
+            metrics[f'f1_class_{class_name}'] = per_class_f1[idx]
+    
+    return metrics
+
+
+# -------------------------------------------------------------------
+# 6. TrainingArguments (old/new transformers compatible)
+# -------------------------------------------------------------------
+ta_sig = signature(TrainingArguments.__init__)
+ta_params = set(ta_sig.parameters.keys())
+
+is_bf16_supported = (
+    torch.cuda.is_available()
+    and hasattr(torch.cuda, "is_bf16_supported")
+    and torch.cuda.is_bf16_supported()
+)
+use_bf16 = bool(is_bf16_supported)
+use_fp16 = not use_bf16
+
+print(f"bf16 supported: {is_bf16_supported} -> using bf16={use_bf16}, fp16={use_fp16}")
+
+base_kwargs = {
+    "output_dir": OUTPUT_DIR,
+    "num_train_epochs": 50,
+    "per_device_train_batch_size": 32,
+    "per_device_eval_batch_size": 64,
+    "learning_rate": 1e-5,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.1,
+    "logging_steps": 50,
+    "save_total_limit": 2,
+    "dataloader_num_workers": 4,
+}
+
+# Mixed precision flags if supported
+if "bf16" in ta_params:
+    base_kwargs["bf16"] = use_bf16
+if "fp16" in ta_params:
+    base_kwargs["fp16"] = use_fp16
+
+# Handle evaluation_strategy compatibility
+if "evaluation_strategy" in ta_params:
+    base_kwargs["evaluation_strategy"] = "epoch"
+    if "save_strategy" in ta_params:
+        base_kwargs["save_strategy"] = "epoch"
+    if "logging_strategy" in ta_params:
+        base_kwargs["logging_strategy"] = "steps"
+    if "load_best_model_at_end" in ta_params:
+        base_kwargs["load_best_model_at_end"] = True
+    if "metric_for_best_model" in ta_params:
+        base_kwargs["metric_for_best_model"] = "f1_weighted"
+    if "greater_is_better" in ta_params:
+        base_kwargs["greater_is_better"] = True
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+else:
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+    print("[TrainingArguments] Old transformers version: no evaluation_strategy argument. Using simple setup.")
+
+filtered_kwargs = {}
+for k, v in base_kwargs.items():
+    if k in ta_params:
+        filtered_kwargs[k] = v
+    else:
+        print(f"[TrainingArguments] Skipping unsupported arg: {k}={v}")
+
+training_args = TrainingArguments(**filtered_kwargs)
+
+# -------------------------------------------------------------------
+# 7. Trainer
+# -------------------------------------------------------------------
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    tokenizer=tokenizer,
+    compute_metrics=compute_metrics,
+)
+
+# -------------------------------------------------------------------
+# 8. Train & eval
+# -------------------------------------------------------------------
+if __name__ == "__main__":
+    print("\nStarting DziriBERT telecom classification training...")
+    trainer.train()
+
+    print("\nEvaluating on validation split...")
+    metrics = trainer.evaluate()
+    print("Validation metrics:", metrics)
+
+    print("\nSaving final model & tokenizer...")
+    trainer.save_model(OUTPUT_DIR)
+    tokenizer.save_pretrained(OUTPUT_DIR)
+    
+    # Save label mappings to config
+    import json
+    config_path = os.path.join(OUTPUT_DIR, "config.json")
+    if os.path.exists(config_path):
+        with open(config_path, 'r') as f:
+            config_data = json.load(f)
+        
+        # Update with actual label mappings
+        config_data['id2label'] = {str(idx): str(label) for idx, label in ID2LABEL.items()}
+        config_data['label2id'] = {str(label): idx for label, idx in LABEL2ID.items()}
+        config_data['num_labels'] = NUM_LABELS
+        config_data['problem_type'] = 'single_label_classification'
+        
+        with open(config_path, 'w') as f:
+            json.dump(config_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"Saved label mappings to config: {ID2LABEL}")
+    
+    print(f"\nLabel mappings:")
+    print(f"  ID to Label: {ID2LABEL}")
+    print(f"  Label to ID: {LABEL2ID}")
+
+    # Quick sanity-check inference
+    example_texts = [
+        "الخدمة ممتازة جدا وسريعة",
+        "سيء للغاية ولا يستجيبون",
+        "متوسط الجودة"
+    ]
+    
+    inputs = tokenizer(
+        example_texts,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH
+    ).to(model.device)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+    
+    logits = outputs.logits.cpu().numpy()
+    predictions = np.argmax(logits, axis=-1)
+
+    print("\nSanity-check predictions:")
+    for text, pred_idx in zip(example_texts, predictions):
+        pred_class = ID2LABEL[pred_idx]
+        print(f"Text: {text}")
+        print(f"  -> Predicted Class: {pred_class}")
+        print()
+
+    print("Training complete and model saved to:", OUTPUT_DIR)

Code/train_marbertv2.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Fine-tune UBC-NLP/MARBERTv2 for Arabic telecom customer comment classification.
+
+Dataset (CSV):
+  /home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--telecom-ch1/snapshots/be06acac69aa411636dbe0e3bef5f0072e670765/train.csv
+
+Columns:
+  Commentaire client: str (text)
+  Class: int (label - values 1 through 9)
+
+Model:
+  - MARBERTv2 encoder
+  - Classification head for multi-class prediction (9 classes)
+"""
+
+import os
+import numpy as np
+import torch
+
+from inspect import signature
+from datasets import load_dataset
+from sklearn.metrics import accuracy_score, f1_score, precision_recall_fscore_support
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    TrainingArguments,
+    Trainer,
+)
+
+# Slight speed boost on Ampere GPUs
+if hasattr(torch, "set_float32_matmul_precision"):
+    torch.set_float32_matmul_precision("high")
+
+# -------------------------------------------------------------------
+# 1. Paths & config
+# -------------------------------------------------------------------
+DATA_FILE = "/home/houssam-nojoom/.cache/huggingface/hub/datasets--houssamboukhalfa--labelds/snapshots/48f016fd5987875b0e9f79d0689cef2ec3b2ce0b/train.csv"
+
+MODEL_NAME = "UBC-NLP/MARBERTv2"
+OUTPUT_DIR = "./telecom_marbertv2_final"
+
+MAX_LENGTH = 256
+
+# Define label mapping - classes are 1-9
+LABEL2ID = {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8}
+ID2LABEL = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9}
+NUM_LABELS = 9
+
+# -------------------------------------------------------------------
+# 2. Dataset loading
+# -------------------------------------------------------------------
+print("Loading telecom dataset from CSV...")
+dataset = load_dataset(
+    "csv",
+    data_files=DATA_FILE,
+    split="train",
+)
+
+print("Sample example:", dataset[0])
+print(f"Total examples: {len(dataset)}")
+
+print(f"Number of classes: {NUM_LABELS}")
+print("Label mapping (class -> model index):", LABEL2ID)
+print("Inverse mapping (model index -> class):", ID2LABEL)
+
+
+def encode_labels(example):
+    """Convert class (1-9) to model label index (0-8)."""
+    class_val = example["Class"]
+    
+    # Handle both int and string types
+    if isinstance(class_val, str):
+        class_val = int(class_val)
+    
+    if class_val not in LABEL2ID:
+        raise ValueError(f"Unknown class: {class_val}. Expected 1-9.")
+    
+    example["labels"] = LABEL2ID[class_val]
+    return example
+
+
+dataset = dataset.map(encode_labels)
+
+# Train/val split (90/10)
+dataset = dataset.train_test_split(test_size=0.1, seed=42)
+train_dataset = dataset["train"]
+eval_dataset = dataset["test"]
+
+print("Train size:", len(train_dataset))
+print("Eval size:", len(eval_dataset))
+
+# -------------------------------------------------------------------
+# 3. Tokenization
+# -------------------------------------------------------------------
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+
+def preprocess_function(examples):
+    return tokenizer(
+        examples["Commentaire client"],
+        padding="max_length",
+        truncation=True,
+        max_length=MAX_LENGTH,
+    )
+
+
+train_dataset = train_dataset.map(preprocess_function, batched=True, num_proc=4)
+eval_dataset = eval_dataset.map(preprocess_function, batched=True, num_proc=4)
+
+train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+eval_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+# -------------------------------------------------------------------
+# 4. Model - Using AutoModelForSequenceClassification
+# -------------------------------------------------------------------
+model = AutoModelForSequenceClassification.from_pretrained(
+    MODEL_NAME,
+    num_labels=NUM_LABELS,
+    id2label=ID2LABEL,
+    label2id=LABEL2ID,
+)
+
+print("Model initialized with classification head")
+print(f"Number of labels: {NUM_LABELS}")
+print(f"Classes: {list(ID2LABEL.values())}")
+
+# -------------------------------------------------------------------
+# 5. Metrics
+# -------------------------------------------------------------------
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    
+    # Overall metrics
+    accuracy = accuracy_score(labels, predictions)
+    
+    # Weighted average (accounts for class imbalance)
+    precision_w, recall_w, f1_w, _ = precision_recall_fscore_support(
+        labels, predictions, average='weighted', zero_division=0
+    )
+    
+    # Macro average (treats all classes equally)
+    precision_m, recall_m, f1_m, _ = precision_recall_fscore_support(
+        labels, predictions, average='macro', zero_division=0
+    )
+    
+    metrics = {
+        'accuracy': accuracy,
+        'f1_weighted': f1_w,
+        'f1_macro': f1_m,
+        'precision_weighted': precision_w,
+        'recall_weighted': recall_w,
+        'precision_macro': precision_m,
+        'recall_macro': recall_m,
+    }
+    
+    # Per-class F1 scores
+    per_class_f1 = f1_score(labels, predictions, average=None, zero_division=0)
+    for idx in range(NUM_LABELS):
+        class_name = ID2LABEL[idx]
+        if idx < len(per_class_f1):
+            metrics[f'f1_class_{class_name}'] = per_class_f1[idx]
+    
+    return metrics
+
+
+# -------------------------------------------------------------------
+# 6. TrainingArguments (old/new transformers compatible)
+# -------------------------------------------------------------------
+ta_sig = signature(TrainingArguments.__init__)
+ta_params = set(ta_sig.parameters.keys())
+
+is_bf16_supported = (
+    torch.cuda.is_available()
+    and hasattr(torch.cuda, "is_bf16_supported")
+    and torch.cuda.is_bf16_supported()
+)
+use_bf16 = bool(is_bf16_supported)
+use_fp16 = not use_bf16
+
+print(f"bf16 supported: {is_bf16_supported} -> using bf16={use_bf16}, fp16={use_fp16}")
+
+base_kwargs = {
+    "output_dir": OUTPUT_DIR,
+    "num_train_epochs": 10,
+    "per_device_train_batch_size": 32,
+    "per_device_eval_batch_size": 64,
+    "learning_rate": 1e-4,
+    "weight_decay": 0.02,
+    "warmup_ratio": 0.1,
+    "logging_steps": 50,
+    "save_total_limit": 2,
+    "dataloader_num_workers": 4,
+}
+
+# Mixed precision flags if supported
+if "bf16" in ta_params:
+    base_kwargs["bf16"] = use_bf16
+if "fp16" in ta_params:
+    base_kwargs["fp16"] = use_fp16
+
+# Handle evaluation_strategy compatibility
+if "evaluation_strategy" in ta_params:
+    base_kwargs["evaluation_strategy"] = "epoch"
+    if "save_strategy" in ta_params:
+        base_kwargs["save_strategy"] = "epoch"
+    if "logging_strategy" in ta_params:
+        base_kwargs["logging_strategy"] = "steps"
+    if "load_best_model_at_end" in ta_params:
+        base_kwargs["load_best_model_at_end"] = True
+    if "metric_for_best_model" in ta_params:
+        base_kwargs["metric_for_best_model"] = "f1_weighted"
+    if "greater_is_better" in ta_params:
+        base_kwargs["greater_is_better"] = True
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+else:
+    if "report_to" in ta_params:
+        base_kwargs["report_to"] = "none"
+    print("[TrainingArguments] Old transformers version: no evaluation_strategy argument. Using simple setup.")
+
+filtered_kwargs = {}
+for k, v in base_kwargs.items():
+    if k in ta_params:
+        filtered_kwargs[k] = v
+    else:
+        print(f"[TrainingArguments] Skipping unsupported arg: {k}={v}")
+
+training_args = TrainingArguments(**filtered_kwargs)
+
+# -------------------------------------------------------------------
+# 7. Trainer
+# -------------------------------------------------------------------
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=eval_dataset,
+    tokenizer=tokenizer,
+    compute_metrics=compute_metrics,
+)
+
+# -------------------------------------------------------------------
+# 8. Train & eval
+# -------------------------------------------------------------------
+if __name__ == "__main__":
+    print("Starting telecom classification training...")
+    trainer.train()
+
+    print("Evaluating on validation split...")
+    metrics = trainer.evaluate()
+    print("Validation metrics:", metrics)
+
+    print("Saving final model & tokenizer...")
+    trainer.save_model(OUTPUT_DIR)
+    tokenizer.save_pretrained(OUTPUT_DIR)
+    
+    print(f"Label mappings saved in config:")
+    print(f"  ID to Label: {ID2LABEL}")
+    print(f"  Label to ID: {LABEL2ID}")
+
+    # Quick sanity-check inference
+    example_texts = [
+        "الخدمة ممتازة جدا وسريعة",
+        "سيء للغاية ولا يستجيبون",
+        "متوسط الجودة"
+    ]
+    
+    inputs = tokenizer(
+        example_texts,
+        return_tensors="pt",
+        padding=True,
+        truncation=True,
+        max_length=MAX_LENGTH
+    ).to(model.device)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+    
+    logits = outputs.logits.cpu().numpy()
+    predictions = np.argmax(logits, axis=-1)
+
+    print("\nSanity-check predictions:")
+    for text, pred_idx in zip(example_texts, predictions):
+        pred_class = ID2LABEL[pred_idx]
+        print(f"Text: {text}")
+        print(f"  -> Predicted Class: {pred_class}")
+        print()
+
+    print("Training complete and model saved to:", OUTPUT_DIR)

Code/voting.ipynb

@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f4d51cb4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "344fbcef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_gemma1 = pd.read_csv('test_predictions_gemma3.csv')\n",
+    "df_camel = pd.read_csv('test_predictions_camelbert_cpt_ftbeforesleep.csv')\n",
+    "df_arabert = pd.read_csv('test_predictions_arabert_full_pipeline.csv')\n",
+    "df_dziribert = pd.read_csv('test_predictions_dziribert.csv')\n",
+    "df_marbert= pd.read_csv('test_predictions_marbertv2_cpt_ft5.csv')\n",
+    "df_gemma2 = pd.read_csv('test_predictions_gemma3-2nd.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29486bd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Perform soft voting with weights\n",
+    "# Give more weight to df_gemma1 (weight = 2), others get weight = 1\n",
+    "\n",
+    "# Define weights for each model\n",
+    "weights = {\n",
+    "    'df_gemma1': 2.0,\n",
+    "    'df_camel': 1.0,\n",
+    "    'df_arabert': 1.0,\n",
+    "    'df_dziribert': 1.0,\n",
+    "    'df_gemma2': 1.0,\n",
+    "    'df_marbert': 1.0\n",
+    "}\n",
+    "\n",
+    "# Create a combined dataframe with id and Commentaire client from the first dataframe\n",
+    "result_df = df_gemma1[['id', 'Réseau Social', 'Commentaire client']].copy()\n",
+    "\n",
+    "# Initialize a dictionary to store weighted vote counts for each class\n",
+    "from collections import defaultdict\n",
+    "import numpy as np\n",
+    "\n",
+    "# For each row, calculate weighted votes\n",
+    "final_predictions = []\n",
+    "\n",
+    "for idx in range(len(df_gemma1)):\n",
+    "    vote_counts = defaultdict(float)\n",
+    "    \n",
+    "    # Add weighted votes from each model\n",
+    "    vote_counts[df_gemma1.iloc[idx]['Predicted_Class']] += weights['df_gemma1']\n",
+    "    vote_counts[df_camel.iloc[idx]['Predicted_Class']] += weights['df_camel']\n",
+    "    vote_counts[df_arabert.iloc[idx]['Predicted_Class']] += weights['df_arabert']\n",
+    "    vote_counts[df_dziribert.iloc[idx]['Predicted_Class']] += weights['df_dziribert']\n",
+    "    vote_counts[df_gemma2.iloc[idx]['Predicted_Class']] += weights['df_gemma2']\n",
+    "    vote_counts[df_marbert.iloc[idx]['Predicted_Class']] += weights['df_marbert']\n",
+    "    \n",
+    "    # Select class with highest weighted vote\n",
+    "    final_prediction = max(vote_counts.items(), key=lambda x: x[1])[0]\n",
+    "    final_predictions.append(final_prediction)\n",
+    "\n",
+    "# Add predictions to result dataframe\n",
+    "result_df['Predicted_Class'] = final_predictions\n",
+    "\n",
+    "# Display statistics\n",
+    "print(f\"Total samples: {len(result_df)}\")\n",
+    "print(f\"\\nClass distribution:\")\n",
+    "print(result_df['Predicted_Class'].value_counts().sort_index())\n",
+    "print(f\"\\nWeight configuration:\")\n",
+    "for model, weight in weights.items():\n",
+    "    print(f\"  {model}: {weight}\")\n",
+    "\n",
+    "# Display first few rows\n",
+    "print(f\"\\nFirst 5 predictions:\")\n",
+    "result_df.head()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "543f2936",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save results to CSV (only id and Class)\n",
+    "output_filename = 'test_predictions_weighted_voting_ensemble.csv'\n",
+    "output_df = result_df[['id', 'Predicted_Class']].copy()\n",
+    "output_df.rename(columns={'Predicted_Class': 'Class'}, inplace=True)\n",
+    "output_df.to_csv(output_filename, index=False)\n",
+    "print(f\"Results saved to: {output_filename}\")\n",
+    "print(f\"Columns in output: id, Class\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,54 @@
+# ============================================================================
+# Requirements for Arabic LLM Training & Inference
+# ============================================================================
+# Install: pip install -r requirements.txt
+# ============================================================================
+
+# Core ML Frameworks
+torch>=2.1.0
+transformers>=4.45.0
+accelerate>=0.25.0
+datasets>=2.16.0
+
+# Parameter-Efficient Fine-Tuning
+peft>=0.7.0
+
+# Tokenizers
+tokenizers>=0.15.0
+sentencepiece>=0.1.99
+
+# Data Processing
+pandas>=2.0.0
+numpy>=1.24.0
+scikit-learn>=1.3.0
+
+# Progress & Logging
+tqdm>=4.66.0
+
+# Evaluation Metrics
+evaluate>=0.4.0
+seqeval>=1.2.2
+
+# Optional: Weights & Biases for experiment tracking
+# wandb>=0.16.0
+
+# Optional: Flash Attention 2 (requires CUDA)
+# flash-attn>=2.3.0
+
+# Optional: BitsAndBytes for quantization
+# bitsandbytes>=0.41.0
+
+# Jupyter Support (optional)
+# jupyter>=1.0.0
+# ipykernel>=6.25.0
+
+# Hugging Face Hub
+huggingface-hub>=0.19.0
+
+# SafeTensors for efficient model loading
+safetensors>=0.4.0
+
+# Regex for text preprocessing
+regex>=2023.10.0
+
+

telecom_camelbert_cpt_ft/checkpoint-3500/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "1",
+    "1": "2",
+    "2": "3",
+    "3": "4",
+    "4": "5",
+    "5": "6",
+    "6": "7",
+    "7": "8",
+    "8": "9"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "4": 5,
+    "5": 6,
+    "6": 7,
+    "7": 8,
+    "8": 9
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.57.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30000
+}

telecom_camelbert_cpt_ft/checkpoint-3500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2a6a9d45f9658c97abc75e064c8148f35c6b1cd6f9740d212d898d79b77f591
+size 436376588

telecom_camelbert_cpt_ft/checkpoint-3500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c98dfe00bf6e8edda2b26395a77170bcec1a4cfaa60b9bf81982603ae218a218
+size 872877451

telecom_camelbert_cpt_ft/checkpoint-3500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16c4b35aee0bbaeee4ff40ea0ec57a691c91ae4126d7f2611b0621dcea397d18
+size 14645

telecom_camelbert_cpt_ft/checkpoint-3500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fba3938712732ea9a2d4d892486d311187e6b050bac015163f3186f8c0ea04f4
+size 1465

telecom_camelbert_cpt_ft/checkpoint-3500/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

telecom_camelbert_cpt_ft/checkpoint-3500/tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "extra_special_tokens": {},
+  "full_tokenizer_file": null,
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

telecom_camelbert_cpt_ft/checkpoint-3500/trainer_state.json

@@ -0,0 +1,524 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 68.62745098039215,
+  "eval_steps": 500,
+  "global_step": 3500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.9803921568627451,
+      "grad_norm": 2.6230194568634033,
+      "learning_rate": 2.7450980392156867e-06,
+      "loss": 2.2283,
+      "step": 50
+    },
+    {
+      "epoch": 1.9607843137254903,
+      "grad_norm": 3.465364933013916,
+      "learning_rate": 5.546218487394959e-06,
+      "loss": 2.1335,
+      "step": 100
+    },
+    {
+      "epoch": 2.9411764705882355,
+      "grad_norm": 4.075287818908691,
+      "learning_rate": 8.34733893557423e-06,
+      "loss": 1.8493,
+      "step": 150
+    },
+    {
+      "epoch": 3.9215686274509802,
+      "grad_norm": 4.183288097381592,
+      "learning_rate": 1.1148459383753503e-05,
+      "loss": 1.3306,
+      "step": 200
+    },
+    {
+      "epoch": 4.901960784313726,
+      "grad_norm": 6.412795066833496,
+      "learning_rate": 1.3949579831932774e-05,
+      "loss": 0.8739,
+      "step": 250
+    },
+    {
+      "epoch": 5.882352941176471,
+      "grad_norm": 9.716330528259277,
+      "learning_rate": 1.6750700280112046e-05,
+      "loss": 0.6337,
+      "step": 300
+    },
+    {
+      "epoch": 6.862745098039216,
+      "grad_norm": 6.941835403442383,
+      "learning_rate": 1.9551820728291318e-05,
+      "loss": 0.4314,
+      "step": 350
+    },
+    {
+      "epoch": 7.8431372549019605,
+      "grad_norm": 3.7662336826324463,
+      "learning_rate": 1.973856209150327e-05,
+      "loss": 0.2944,
+      "step": 400
+    },
+    {
+      "epoch": 8.823529411764707,
+      "grad_norm": 4.981897354125977,
+      "learning_rate": 1.9427326486150017e-05,
+      "loss": 0.1731,
+      "step": 450
+    },
+    {
+      "epoch": 9.803921568627452,
+      "grad_norm": 9.845929145812988,
+      "learning_rate": 1.9116090880796766e-05,
+      "loss": 0.1273,
+      "step": 500
+    },
+    {
+      "epoch": 10.784313725490197,
+      "grad_norm": 8.743799209594727,
+      "learning_rate": 1.880485527544351e-05,
+      "loss": 0.077,
+      "step": 550
+    },
+    {
+      "epoch": 11.764705882352942,
+      "grad_norm": 0.8690173625946045,
+      "learning_rate": 1.849361967009026e-05,
+      "loss": 0.0603,
+      "step": 600
+    },
+    {
+      "epoch": 12.745098039215687,
+      "grad_norm": 0.22123952209949493,
+      "learning_rate": 1.8182384064737007e-05,
+      "loss": 0.0304,
+      "step": 650
+    },
+    {
+      "epoch": 13.72549019607843,
+      "grad_norm": 0.263492226600647,
+      "learning_rate": 1.7871148459383755e-05,
+      "loss": 0.0263,
+      "step": 700
+    },
+    {
+      "epoch": 14.705882352941176,
+      "grad_norm": 0.9709652066230774,
+      "learning_rate": 1.7559912854030504e-05,
+      "loss": 0.0307,
+      "step": 750
+    },
+    {
+      "epoch": 15.686274509803921,
+      "grad_norm": 0.03851320594549179,
+      "learning_rate": 1.724867724867725e-05,
+      "loss": 0.0237,
+      "step": 800
+    },
+    {
+      "epoch": 16.666666666666668,
+      "grad_norm": 3.8813209533691406,
+      "learning_rate": 1.6937441643323997e-05,
+      "loss": 0.0244,
+      "step": 850
+    },
+    {
+      "epoch": 17.647058823529413,
+      "grad_norm": 0.14205335080623627,
+      "learning_rate": 1.6626206037970745e-05,
+      "loss": 0.0103,
+      "step": 900
+    },
+    {
+      "epoch": 18.627450980392158,
+      "grad_norm": 0.03730874881148338,
+      "learning_rate": 1.631497043261749e-05,
+      "loss": 0.0123,
+      "step": 950
+    },
+    {
+      "epoch": 19.607843137254903,
+      "grad_norm": 0.047113899141550064,
+      "learning_rate": 1.6003734827264242e-05,
+      "loss": 0.0203,
+      "step": 1000
+    },
+    {
+      "epoch": 20.58823529411765,
+      "grad_norm": 0.019405728206038475,
+      "learning_rate": 1.5692499221910987e-05,
+      "loss": 0.0105,
+      "step": 1050
+    },
+    {
+      "epoch": 21.568627450980394,
+      "grad_norm": 0.022607196122407913,
+      "learning_rate": 1.5381263616557735e-05,
+      "loss": 0.0133,
+      "step": 1100
+    },
+    {
+      "epoch": 22.54901960784314,
+      "grad_norm": 0.14680148661136627,
+      "learning_rate": 1.5070028011204482e-05,
+      "loss": 0.012,
+      "step": 1150
+    },
+    {
+      "epoch": 23.529411764705884,
+      "grad_norm": 0.07145881652832031,
+      "learning_rate": 1.475879240585123e-05,
+      "loss": 0.011,
+      "step": 1200
+    },
+    {
+      "epoch": 24.509803921568626,
+      "grad_norm": 0.017874594777822495,
+      "learning_rate": 1.4447556800497977e-05,
+      "loss": 0.0121,
+      "step": 1250
+    },
+    {
+      "epoch": 25.49019607843137,
+      "grad_norm": 0.014184300787746906,
+      "learning_rate": 1.4136321195144727e-05,
+      "loss": 0.0082,
+      "step": 1300
+    },
+    {
+      "epoch": 26.470588235294116,
+      "grad_norm": 0.4630926847457886,
+      "learning_rate": 1.3825085589791474e-05,
+      "loss": 0.0089,
+      "step": 1350
+    },
+    {
+      "epoch": 27.45098039215686,
+      "grad_norm": 1.217696189880371,
+      "learning_rate": 1.351384998443822e-05,
+      "loss": 0.0037,
+      "step": 1400
+    },
+    {
+      "epoch": 28.431372549019606,
+      "grad_norm": 0.036678045988082886,
+      "learning_rate": 1.3202614379084969e-05,
+      "loss": 0.0072,
+      "step": 1450
+    },
+    {
+      "epoch": 29.41176470588235,
+      "grad_norm": 0.014452925883233547,
+      "learning_rate": 1.2891378773731715e-05,
+      "loss": 0.0047,
+      "step": 1500
+    },
+    {
+      "epoch": 30.392156862745097,
+      "grad_norm": 0.016204573214054108,
+      "learning_rate": 1.2580143168378462e-05,
+      "loss": 0.0053,
+      "step": 1550
+    },
+    {
+      "epoch": 31.372549019607842,
+      "grad_norm": 0.01764889620244503,
+      "learning_rate": 1.2268907563025212e-05,
+      "loss": 0.0049,
+      "step": 1600
+    },
+    {
+      "epoch": 32.35294117647059,
+      "grad_norm": 0.01166362501680851,
+      "learning_rate": 1.1957671957671959e-05,
+      "loss": 0.0054,
+      "step": 1650
+    },
+    {
+      "epoch": 33.333333333333336,
+      "grad_norm": 0.0751669779419899,
+      "learning_rate": 1.1646436352318707e-05,
+      "loss": 0.0048,
+      "step": 1700
+    },
+    {
+      "epoch": 34.31372549019608,
+      "grad_norm": 0.01111924834549427,
+      "learning_rate": 1.1335200746965454e-05,
+      "loss": 0.0064,
+      "step": 1750
+    },
+    {
+      "epoch": 35.294117647058826,
+      "grad_norm": 0.040332596749067307,
+      "learning_rate": 1.10239651416122e-05,
+      "loss": 0.0061,
+      "step": 1800
+    },
+    {
+      "epoch": 36.27450980392157,
+      "grad_norm": 0.015221468172967434,
+      "learning_rate": 1.0712729536258948e-05,
+      "loss": 0.0051,
+      "step": 1850
+    },
+    {
+      "epoch": 37.254901960784316,
+      "grad_norm": 0.010177787393331528,
+      "learning_rate": 1.0401493930905697e-05,
+      "loss": 0.0056,
+      "step": 1900
+    },
+    {
+      "epoch": 38.23529411764706,
+      "grad_norm": 0.006494768429547548,
+      "learning_rate": 1.0090258325552445e-05,
+      "loss": 0.0038,
+      "step": 1950
+    },
+    {
+      "epoch": 39.21568627450981,
+      "grad_norm": 0.021074198186397552,
+      "learning_rate": 9.779022720199192e-06,
+      "loss": 0.0037,
+      "step": 2000
+    },
+    {
+      "epoch": 40.19607843137255,
+      "grad_norm": 0.7565334439277649,
+      "learning_rate": 9.467787114845938e-06,
+      "loss": 0.0075,
+      "step": 2050
+    },
+    {
+      "epoch": 41.1764705882353,
+      "grad_norm": 0.007625663187354803,
+      "learning_rate": 9.156551509492687e-06,
+      "loss": 0.0025,
+      "step": 2100
+    },
+    {
+      "epoch": 42.15686274509804,
+      "grad_norm": 0.006594022735953331,
+      "learning_rate": 8.845315904139435e-06,
+      "loss": 0.0041,
+      "step": 2150
+    },
+    {
+      "epoch": 43.13725490196079,
+      "grad_norm": 0.5956095457077026,
+      "learning_rate": 8.534080298786182e-06,
+      "loss": 0.0052,
+      "step": 2200
+    },
+    {
+      "epoch": 44.11764705882353,
+      "grad_norm": 0.005290526431053877,
+      "learning_rate": 8.22284469343293e-06,
+      "loss": 0.0037,
+      "step": 2250
+    },
+    {
+      "epoch": 45.09803921568628,
+      "grad_norm": 0.12142275273799896,
+      "learning_rate": 7.911609088079677e-06,
+      "loss": 0.0049,
+      "step": 2300
+    },
+    {
+      "epoch": 46.07843137254902,
+      "grad_norm": 0.512083888053894,
+      "learning_rate": 7.600373482726424e-06,
+      "loss": 0.0038,
+      "step": 2350
+    },
+    {
+      "epoch": 47.05882352941177,
+      "grad_norm": 0.009066535159945488,
+      "learning_rate": 7.2891378773731725e-06,
+      "loss": 0.0038,
+      "step": 2400
+    },
+    {
+      "epoch": 48.03921568627451,
+      "grad_norm": 0.0053984676487743855,
+      "learning_rate": 6.97790227201992e-06,
+      "loss": 0.0037,
+      "step": 2450
+    },
+    {
+      "epoch": 49.01960784313726,
+      "grad_norm": 0.0097350450232625,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.0031,
+      "step": 2500
+    },
+    {
+      "epoch": 50.0,
+      "grad_norm": 0.00987960398197174,
+      "learning_rate": 6.355431061313415e-06,
+      "loss": 0.0039,
+      "step": 2550
+    },
+    {
+      "epoch": 50.98039215686274,
+      "grad_norm": 0.2973344624042511,
+      "learning_rate": 6.0441954559601625e-06,
+      "loss": 0.0044,
+      "step": 2600
+    },
+    {
+      "epoch": 51.96078431372549,
+      "grad_norm": 0.009365350939333439,
+      "learning_rate": 5.732959850606909e-06,
+      "loss": 0.0036,
+      "step": 2650
+    },
+    {
+      "epoch": 52.94117647058823,
+      "grad_norm": 0.005942502524703741,
+      "learning_rate": 5.4217242452536574e-06,
+      "loss": 0.0038,
+      "step": 2700
+    },
+    {
+      "epoch": 53.92156862745098,
+      "grad_norm": 0.008297664113342762,
+      "learning_rate": 5.110488639900405e-06,
+      "loss": 0.0034,
+      "step": 2750
+    },
+    {
+      "epoch": 54.90196078431372,
+      "grad_norm": 0.7482114434242249,
+      "learning_rate": 4.799253034547152e-06,
+      "loss": 0.0037,
+      "step": 2800
+    },
+    {
+      "epoch": 55.88235294117647,
+      "grad_norm": 0.00625429954379797,
+      "learning_rate": 4.4880174291939e-06,
+      "loss": 0.0026,
+      "step": 2850
+    },
+    {
+      "epoch": 56.86274509803921,
+      "grad_norm": 0.008189293555915356,
+      "learning_rate": 4.176781823840647e-06,
+      "loss": 0.0036,
+      "step": 2900
+    },
+    {
+      "epoch": 57.84313725490196,
+      "grad_norm": 0.006718316115438938,
+      "learning_rate": 3.865546218487396e-06,
+      "loss": 0.0035,
+      "step": 2950
+    },
+    {
+      "epoch": 58.8235294117647,
+      "grad_norm": 0.024499941617250443,
+      "learning_rate": 3.5543106131341428e-06,
+      "loss": 0.0027,
+      "step": 3000
+    },
+    {
+      "epoch": 59.80392156862745,
+      "grad_norm": 0.007647670805454254,
+      "learning_rate": 3.2430750077808903e-06,
+      "loss": 0.0039,
+      "step": 3050
+    },
+    {
+      "epoch": 60.78431372549019,
+      "grad_norm": 0.0057996599934995174,
+      "learning_rate": 2.931839402427638e-06,
+      "loss": 0.0036,
+      "step": 3100
+    },
+    {
+      "epoch": 61.76470588235294,
+      "grad_norm": 0.004913662560284138,
+      "learning_rate": 2.6206037970743852e-06,
+      "loss": 0.0038,
+      "step": 3150
+    },
+    {
+      "epoch": 62.745098039215684,
+      "grad_norm": 0.004377233795821667,
+      "learning_rate": 2.309368191721133e-06,
+      "loss": 0.0021,
+      "step": 3200
+    },
+    {
+      "epoch": 63.72549019607843,
+      "grad_norm": 0.00702957296743989,
+      "learning_rate": 1.9981325863678806e-06,
+      "loss": 0.0044,
+      "step": 3250
+    },
+    {
+      "epoch": 64.70588235294117,
+      "grad_norm": 0.004603248089551926,
+      "learning_rate": 1.6868969810146283e-06,
+      "loss": 0.0039,
+      "step": 3300
+    },
+    {
+      "epoch": 65.68627450980392,
+      "grad_norm": 0.009002058766782284,
+      "learning_rate": 1.3756613756613758e-06,
+      "loss": 0.0018,
+      "step": 3350
+    },
+    {
+      "epoch": 66.66666666666667,
+      "grad_norm": 0.005587454419583082,
+      "learning_rate": 1.0644257703081233e-06,
+      "loss": 0.004,
+      "step": 3400
+    },
+    {
+      "epoch": 67.6470588235294,
+      "grad_norm": 0.007010480388998985,
+      "learning_rate": 7.531901649548709e-07,
+      "loss": 0.0026,
+      "step": 3450
+    },
+    {
+      "epoch": 68.62745098039215,
+      "grad_norm": 0.004527617245912552,
+      "learning_rate": 4.419545596016184e-07,
+      "loss": 0.0043,
+      "step": 3500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 70,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.909454409554739e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

telecom_camelbert_cpt_ft/checkpoint-3500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b5f9538422ac3ff516fbd5394c01499e3df51a8944c66ec759cd64011b08382
+size 5841

telecom_camelbert_cpt_ft/checkpoint-3570/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "1",
+    "1": "2",
+    "2": "3",
+    "3": "4",
+    "4": "5",
+    "5": "6",
+    "6": "7",
+    "7": "8",
+    "8": "9"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "4": 5,
+    "5": 6,
+    "6": 7,
+    "7": 8,
+    "8": 9
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.57.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30000
+}

telecom_camelbert_cpt_ft/checkpoint-3570/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fd4a9bdb6b95896f2d56f3c3b16e3a65a00020c2a659d7c25b29533d04b2ebe
+size 436376588

telecom_camelbert_cpt_ft/checkpoint-3570/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea3f001a73b720b183d52fdf114366fb67b446a305b0ba6f34d49835c2d8efb7
+size 872877451

telecom_camelbert_cpt_ft/checkpoint-3570/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f9c5ee45d5ae8a5fde2e385c98fb73e0e572ce4d71b29f086ecc189bb285563
+size 14645

telecom_camelbert_cpt_ft/checkpoint-3570/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffad826366e2d47eed8cd887fe4778e4084716a2369d14b2d2b195e111ed1c29
+size 1465

telecom_camelbert_cpt_ft/checkpoint-3570/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

telecom_camelbert_cpt_ft/checkpoint-3570/tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "extra_special_tokens": {},
+  "full_tokenizer_file": null,
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

telecom_camelbert_cpt_ft/checkpoint-3570/trainer_state.json

@@ -0,0 +1,531 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 70.0,
+  "eval_steps": 500,
+  "global_step": 3570,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.9803921568627451,
+      "grad_norm": 2.6230194568634033,
+      "learning_rate": 2.7450980392156867e-06,
+      "loss": 2.2283,
+      "step": 50
+    },
+    {
+      "epoch": 1.9607843137254903,
+      "grad_norm": 3.465364933013916,
+      "learning_rate": 5.546218487394959e-06,
+      "loss": 2.1335,
+      "step": 100
+    },
+    {
+      "epoch": 2.9411764705882355,
+      "grad_norm": 4.075287818908691,
+      "learning_rate": 8.34733893557423e-06,
+      "loss": 1.8493,
+      "step": 150
+    },
+    {
+      "epoch": 3.9215686274509802,
+      "grad_norm": 4.183288097381592,
+      "learning_rate": 1.1148459383753503e-05,
+      "loss": 1.3306,
+      "step": 200
+    },
+    {
+      "epoch": 4.901960784313726,
+      "grad_norm": 6.412795066833496,
+      "learning_rate": 1.3949579831932774e-05,
+      "loss": 0.8739,
+      "step": 250
+    },
+    {
+      "epoch": 5.882352941176471,
+      "grad_norm": 9.716330528259277,
+      "learning_rate": 1.6750700280112046e-05,
+      "loss": 0.6337,
+      "step": 300
+    },
+    {
+      "epoch": 6.862745098039216,
+      "grad_norm": 6.941835403442383,
+      "learning_rate": 1.9551820728291318e-05,
+      "loss": 0.4314,
+      "step": 350
+    },
+    {
+      "epoch": 7.8431372549019605,
+      "grad_norm": 3.7662336826324463,
+      "learning_rate": 1.973856209150327e-05,
+      "loss": 0.2944,
+      "step": 400
+    },
+    {
+      "epoch": 8.823529411764707,
+      "grad_norm": 4.981897354125977,
+      "learning_rate": 1.9427326486150017e-05,
+      "loss": 0.1731,
+      "step": 450
+    },
+    {
+      "epoch": 9.803921568627452,
+      "grad_norm": 9.845929145812988,
+      "learning_rate": 1.9116090880796766e-05,
+      "loss": 0.1273,
+      "step": 500
+    },
+    {
+      "epoch": 10.784313725490197,
+      "grad_norm": 8.743799209594727,
+      "learning_rate": 1.880485527544351e-05,
+      "loss": 0.077,
+      "step": 550
+    },
+    {
+      "epoch": 11.764705882352942,
+      "grad_norm": 0.8690173625946045,
+      "learning_rate": 1.849361967009026e-05,
+      "loss": 0.0603,
+      "step": 600
+    },
+    {
+      "epoch": 12.745098039215687,
+      "grad_norm": 0.22123952209949493,
+      "learning_rate": 1.8182384064737007e-05,
+      "loss": 0.0304,
+      "step": 650
+    },
+    {
+      "epoch": 13.72549019607843,
+      "grad_norm": 0.263492226600647,
+      "learning_rate": 1.7871148459383755e-05,
+      "loss": 0.0263,
+      "step": 700
+    },
+    {
+      "epoch": 14.705882352941176,
+      "grad_norm": 0.9709652066230774,
+      "learning_rate": 1.7559912854030504e-05,
+      "loss": 0.0307,
+      "step": 750
+    },
+    {
+      "epoch": 15.686274509803921,
+      "grad_norm": 0.03851320594549179,
+      "learning_rate": 1.724867724867725e-05,
+      "loss": 0.0237,
+      "step": 800
+    },
+    {
+      "epoch": 16.666666666666668,
+      "grad_norm": 3.8813209533691406,
+      "learning_rate": 1.6937441643323997e-05,
+      "loss": 0.0244,
+      "step": 850
+    },
+    {
+      "epoch": 17.647058823529413,
+      "grad_norm": 0.14205335080623627,
+      "learning_rate": 1.6626206037970745e-05,
+      "loss": 0.0103,
+      "step": 900
+    },
+    {
+      "epoch": 18.627450980392158,
+      "grad_norm": 0.03730874881148338,
+      "learning_rate": 1.631497043261749e-05,
+      "loss": 0.0123,
+      "step": 950
+    },
+    {
+      "epoch": 19.607843137254903,
+      "grad_norm": 0.047113899141550064,
+      "learning_rate": 1.6003734827264242e-05,
+      "loss": 0.0203,
+      "step": 1000
+    },
+    {
+      "epoch": 20.58823529411765,
+      "grad_norm": 0.019405728206038475,
+      "learning_rate": 1.5692499221910987e-05,
+      "loss": 0.0105,
+      "step": 1050
+    },
+    {
+      "epoch": 21.568627450980394,
+      "grad_norm": 0.022607196122407913,
+      "learning_rate": 1.5381263616557735e-05,
+      "loss": 0.0133,
+      "step": 1100
+    },
+    {
+      "epoch": 22.54901960784314,
+      "grad_norm": 0.14680148661136627,
+      "learning_rate": 1.5070028011204482e-05,
+      "loss": 0.012,
+      "step": 1150
+    },
+    {
+      "epoch": 23.529411764705884,
+      "grad_norm": 0.07145881652832031,
+      "learning_rate": 1.475879240585123e-05,
+      "loss": 0.011,
+      "step": 1200
+    },
+    {
+      "epoch": 24.509803921568626,
+      "grad_norm": 0.017874594777822495,
+      "learning_rate": 1.4447556800497977e-05,
+      "loss": 0.0121,
+      "step": 1250
+    },
+    {
+      "epoch": 25.49019607843137,
+      "grad_norm": 0.014184300787746906,
+      "learning_rate": 1.4136321195144727e-05,
+      "loss": 0.0082,
+      "step": 1300
+    },
+    {
+      "epoch": 26.470588235294116,
+      "grad_norm": 0.4630926847457886,
+      "learning_rate": 1.3825085589791474e-05,
+      "loss": 0.0089,
+      "step": 1350
+    },
+    {
+      "epoch": 27.45098039215686,
+      "grad_norm": 1.217696189880371,
+      "learning_rate": 1.351384998443822e-05,
+      "loss": 0.0037,
+      "step": 1400
+    },
+    {
+      "epoch": 28.431372549019606,
+      "grad_norm": 0.036678045988082886,
+      "learning_rate": 1.3202614379084969e-05,
+      "loss": 0.0072,
+      "step": 1450
+    },
+    {
+      "epoch": 29.41176470588235,
+      "grad_norm": 0.014452925883233547,
+      "learning_rate": 1.2891378773731715e-05,
+      "loss": 0.0047,
+      "step": 1500
+    },
+    {
+      "epoch": 30.392156862745097,
+      "grad_norm": 0.016204573214054108,
+      "learning_rate": 1.2580143168378462e-05,
+      "loss": 0.0053,
+      "step": 1550
+    },
+    {
+      "epoch": 31.372549019607842,
+      "grad_norm": 0.01764889620244503,
+      "learning_rate": 1.2268907563025212e-05,
+      "loss": 0.0049,
+      "step": 1600
+    },
+    {
+      "epoch": 32.35294117647059,
+      "grad_norm": 0.01166362501680851,
+      "learning_rate": 1.1957671957671959e-05,
+      "loss": 0.0054,
+      "step": 1650
+    },
+    {
+      "epoch": 33.333333333333336,
+      "grad_norm": 0.0751669779419899,
+      "learning_rate": 1.1646436352318707e-05,
+      "loss": 0.0048,
+      "step": 1700
+    },
+    {
+      "epoch": 34.31372549019608,
+      "grad_norm": 0.01111924834549427,
+      "learning_rate": 1.1335200746965454e-05,
+      "loss": 0.0064,
+      "step": 1750
+    },
+    {
+      "epoch": 35.294117647058826,
+      "grad_norm": 0.040332596749067307,
+      "learning_rate": 1.10239651416122e-05,
+      "loss": 0.0061,
+      "step": 1800
+    },
+    {
+      "epoch": 36.27450980392157,
+      "grad_norm": 0.015221468172967434,
+      "learning_rate": 1.0712729536258948e-05,
+      "loss": 0.0051,
+      "step": 1850
+    },
+    {
+      "epoch": 37.254901960784316,
+      "grad_norm": 0.010177787393331528,
+      "learning_rate": 1.0401493930905697e-05,
+      "loss": 0.0056,
+      "step": 1900
+    },
+    {
+      "epoch": 38.23529411764706,
+      "grad_norm": 0.006494768429547548,
+      "learning_rate": 1.0090258325552445e-05,
+      "loss": 0.0038,
+      "step": 1950
+    },
+    {
+      "epoch": 39.21568627450981,
+      "grad_norm": 0.021074198186397552,
+      "learning_rate": 9.779022720199192e-06,
+      "loss": 0.0037,
+      "step": 2000
+    },
+    {
+      "epoch": 40.19607843137255,
+      "grad_norm": 0.7565334439277649,
+      "learning_rate": 9.467787114845938e-06,
+      "loss": 0.0075,
+      "step": 2050
+    },
+    {
+      "epoch": 41.1764705882353,
+      "grad_norm": 0.007625663187354803,
+      "learning_rate": 9.156551509492687e-06,
+      "loss": 0.0025,
+      "step": 2100
+    },
+    {
+      "epoch": 42.15686274509804,
+      "grad_norm": 0.006594022735953331,
+      "learning_rate": 8.845315904139435e-06,
+      "loss": 0.0041,
+      "step": 2150
+    },
+    {
+      "epoch": 43.13725490196079,
+      "grad_norm": 0.5956095457077026,
+      "learning_rate": 8.534080298786182e-06,
+      "loss": 0.0052,
+      "step": 2200
+    },
+    {
+      "epoch": 44.11764705882353,
+      "grad_norm": 0.005290526431053877,
+      "learning_rate": 8.22284469343293e-06,
+      "loss": 0.0037,
+      "step": 2250
+    },
+    {
+      "epoch": 45.09803921568628,
+      "grad_norm": 0.12142275273799896,
+      "learning_rate": 7.911609088079677e-06,
+      "loss": 0.0049,
+      "step": 2300
+    },
+    {
+      "epoch": 46.07843137254902,
+      "grad_norm": 0.512083888053894,
+      "learning_rate": 7.600373482726424e-06,
+      "loss": 0.0038,
+      "step": 2350
+    },
+    {
+      "epoch": 47.05882352941177,
+      "grad_norm": 0.009066535159945488,
+      "learning_rate": 7.2891378773731725e-06,
+      "loss": 0.0038,
+      "step": 2400
+    },
+    {
+      "epoch": 48.03921568627451,
+      "grad_norm": 0.0053984676487743855,
+      "learning_rate": 6.97790227201992e-06,
+      "loss": 0.0037,
+      "step": 2450
+    },
+    {
+      "epoch": 49.01960784313726,
+      "grad_norm": 0.0097350450232625,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.0031,
+      "step": 2500
+    },
+    {
+      "epoch": 50.0,
+      "grad_norm": 0.00987960398197174,
+      "learning_rate": 6.355431061313415e-06,
+      "loss": 0.0039,
+      "step": 2550
+    },
+    {
+      "epoch": 50.98039215686274,
+      "grad_norm": 0.2973344624042511,
+      "learning_rate": 6.0441954559601625e-06,
+      "loss": 0.0044,
+      "step": 2600
+    },
+    {
+      "epoch": 51.96078431372549,
+      "grad_norm": 0.009365350939333439,
+      "learning_rate": 5.732959850606909e-06,
+      "loss": 0.0036,
+      "step": 2650
+    },
+    {
+      "epoch": 52.94117647058823,
+      "grad_norm": 0.005942502524703741,
+      "learning_rate": 5.4217242452536574e-06,
+      "loss": 0.0038,
+      "step": 2700
+    },
+    {
+      "epoch": 53.92156862745098,
+      "grad_norm": 0.008297664113342762,
+      "learning_rate": 5.110488639900405e-06,
+      "loss": 0.0034,
+      "step": 2750
+    },
+    {
+      "epoch": 54.90196078431372,
+      "grad_norm": 0.7482114434242249,
+      "learning_rate": 4.799253034547152e-06,
+      "loss": 0.0037,
+      "step": 2800
+    },
+    {
+      "epoch": 55.88235294117647,
+      "grad_norm": 0.00625429954379797,
+      "learning_rate": 4.4880174291939e-06,
+      "loss": 0.0026,
+      "step": 2850
+    },
+    {
+      "epoch": 56.86274509803921,
+      "grad_norm": 0.008189293555915356,
+      "learning_rate": 4.176781823840647e-06,
+      "loss": 0.0036,
+      "step": 2900
+    },
+    {
+      "epoch": 57.84313725490196,
+      "grad_norm": 0.006718316115438938,
+      "learning_rate": 3.865546218487396e-06,
+      "loss": 0.0035,
+      "step": 2950
+    },
+    {
+      "epoch": 58.8235294117647,
+      "grad_norm": 0.024499941617250443,
+      "learning_rate": 3.5543106131341428e-06,
+      "loss": 0.0027,
+      "step": 3000
+    },
+    {
+      "epoch": 59.80392156862745,
+      "grad_norm": 0.007647670805454254,
+      "learning_rate": 3.2430750077808903e-06,
+      "loss": 0.0039,
+      "step": 3050
+    },
+    {
+      "epoch": 60.78431372549019,
+      "grad_norm": 0.0057996599934995174,
+      "learning_rate": 2.931839402427638e-06,
+      "loss": 0.0036,
+      "step": 3100
+    },
+    {
+      "epoch": 61.76470588235294,
+      "grad_norm": 0.004913662560284138,
+      "learning_rate": 2.6206037970743852e-06,
+      "loss": 0.0038,
+      "step": 3150
+    },
+    {
+      "epoch": 62.745098039215684,
+      "grad_norm": 0.004377233795821667,
+      "learning_rate": 2.309368191721133e-06,
+      "loss": 0.0021,
+      "step": 3200
+    },
+    {
+      "epoch": 63.72549019607843,
+      "grad_norm": 0.00702957296743989,
+      "learning_rate": 1.9981325863678806e-06,
+      "loss": 0.0044,
+      "step": 3250
+    },
+    {
+      "epoch": 64.70588235294117,
+      "grad_norm": 0.004603248089551926,
+      "learning_rate": 1.6868969810146283e-06,
+      "loss": 0.0039,
+      "step": 3300
+    },
+    {
+      "epoch": 65.68627450980392,
+      "grad_norm": 0.009002058766782284,
+      "learning_rate": 1.3756613756613758e-06,
+      "loss": 0.0018,
+      "step": 3350
+    },
+    {
+      "epoch": 66.66666666666667,
+      "grad_norm": 0.005587454419583082,
+      "learning_rate": 1.0644257703081233e-06,
+      "loss": 0.004,
+      "step": 3400
+    },
+    {
+      "epoch": 67.6470588235294,
+      "grad_norm": 0.007010480388998985,
+      "learning_rate": 7.531901649548709e-07,
+      "loss": 0.0026,
+      "step": 3450
+    },
+    {
+      "epoch": 68.62745098039215,
+      "grad_norm": 0.004527617245912552,
+      "learning_rate": 4.419545596016184e-07,
+      "loss": 0.0043,
+      "step": 3500
+    },
+    {
+      "epoch": 69.6078431372549,
+      "grad_norm": 0.004667927045375109,
+      "learning_rate": 1.3071895424836603e-07,
+      "loss": 0.0021,
+      "step": 3550
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3570,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 70,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.967289854262272e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

telecom_camelbert_cpt_ft/checkpoint-3570/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b5f9538422ac3ff516fbd5394c01499e3df51a8944c66ec759cd64011b08382
+size 5841

telecom_camelbert_cpt_ft/config.json

@@ -0,0 +1,49 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "1",
+    "1": "2",
+    "2": "3",
+    "3": "4",
+    "4": "5",
+    "5": "6",
+    "6": "7",
+    "7": "8",
+    "8": "9"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "4": 5,
+    "5": 6,
+    "6": 7,
+    "7": 8,
+    "8": 9
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.57.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30000,
+  "num_labels": 9
+}

telecom_camelbert_cpt_ft/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fd4a9bdb6b95896f2d56f3c3b16e3a65a00020c2a659d7c25b29533d04b2ebe
+size 436376588

telecom_camelbert_cpt_ft/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

telecom_camelbert_cpt_ft/tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "extra_special_tokens": {},
+  "full_tokenizer_file": null,
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

telecom_camelbert_cpt_ft/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b5f9538422ac3ff516fbd5394c01499e3df51a8944c66ec759cd64011b08382
+size 5841

telecom_dziribert_final/checkpoint-8000/config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "4": 5,
+    "5": 6,
+    "6": 7,
+    "7": 8,
+    "8": 9
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "1": 0,
+    "2": 1,
+    "3": 2,
+    "4": 3,
+    "5": 4,
+    "6": 5,
+    "7": 6,
+    "8": 7,
+    "9": 8
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.57.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 50000
+}

telecom_dziribert_final/checkpoint-8000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f1b3eaaad3d15a8ef1c5277c33a0400c15e4ff0af900705aa9144de6f07a699
+size 497816604

telecom_dziribert_final/checkpoint-8000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12fe1e0aa55d8f2aab0ef48874b83f62f443e5e661e80f6a8cbb1102d6235a15
+size 995757451

telecom_dziribert_final/checkpoint-8000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd0507e2d7b94351111d5badfee1b3ed774bb8916d5cdd863b137ee35c00408
+size 14645

telecom_dziribert_final/checkpoint-8000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18caf96b2e25a0f89da808b5c5aed5fae7eea1994ec0fe0a96700f37665bfa6e
+size 1465

telecom_dziribert_final/checkpoint-8000/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

telecom_dziribert_final/checkpoint-8000/tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "max_len": 512,
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}