Upload train2.py

train2.py

@@ -0,0 +1,124 @@
+import os
+import torch
+import pandas as pd
+import numpy as np
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    Trainer,
+    TrainingArguments,
+    DataCollatorWithPadding
+)
+from datasets import Dataset, load_from_disk
+from sklearn.metrics import accuracy_score, f1_score
+from sklearn.utils.class_weight import compute_class_weight
+from tqdm import tqdm
+
+# Set paths
+RAW_CSV = "data.csv"
+CACHE_DIR = "./cached_deberta_dataset"
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained("microsoft/deberta-v3-small")
+
+# Load or process dataset
+if os.path.exists(CACHE_DIR):
+    print("📦 Loading cached dataset...")
+    dataset = load_from_disk(CACHE_DIR)
+    train_ds, val_ds = dataset["train"], dataset["test"]
+else:
+    print("🔧 Processing and caching dataset...")
+    df = pd.read_csv(RAW_CSV)
+    df = df[["text", "organic"]]
+    df["organic"] = df["organic"].astype(int)
+
+    data = {
+        "text": df["text"].tolist(),
+        "label": df["organic"].tolist()
+    }
+
+    full_dataset = Dataset.from_dict(data)
+    dataset = full_dataset.train_test_split(test_size=0.1, seed=42)
+
+    def tokenize(batch):
+        tokenized = tokenizer(
+            batch["text"],
+            truncation=True,
+            padding="max_length",
+            max_length=512
+        )
+        tokenized["label"] = batch["label"]
+        return tokenized
+
+    dataset = dataset.map(tokenize, batched=True)
+    dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "label"])
+    dataset.save_to_disk(CACHE_DIR)
+    train_ds, val_ds = dataset["train"], dataset["test"]
+
+# Calculate class weights from training labels
+train_labels = np.array(train_ds["label"])
+class_weights = compute_class_weight(
+    class_weight="balanced",
+    classes=np.array([0, 1]),
+    y=train_labels
+)
+class_weights_tensor = torch.tensor(class_weights, dtype=torch.float)
+
+# Load model
+model = AutoModelForSequenceClassification.from_pretrained("microsoft/deberta-v3-small", num_labels=2)
+
+# Custom Trainer with weighted loss
+class WeightedLossTrainer(Trainer):
+    def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+        labels = inputs.pop("labels")
+        outputs = model(**inputs)
+        logits = outputs.logits
+        loss_fct = torch.nn.CrossEntropyLoss(weight=class_weights_tensor.to(logits.device))
+        loss = loss_fct(logits, labels)
+        return (loss, outputs) if return_outputs else loss
+
+# Evaluation metrics
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    preds = torch.tensor(logits).argmax(dim=-1)
+    acc = accuracy_score(labels, preds)
+    f1 = f1_score(labels, preds)
+    return {"accuracy": acc, "f1": f1}
+
+# Training arguments
+training_args = TrainingArguments(
+    output_dir="./ai-small-weighted",
+    evaluation_strategy="steps",
+    eval_steps=5000,
+    save_strategy="steps",
+    save_steps=5000,
+    save_total_limit=20,
+    logging_steps=10,
+    per_device_train_batch_size=48,
+    gradient_accumulation_steps=8,
+    num_train_epochs=3,
+    learning_rate=1e-6,
+    weight_decay=0.01,
+    max_grad_norm=1.0,
+    fp16=torch.cuda.is_available(),
+    load_best_model_at_end=True,
+    metric_for_best_model="f1",
+    greater_is_better=True,
+    logging_dir="./logs",
+)
+
+# Trainer
+trainer = WeightedLossTrainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_ds,
+    eval_dataset=val_ds,
+    tokenizer=tokenizer,
+    data_collator=DataCollatorWithPadding(tokenizer),
+    compute_metrics=compute_metrics,
+)
+
+# Train and save
+trainer.train()
+trainer.save_model("./ai-small-weighted/final_model")
+tokenizer.save_pretrained("./ai-small-weighted/final_model")