Rename ine_tune.py to fine_tune.py

ine_tune.py → fine_tune.py

@@ -2,8 +2,8 @@
 from datasets import load_dataset, load_metric
 from transformers import BartTokenizer, BartForConditionalGeneration, Trainer, TrainingArguments
 
-# 1️⃣ Деректерді жүктеу (ArXiv)
-dataset = load_dataset("scientific_papers", "arxiv")
+# 1️⃣ Датасетті жүктеу (Marcov ArXiv)
+dataset = load_dataset("marcov/scientific_papers_arxiv_promptsource")
 
 # Шағын subset (тест үшін)
 dataset["train"] = dataset["train"].select(range(1000))
@@ -20,7 +20,7 @@ max_output_length = 200
 # 3️⃣ Tokenization
 def preprocess_function(batch):
     inputs = tokenizer(batch["article"], max_length=max_input_length, truncation=True)
-    outputs = tokenizer(batch["abstract"], max_length=max_output_length, truncation=True)
+    outputs = tokenizer(batch["summary"], max_length=max_output_length, truncation=True)
     batch["input_ids"] = inputs["input_ids"]
     batch["attention_mask"] = inputs["attention_mask"]
     batch["labels"] = outputs["input_ids"]
@@ -41,7 +41,7 @@ def compute_metrics(eval_pred):
 
 # 5️⃣ TrainingArguments
 training_args = TrainingArguments(
-    output_dir="./bart-finetuned-arxiv",
+    output_dir="./bart-finetuned-arxiv-hub",
     evaluation_strategy="steps",
     eval_steps=500,
     save_steps=500,
@@ -54,6 +54,7 @@ training_args = TrainingArguments(
     fp16=True,
     logging_dir="./logs",
     logging_steps=100,
+    push_to_hub=True  # Fine-tuned моделді HuggingFace Hub-қа жіберу
 )
 
 # 6️⃣ Trainer
@@ -69,8 +70,7 @@ trainer = Trainer(
 # 7️⃣ Fine-tune бастау
 trainer.train()
 
-# 8️⃣ Модельді сақтау
-model.save_pretrained("./bart-finetuned-arxiv")
-tokenizer.save_pretrained("./bart-finetuned-arxiv")
+# 8️⃣ Модельді HuggingFace Hub-қа push жасау
+trainer.push_to_hub("username/bart-finetuned-arxiv")
 
-print("Fine-tuning аяқталды! Модель сақталды ./bart-finetuned-arxiv")
+print("Fine-tuning аяқталды! Модель Hub-қа жіберілді.")