Upload 7 files

training/data/hausa_en.txt

@@ -0,0 +1,5 @@
+Yaya kake \t How are you
+Lafiya lau \t I am fine
+Na gode \t Thank you
+Don Allah \t Please
+Ya isa \t Enough

training/data/yoruba_en.txt

@@ -0,0 +1,5 @@
+Bawo ni \t How are you
+Mo wa daadaa \t I am fine
+E se \t Thank you
+Jowo \t Please
+O to \t Enough

training/train_trenslation.py

@@ -0,0 +1,52 @@
+import os
+import json
+from datasets import load_dataset
+from transformers import MarianTokenizer, MarianMTModel, Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq
+
+MODEL_NAME = "Helsinki-NLP/opus-mt-ha-en"   # Hausa-English base model
+OUTPUT_DIR = "./training/outputs/model"
+
+def train_from_jsonl(jsonl_path):
+    dataset = load_dataset("json", data_files={"train": jsonl_path}, split="train")
+
+    # Train/validation split
+    dataset = dataset.train_test_split(test_size=0.1)
+
+    tokenizer = MarianTokenizer.from_pretrained(MODEL_NAME)
+    model = MarianMTModel.from_pretrained(MODEL_NAME)
+
+    def preprocess(batch):
+        inputs = tokenizer(batch["src"], truncation=True, padding="max_length", max_length=128)
+        targets = tokenizer(batch["tgt"], truncation=True, padding="max_length", max_length=128)
+        inputs["labels"] = targets["input_ids"]
+        return inputs
+
+    tokenized = dataset.map(preprocess, batched=True, remove_columns=["src", "tgt"])
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    training_args = Seq2SeqTrainingArguments(
+        output_dir=OUTPUT_DIR,
+        evaluation_strategy="epoch",
+        learning_rate=5e-5,
+        per_device_train_batch_size=8,
+        per_device_eval_batch_size=8,
+        num_train_epochs=3,
+        weight_decay=0.01,
+        save_total_limit=2,
+        predict_with_generate=True,
+        logging_dir="./training/logs",
+    )
+
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized["train"],
+        eval_dataset=tokenized["test"],
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+    )
+
+    trainer.train()
+    trainer.save_model(OUTPUT_DIR)
+    tokenizer.save_pretrained(OUTPUT_DIR)
+    print("✅ Training complete. Model saved at", OUTPUT_DIR)