" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# ==========================================\n", "# 3. METRIC CALCULATION (SacreBLEU)\n", "# ==========================================\n", "metric = evaluate.load(\"sacrebleu\")\n", "\n", "def compute_metrics(eval_preds):\n", " preds, labels = eval_preds\n", " if isinstance(preds, tuple):\n", " preds = preds[0]\n", " \n", " # Decode predictions\n", " decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n", "\n", " # Replace -100 (ignored index) in labels\n", " labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n", " decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n", "\n", " # Post-processing for SacreBLEU\n", " decoded_preds = [pred.strip() for pred in decoded_preds]\n", " decoded_labels = [[label.strip()] for label in decoded_labels]\n", "\n", " result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n", " return {\"bleu\": result[\"score\"]}\n", "\n", "# ==========================================\n", "# 4. TRAINING CONFIGURATION\n", "# ==========================================\n", "training_args = Seq2SeqTrainingArguments(\n", " output_dir=\"./nllb-mni-lsftl\",\n", " evaluation_strategy=\"epoch\",\n", " save_strategy=\"epoch\",\n", " learning_rate=2e-4,\n", " per_device_train_batch_size=4,\n", " gradient_accumulation_steps=4,\n", " weight_decay=0.01,\n", " num_train_epochs=3,\n", " predict_with_generate=True, # Critical for BLEU calculation\n", " fp16=True,\n", " optim=\"adamw_torch\", # Forcing standard PyTorch optimizer\n", " logging_steps=10,\n", " save_total_limit=1,\n", " load_best_model_at_end=True\n", ")\n", "\n", "trainer = Seq2SeqTrainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=tokenized_ds[\"train\"],\n", " eval_dataset=tokenized_ds[\"validation\"],\n", " tokenizer=tokenizer,\n", " data_collator=DataCollatorForSeq2Seq(tokenizer, model=model),\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "# ==========================================\n", "# 5. EXECUTION: TRAIN & TEST\n", "# ==========================================\n", "print(\"Starting Fine-tuning...\")\n", "trainer.train()\n", "\n", "print(\"\\n--- Final Evaluation on Test Set ---\")\n", "test_results = trainer.evaluate(eval_dataset=tokenized_ds[\"test\"])\n", "print(f\"Test BLEU Score: {test_results['eval_bleu']:.2f}\")\n", "\n", "# ==========================================\n", "# 6. SAMPLE INFERENCE CHECK\n", "# ==========================================\n", "print(\"\\n--- Sample Predictions from Test Set ---\")\n", "model.eval()\n", "mni_id = tokenizer.convert_tokens_to_ids(\"mni_Beng\")\n", "\n", "for i in range(5):\n", " input_text = dataset[\"test\"][\"english\"][i]\n", " inputs = tokenizer(input_text, return_tensors=\"pt\").to(model.device)\n", " \n", " with torch.no_grad():\n", " generated_tokens = model.generate(\n", " **inputs, \n", " forced_bos_token_id=mni_id, \n", " max_length=128\n", " )\n", " \n", " prediction = tokenizer.decode(generated_tokens[0], skip_special_tokens=True)\n", " reference = dataset[\"test\"][\"manipuri\"][i]\n", " \n", " print(f\"Input: {input_text}\")\n", " print(f\"Pred: {prediction}\")\n", " print(f\"Truth: {reference}\\n\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "4b8abf64-f671-49f6-9fed-006c875a8485", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/teamspace/studios/this_studio'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pwd" ] }, { "cell_type": "code", "execution_count": 3, "id": "f4b05fb0-0738-4370-9686-6f1df42314a5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[0m\u001b[01;32mcheckpoint1.pt\u001b[0m* \u001b[01;32mcheckpoint15.pt\u001b[0m* \u001b[01;32mcheckpoint20.pt\u001b[0m* \u001b[01;32mcheckpoint8.pt\u001b[0m*\n", "\u001b[01;32mcheckpoint10.pt\u001b[0m* \u001b[01;32mcheckpoint16.pt\u001b[0m* \u001b[01;32mcheckpoint3.pt\u001b[0m* \u001b[01;32mcheckpoint9.pt\u001b[0m*\n", "\u001b[01;32mcheckpoint11.pt\u001b[0m* \u001b[01;32mcheckpoint17.pt\u001b[0m* \u001b[01;32mcheckpoint4.pt\u001b[0m* \u001b[01;32mcheckpoint_best.pt\u001b[0m*\n", "\u001b[01;32mcheckpoint12.pt\u001b[0m* \u001b[01;32mcheckpoint18.pt\u001b[0m* \u001b[01;32mcheckpoint5.pt\u001b[0m* \u001b[01;32mcheckpoint_last.pt\u001b[0m*\n", "\u001b[01;32mcheckpoint13.pt\u001b[0m* \u001b[01;32mcheckpoint19.pt\u001b[0m* \u001b[01;32mcheckpoint6.pt\u001b[0m*\n", "\u001b[01;32mcheckpoint14.pt\u001b[0m* \u001b[01;32mcheckpoint2.pt\u001b[0m* \u001b[01;32mcheckpoint7.pt\u001b[0m*\n" ] } ], "source": [ "ls\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "5b279caa-ceed-483c-b174-bb3eede61772", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/teamspace/studios/this_studio/checkpoints\n" ] } ], "source": [ "cd checkpoints" ] }, { "cell_type": "code", "execution_count": null, "id": "cc0f8abd-63e9-4de0-8c5b-ea9ff61d71c3", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.20" } }, "nbformat": 4, "nbformat_minor": 5 }