{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install datasets evaluate sacrebleu rouge-score bert-score -q","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:14:11.622000Z","iopub.execute_input":"2026-04-13T14:14:11.622715Z","iopub.status.idle":"2026-04-13T14:14:16.140334Z","shell.execute_reply.started":"2026-04-13T14:14:11.622665Z","shell.execute_reply":"2026-04-13T14:14:16.139284Z"}},"outputs":[{"name":"stdout","text":"\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.8/100.8 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.1/61.1 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h","output_type":"stream"}],"execution_count":17},{"cell_type":"code","source":"!pip install sentence-transformers -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:14:20.422711Z","iopub.execute_input":"2026-04-13T14:14:20.423409Z","iopub.status.idle":"2026-04-13T14:14:24.046218Z","shell.execute_reply.started":"2026-04-13T14:14:20.423372Z","shell.execute_reply":"2026-04-13T14:14:24.045336Z"}},"outputs":[],"execution_count":18},{"cell_type":"code","source":"!pip install evaluate sacrebleu rouge-score meteor-score -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:26.404687Z","iopub.execute_input":"2026-04-07T13:58:26.405469Z","iopub.status.idle":"2026-04-07T13:58:27.833303Z","shell.execute_reply.started":"2026-04-07T13:58:26.405432Z","shell.execute_reply":"2026-04-07T13:58:27.832472Z"}},"outputs":[{"name":"stdout","text":"\u001b[31mERROR: Could not find a version that satisfies the requirement meteor-score (from versions: none)\u001b[0m\u001b[31m\n\u001b[0m\u001b[31mERROR: No matching distribution found for meteor-score\u001b[0m\u001b[31m\n\u001b[0m","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"ai4bharat/IndicMTEval\")\n\nprint(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:57:56.518338Z","iopub.execute_input":"2026-04-07T13:57:56.518674Z","iopub.status.idle":"2026-04-07T13:58:04.179446Z","shell.execute_reply.started":"2026-04-07T13:57:56.518651Z","shell.execute_reply":"2026-04-07T13:58:04.178522Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bbf79717c7914d9bb57bd0544da75508"}},"metadata":{}},{"name":"stderr","text":"Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"data/test-00000-of-00001.parquet:   0%|          | 0.00/498k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"565c3cb7ad5a43e2a29724bd1e435a2f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/validation-00000-of-00001.parquet:   0%|          | 0.00/200k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e90d4fcdd024be999e3f586dfc11246"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/train-00000-of-00001.parquet:   0%|          | 0.00/1.13M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"425ca729b71e49feb37fab22b48e1912"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a45849449bd5417aa09b4c3538e53765"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/1000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3887b978c91542ad8574f5891ff6e25c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/4997 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"32c3ab646a1f4e719b507d54b501fb13"}},"metadata":{}},{"name":"stdout","text":"DatasetDict({\n    test: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 2380\n    })\n    validation: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 1000\n    })\n    train: Dataset({\n        features: ['src', 'ref', 'translation', 'mqm_norm_score', 'da_norm_score', 'error_spans', 'language', 'split'],\n        num_rows: 4997\n    })\n})\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"print(dataset['test'][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:38.291929Z","iopub.execute_input":"2026-04-07T13:58:38.292406Z","iopub.status.idle":"2026-04-07T13:58:38.298478Z","shell.execute_reply.started":"2026-04-07T13:58:38.292357Z","shell.execute_reply":"2026-04-07T13:58:38.297632Z"}},"outputs":[{"name":"stdout","text":"{'src': 'The Chaco region was home to other groups of indigenous tribes such as the Guaycurú and Payaguá, who survived by hunting, gathering and fishing.', 'ref': 'ਚਾਕੋ ਖੇਤਰ ਦੇਸੀ ਕਬੀਲਿਆਂ ਦੇ ਹੋਰ ਸਮੂਹਾਂ ਜਿਵੇਂ ਗਾਇਕੂਰੁ ਅਤੇ ਪਾਇਗੂਆ ਦਾ ਘਰ ਸੀ, ਜੋ ਸ਼ਿਕਾਰ, ਇਕੱਠੇ ਰਹਿਣ ਅਤੇ ਫਿਸ਼ਿੰਗ ਦੁਆਰਾ ਬਚ ਗਏ ਸਨ।', 'translation': 'ਚਾਕੋ ਖੇਤਰ ਸਵਦੇਸ਼ੀ ਕਬੀਲਿਆਂ ਦੇ ਹੋਰ ਸਮੂਹਾਂ ਦਾ ਘਰ ਸੀ ਜਿਵੇਂ ਕਿ ਗੁਆਏਕੁਰੂ ਅਤੇ ਪਯਾਗੁਆ, ਜੋ ਸ਼ਿਕਾਰ, ਇਕੱਠੇ ਕਰਨ ਅਤੇ ਮੱਛੀਆਂ ਫੜ ਕੇ ਬਚੇ ਸਨ।', 'mqm_norm_score': '0.84', 'da_norm_score': '0.4', 'error_spans': [{'span_end_offset': 24, 'span_no': 0, 'span_severity': 'High', 'span_start_offset': 17, 'span_text': 'ਇਕੱਠੇ ਕਰਨ ਅਤੇ ਮੱਛੀਆਂ ਫੜ ਕੇ ਬਚੇ ਸਨ', 'span_type': 'Style_Awkward'}], 'language': 'Punjabi', 'split': 'test'}\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"dataset_small = dataset['test'].shuffle(seed=42).select(range(1000))\n\nprint(len(dataset_small))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:44.612036Z","iopub.execute_input":"2026-04-07T13:58:44.612492Z","iopub.status.idle":"2026-04-07T13:58:44.629557Z","shell.execute_reply.started":"2026-04-07T13:58:44.612462Z","shell.execute_reply":"2026-04-07T13:58:44.628913Z"}},"outputs":[{"name":"stdout","text":"1000\n","output_type":"stream"}],"execution_count":6},{"cell_type":"code","source":"print(dataset_small[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:47.726217Z","iopub.execute_input":"2026-04-07T13:58:47.726554Z","iopub.status.idle":"2026-04-07T13:58:47.731521Z","shell.execute_reply.started":"2026-04-07T13:58:47.726527Z","shell.execute_reply":"2026-04-07T13:58:47.730601Z"}},"outputs":[{"name":"stdout","text":"{'src': 'There were no large forests in the land of Canaan, so wood was extremely expensive.', 'ref': 'கானானின் நாட்டில் பெரிய காடுகள் இல்லாததால், மரம் மிகவும் விலை உயர்ந்ததாய் இருந்தது.', 'translation': 'கானான் தேசத்தில் பெரிய காடுகள் இல்லை, எனவே\\xa0மரம் மிகவும் விலை உயர்ந்தது.', 'mqm_norm_score': '0.92', 'da_norm_score': '0.92', 'error_spans': [{'span_end_offset': 9, 'span_no': 0, 'span_severity': 'Medium', 'span_start_offset': 6, 'span_text': 'மரம் மிகவும் விலை உயர்ந்தது', 'span_type': 'Style_Awkward'}], 'language': 'Tamil', 'split': 'test'}\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"sources = []\nreferences = []\npredictions = []\n\nfor example in dataset_small:\n    sources.append(example['src'])\n    references.append(example['ref'])\n    predictions.append(example['translation'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T13:58:55.658018Z","iopub.execute_input":"2026-04-07T13:58:55.658967Z","iopub.status.idle":"2026-04-07T13:58:55.809195Z","shell.execute_reply.started":"2026-04-07T13:58:55.658924Z","shell.execute_reply":"2026-04-07T13:58:55.808604Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"import re\n\ndef preprocess(text):\n    text = text.lower()\n    text = re.sub(r\"\\s+\", \" \", text)\n    text = text.strip()\n    return text\n\nreferences = [preprocess(t) for t in references]\npredictions = [preprocess(t) for t in predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T14:37:50.127974Z","iopub.execute_input":"2026-03-17T14:37:50.128677Z","iopub.status.idle":"2026-03-17T14:37:50.155174Z","shell.execute_reply.started":"2026-03-17T14:37:50.128650Z","shell.execute_reply":"2026-03-17T14:37:50.154319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"references_bleu = [[ref] for ref in references]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:05:03.980389Z","iopub.execute_input":"2026-03-17T15:05:03.980643Z","iopub.status.idle":"2026-03-17T15:05:03.984662Z","shell.execute_reply.started":"2026-03-17T15:05:03.980622Z","shell.execute_reply":"2026-03-17T15:05:03.983869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import evaluate\n\nbleu = evaluate.load(\"bleu\")\n\nbleu_score = bleu.compute(\n    predictions=predictions,\n    references=references_bleu\n)\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:11:36.761270Z","iopub.execute_input":"2026-03-17T15:11:36.761877Z","iopub.status.idle":"2026-03-17T15:11:38.191906Z","shell.execute_reply.started":"2026-03-17T15:11:36.761843Z","shell.execute_reply":"2026-03-17T15:11:38.191301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rouge = evaluate.load(\"rouge\")\n\nrouge_score = rouge.compute(\n    predictions=predictions,\n    references=references\n)\n\nprint(\"ROUGE Scores:\", rouge_score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:11:52.613038Z","iopub.execute_input":"2026-03-17T15:11:52.613531Z","iopub.status.idle":"2026-03-17T15:11:54.143490Z","shell.execute_reply.started":"2026-03-17T15:11:52.613482Z","shell.execute_reply":"2026-03-17T15:11:54.142701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meteor = evaluate.load(\"meteor\")\n\nmeteor_score = meteor.compute(\n    predictions=predictions,\n    references=references\n)\n\nprint(\"METEOR Score:\", meteor_score[\"meteor\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:06.504153Z","iopub.execute_input":"2026-03-17T15:12:06.504704Z","iopub.status.idle":"2026-03-17T15:12:10.535362Z","shell.execute_reply.started":"2026-03-17T15:12:06.504674Z","shell.execute_reply":"2026-03-17T15:12:10.534441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sacrebleu\n\nchrf = sacrebleu.corpus_chrf(predictions, [references])\n\nprint(\"chrF Score:\", chrf.score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:22.774589Z","iopub.execute_input":"2026-03-17T15:12:22.775220Z","iopub.status.idle":"2026-03-17T15:12:23.304853Z","shell.execute_reply.started":"2026-03-17T15:12:22.775178Z","shell.execute_reply":"2026-03-17T15:12:23.304153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== Machine Translation Evaluation =====\")\n\nprint(\"BLEU:\", bleu_score[\"bleu\"])\nprint(\"ROUGE-L:\", rouge_score[\"rougeL\"])\nprint(\"METEOR:\", meteor_score[\"meteor\"])\nprint(\"chrF:\", chrf.score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:35.004597Z","iopub.execute_input":"2026-03-17T15:12:35.005528Z","iopub.status.idle":"2026-03-17T15:12:35.010226Z","shell.execute_reply.started":"2026-03-17T15:12:35.005496Z","shell.execute_reply":"2026-03-17T15:12:35.009305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mqm_scores = [float(x) for x in dataset_small['mqm_norm_score']]\nda_scores = [float(x) for x in dataset_small['da_norm_score']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:12:49.556214Z","iopub.execute_input":"2026-03-17T15:12:49.556903Z","iopub.status.idle":"2026-03-17T15:12:49.652373Z","shell.execute_reply.started":"2026-03-17T15:12:49.556876Z","shell.execute_reply":"2026-03-17T15:12:49.651807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\nmodel = SentenceTransformer('all-MiniLM-L6-v2')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:21.343969Z","iopub.execute_input":"2026-03-17T15:08:21.344737Z","iopub.status.idle":"2026-03-17T15:08:43.052938Z","shell.execute_reply.started":"2026-03-17T15:08:21.344696Z","shell.execute_reply":"2026-03-17T15:08:43.052173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ref_embeddings = model.encode(references, convert_to_tensor=True)\npred_embeddings = model.encode(predictions, convert_to_tensor=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:48.335680Z","iopub.execute_input":"2026-03-17T15:08:48.336207Z","iopub.status.idle":"2026-03-17T15:08:50.372971Z","shell.execute_reply.started":"2026-03-17T15:08:48.336179Z","shell.execute_reply":"2026-03-17T15:08:50.372145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers.util import cos_sim\n\nsimilarities = []\n\nfor i in range(len(ref_embeddings)):\n    sim = cos_sim(ref_embeddings[i], pred_embeddings[i]).item()\n    similarities.append(sim)\n\nprint(\"Average Semantic Similarity:\", sum(similarities)/len(similarities))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:08:59.230754Z","iopub.execute_input":"2026-03-17T15:08:59.231617Z","iopub.status.idle":"2026-03-17T15:08:59.422544Z","shell.execute_reply.started":"2026-03-17T15:08:59.231584Z","shell.execute_reply":"2026-03-17T15:08:59.421815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== Machine Translation Evaluation =====\")\n\nprint(\"BLEU:\", bleu_score[\"bleu\"])\nprint(\"ROUGE-L:\", rouge_score[\"rougeL\"])\nprint(\"METEOR:\", meteor_score[\"meteor\"])\nprint(\"chrF:\", chrf.score)\nprint(\"Embedding Similarity:\", sum(similarities)/len(similarities))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:21:00.491387Z","iopub.execute_input":"2026-03-17T15:21:00.491756Z","iopub.status.idle":"2026-03-17T15:21:00.497062Z","shell.execute_reply.started":"2026-03-17T15:21:00.491728Z","shell.execute_reply":"2026-03-17T15:21:00.496289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ref_embeddings_np = ref_embeddings.cpu().numpy()\npred_embeddings_np = pred_embeddings.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:46:52.611222Z","iopub.execute_input":"2026-03-17T15:46:52.611643Z","iopub.status.idle":"2026-03-17T15:46:52.617725Z","shell.execute_reply.started":"2026-03-17T15:46:52.611601Z","shell.execute_reply":"2026-03-17T15:46:52.616700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nnp.save(\"reference_embeddings.npy\", ref_embeddings_np)\nnp.save(\"prediction_embeddings.npy\", pred_embeddings_np)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:48:34.950880Z","iopub.execute_input":"2026-03-17T15:48:34.951585Z","iopub.status.idle":"2026-03-17T15:48:34.958657Z","shell.execute_reply.started":"2026-03-17T15:48:34.951552Z","shell.execute_reply":"2026-03-17T15:48:34.957827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndf_embeddings = pd.DataFrame({\n    \"reference\": references,\n    \"prediction\": predictions\n})\n\ndf_embeddings[\"ref_embedding\"] = list(ref_embeddings_np)\ndf_embeddings[\"pred_embedding\"] = list(pred_embeddings_np)\n\ndf_embeddings.to_pickle(\"translation_embeddings.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:49:06.119746Z","iopub.execute_input":"2026-03-17T15:49:06.120375Z","iopub.status.idle":"2026-03-17T15:49:06.138649Z","shell.execute_reply.started":"2026-03-17T15:49:06.120343Z","shell.execute_reply":"2026-03-17T15:49:06.137810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_embeddings = pd.read_pickle(\"translation_embeddings.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-17T15:49:15.540370Z","iopub.execute_input":"2026-03-17T15:49:15.541005Z","iopub.status.idle":"2026-03-17T15:49:15.555991Z","shell.execute_reply.started":"2026-03-17T15:49:15.540975Z","shell.execute_reply":"2026-03-17T15:49:15.555048Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"TAMIL TRANSLATION EVALUATION","metadata":{}},{"cell_type":"code","source":"!pip install transformers sentencepiece -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:06:37.338975Z","iopub.execute_input":"2026-04-13T14:06:37.339379Z","iopub.status.idle":"2026-04-13T14:06:42.392603Z","shell.execute_reply.started":"2026-04-13T14:06:37.339350Z","shell.execute_reply":"2026-04-13T14:06:42.391570Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"!pip install indic-transliteration sentencepiece transformers -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:07:45.876532Z","iopub.execute_input":"2026-04-13T14:07:45.877376Z","iopub.status.idle":"2026-04-13T14:07:49.351416Z","shell.execute_reply.started":"2026-04-13T14:07:45.877344Z","shell.execute_reply":"2026-04-13T14:07:49.350291Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"ai4bharat/IndicMTEval\", split=\"test\")\n\nprint(len(dataset))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:07:53.125379Z","iopub.execute_input":"2026-04-13T14:07:53.126064Z","iopub.status.idle":"2026-04-13T14:07:57.439367Z","shell.execute_reply.started":"2026-04-13T14:07:53.126030Z","shell.execute_reply":"2026-04-13T14:07:57.438532Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5da853946ee4e5b8fccb9ef07fa5015"}},"metadata":{}},{"name":"stderr","text":"Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"data/test-00000-of-00001.parquet:   0%|          | 0.00/498k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2fecf6eff9a94e10a935037c4f9e649c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/validation-00000-of-00001.parquet:   0%|          | 0.00/200k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f1bb0c3ffc0d42d0b15cd5d8f5f34daf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"data/train-00000-of-00001.parquet:   0%|          | 0.00/1.13M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c6694142355e4d47a34e4622ec2304f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9acf411cf9ce4f7780545562f7e48871"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/1000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d74da65a6f9468bb816fc0d37bc5778"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/4997 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e11e8b874bb1450fb41136e3092ecf86"}},"metadata":{}},{"name":"stdout","text":"2380\n","output_type":"stream"}],"execution_count":4},{"cell_type":"code","source":"dataset_tamil = dataset.filter(lambda x: x[\"language\"] == \"Tamil\")\n\nprint(\"Tamil samples:\", len(dataset_tamil))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:03.544138Z","iopub.execute_input":"2026-04-13T14:08:03.545149Z","iopub.status.idle":"2026-04-13T14:08:03.664604Z","shell.execute_reply.started":"2026-04-13T14:08:03.545116Z","shell.execute_reply":"2026-04-13T14:08:03.663492Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Filter:   0%|          | 0/2380 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e1ffd433238c4c439b2b7fbc331b87af"}},"metadata":{}},{"name":"stdout","text":"Tamil samples: 276\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"sources = dataset_tamil[\"src\"]\nreferences = dataset_tamil[\"ref\"]\n\nprint(len(sources))\nprint(sources[0])\nprint(references[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:09.070305Z","iopub.execute_input":"2026-04-13T14:08:09.070864Z","iopub.status.idle":"2026-04-13T14:08:09.078276Z","shell.execute_reply.started":"2026-04-13T14:08:09.070837Z","shell.execute_reply":"2026-04-13T14:08:09.077433Z"}},"outputs":[{"name":"stdout","text":"276\nThe U.N. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nஉலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என U.N. நம்புகிறது.\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"import re\n\ndef preprocess(text):\n    text = text.lower()\n    text = re.sub(r\"\\s+\", \" \", text)\n    text = text.strip()\n    return text\n\nsources = [preprocess(t) for t in sources]\nreferences = [preprocess(t) for t in references]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:08:11.715939Z","iopub.execute_input":"2026-04-13T14:08:11.716215Z","iopub.status.idle":"2026-04-13T14:08:11.758457Z","shell.execute_reply.started":"2026-04-13T14:08:11.716195Z","shell.execute_reply":"2026-04-13T14:08:11.757821Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"from transformers import NllbTokenizer, AutoModelForSeq2SeqLM\n\nmodel_name = \"facebook/nllb-200-distilled-600M\"\n\ntokenizer = NllbTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:14:39.280280Z","iopub.execute_input":"2026-04-07T15:14:39.281107Z","iopub.status.idle":"2026-04-07T15:14:43.590255Z","shell.execute_reply.started":"2026-04-07T15:14:39.281072Z","shell.execute_reply":"2026-04-07T15:14:43.589339Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/4.85M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e0dc960e7be842f9b7fb16e37c1e7858"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/512 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae9ab0e7320e419d872e51dce6b95fa3"}},"metadata":{}},{"name":"stderr","text":"The tied weights mapping and config for this model specifies to tie model.shared.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.decoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.encoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"model_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    inputs = tokenizer(\n        batch,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    translated = model.generate(\n        **inputs,\n        forced_bos_token_id=tokenizer.convert_tokens_to_ids(\"tam_Taml\")\n    )\n\n    preds = tokenizer.batch_decode(\n        translated,\n        skip_special_tokens=True\n    )\n\n    model_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(model_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:14:59.578073Z","iopub.execute_input":"2026-04-07T15:14:59.578798Z","iopub.status.idle":"2026-04-07T15:22:43.366031Z","shell.execute_reply.started":"2026-04-07T15:14:59.578763Z","shell.execute_reply":"2026-04-07T15:22:43.365294Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":30},{"cell_type":"code","source":"model_predictions = [preprocess(t) for t in model_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:24:33.711210Z","iopub.execute_input":"2026-04-07T15:24:33.712107Z","iopub.status.idle":"2026-04-07T15:24:33.718749Z","shell.execute_reply.started":"2026-04-07T15:24:33.712073Z","shell.execute_reply":"2026-04-07T15:24:33.717977Z"}},"outputs":[],"execution_count":31},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\n\nprint(\"Evaluating translations...\\n\")\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\nreferences_bleu = [[ref] for ref in references]\n\nbleu_score = bleu.compute(\n    predictions=model_predictions,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    model_predictions,\n    [references]\n)\n\n# BERTScore\nbertscore = evaluate.load(\"bertscore\")\n\nbert_results = bertscore.compute(\n    predictions=model_predictions,\n    references=references,\n    lang=\"ta\"\n)\n\nbert_f1 = np.mean(bert_results[\"f1\"])\n\n# Embedding Similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(references)\npred_embeddings = embed_model.encode(model_predictions)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\n# Print Results\nprint(\"===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:24:37.489780Z","iopub.execute_input":"2026-04-07T15:24:37.490059Z","iopub.status.idle":"2026-04-07T15:24:43.640811Z","shell.execute_reply.started":"2026-04-07T15:24:37.490034Z","shell.execute_reply":"2026-04-07T15:24:43.640136Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"15f199e3f61945c4a5d670c9efdd15db"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9a600ba2467e4d7e8d71edf445a5d26a"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"name":"stdout","text":"===== Evaluation Results =====\n\nBLEU Score: 0.14073740207540267\nchrF Score: 54.626687127913556\nBERTScore F1: 0.8497447468664335\nEmbedding Similarity: 0.8760938\n","output_type":"stream"}],"execution_count":32},{"cell_type":"code","source":"for i in range(5):\n    print(\"SOURCE:\", sources[i])\n    print(\"REFERENCE:\", references[i])\n    print(\"MODEL:\", model_predictions[i])\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-07T15:25:07.899253Z","iopub.execute_input":"2026-04-07T15:25:07.899926Z","iopub.status.idle":"2026-04-07T15:25:07.905307Z","shell.execute_reply.started":"2026-04-07T15:25:07.899892Z","shell.execute_reply":"2026-04-07T15:25:07.904399Z"}},"outputs":[{"name":"stdout","text":"SOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\nSOURCE: the u.n. also hopes tofinalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. எதிர்பார்த்தது.\n\nSOURCE: the u.n. also hopes to finalize a fund to help countries affected by global warming to cope with the impacts.\nREFERENCE: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு இந்த பாதிப்புகளை சமாளிக்க உதவும் வகையில் ஒரு நிதியை முடிவு செய்ய இயலும் என u.n. நம்புகிறது.\nMODEL: உலக வெப்பமயமாதலால் பாதிக்கப்பட்ட நாடுகளுக்கு தாக்கங்களை சமாளிக்க உதவும் ஒரு நிதியை இறுதி செய்ய ஐ. நா. நம்புகிறது.\n\n","output_type":"stream"}],"execution_count":33},{"cell_type":"markdown","source":"google/t5-small","metadata":{}},{"cell_type":"code","source":"from transformers import T5Tokenizer, T5ForConditionalGeneration\n\nmodel_name = \"t5-base\"\n\ntokenizer = T5Tokenizer.from_pretrained(model_name)\nmodel = T5ForConditionalGeneration.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:10:33.616239Z","iopub.execute_input":"2026-04-13T14:10:33.616644Z","iopub.status.idle":"2026-04-13T14:10:37.245330Z","shell.execute_reply.started":"2026-04-13T14:10:33.616617Z","shell.execute_reply":"2026-04-13T14:10:37.244663Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"28e0cc45a4c848439bf35c3e8c84ac87"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fdf032dce8fb47598bd3fc44234be408"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"67719da1b49149888b6f0e5ebd40bf4f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/892M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"753f135b4bfa43c3a9378027a82fa256"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/257 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e276a949ccaa4cabb6525ae7ebd0af9a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3bcd4b4a554449059ce71cc33b4d1ec4"}},"metadata":{}}],"execution_count":11},{"cell_type":"code","source":"t5_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    inputs = tokenizer(\n        [\"translate English to Tamil: \" + s for s in batch],\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    outputs = model.generate(**inputs)\n\n    preds = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n    t5_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(t5_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:11:03.769110Z","iopub.execute_input":"2026-04-13T14:11:03.769856Z","iopub.status.idle":"2026-04-13T14:12:13.783719Z","shell.execute_reply.started":"2026-04-13T14:11:03.769826Z","shell.execute_reply":"2026-04-13T14:12:13.782942Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"t5_predictions = [preprocess(t) for t in t5_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:12:54.730884Z","iopub.execute_input":"2026-04-13T14:12:54.731506Z","iopub.status.idle":"2026-04-13T14:12:54.737104Z","shell.execute_reply.started":"2026-04-13T14:12:54.731478Z","shell.execute_reply":"2026-04-13T14:12:54.736215Z"}},"outputs":[],"execution_count":14},{"cell_type":"code","source":"!pip install bert-score -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:19:33.365552Z","iopub.execute_input":"2026-04-13T14:19:33.366469Z","iopub.status.idle":"2026-04-13T14:19:37.129365Z","shell.execute_reply.started":"2026-04-13T14:19:33.366437Z","shell.execute_reply":"2026-04-13T14:19:37.128499Z"}},"outputs":[],"execution_count":24},{"cell_type":"code","source":"clean_refs = []\nclean_preds = []\n\nfor ref, pred in zip(references, t5_predictions):\n    if ref.strip() != \"\" and pred.strip() != \"\":\n        clean_refs.append(ref)\n        clean_preds.append(pred)\n\nprint(\"Valid samples:\", len(clean_preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:22:07.864837Z","iopub.execute_input":"2026-04-13T14:22:07.865290Z","iopub.status.idle":"2026-04-13T14:22:07.870903Z","shell.execute_reply.started":"2026-04-13T14:22:07.865260Z","shell.execute_reply":"2026-04-13T14:22:07.870257Z"}},"outputs":[{"name":"stdout","text":"Valid samples: 270\n","output_type":"stream"}],"execution_count":27},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\nfrom bert_score import score\n\nprint(\"Evaluating translations...\\n\")\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\n\nreferences_bleu = [[ref] for ref in clean_refs]\n\nbleu_score = bleu.compute(\n    predictions=clean_preds,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    clean_preds,\n    [clean_refs]\n)\n\n# BERTScore\nP, R, F1 = score(\n    clean_preds,\n    clean_refs,\n    model_type=\"bert-base-multilingual-cased\"\n)\n\nbert_f1 = F1.mean().item()\n\n# Embedding similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(clean_refs)\npred_embeddings = embed_model.encode(clean_preds)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\nprint(\"\\n===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:22:26.857643Z","iopub.execute_input":"2026-04-13T14:22:26.858027Z","iopub.status.idle":"2026-04-13T14:22:31.498808Z","shell.execute_reply.started":"2026-04-13T14:22:26.858000Z","shell.execute_reply":"2026-04-13T14:22:31.497308Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fde80442a64741e7b936685019b39dff"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"151a986d6ee640e294739d16b6524a88"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config_sentence_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc3f6bcdcc684951807ecb8b3d28216d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"README.md: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ce53da5026404fc181f10a7187706ebf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7f1c300802464ac2b959a595e888e7a4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/612 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0ce5691c8d5342c28d187f89943d21e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5f078f598b414670a822897714c93cfd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"628353cc60894f0fb1254bcfb9b2fb81"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/350 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a137236ace5948bc86d3a5197426f9a9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.txt: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3dd1e12999cc4e4495e3bbb8ffbfbf0c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"41b0721192b44377b6b742c577c6135a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c4acad0a952d4abd846a859daa7a0328"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4adefd2952fd49519132752f307357c0"}},"metadata":{}},{"name":"stdout","text":"\n===== Evaluation Results =====\n\nBLEU Score: 0.0\nchrF Score: 0.22674024250644748\nBERTScore F1: 0.6173190474510193\nEmbedding Similarity: 0.28766814\n","output_type":"stream"}],"execution_count":28},{"cell_type":"code","source":"from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer\n\nmodel_name = \"facebook/m2m100_418M\"\n\ntokenizer = M2M100Tokenizer.from_pretrained(model_name)\nmodel = M2M100ForConditionalGeneration.from_pretrained(model_name)\n\nprint(\"M2M100 model loaded\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:05.783392Z","iopub.execute_input":"2026-04-13T14:24:05.784332Z","iopub.status.idle":"2026-04-13T14:24:15.849381Z","shell.execute_reply.started":"2026-04-13T14:24:05.784298Z","shell.execute_reply":"2026-04-13T14:24:15.845985Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/298 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"43521045de01425f909925bf58b0545b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"vocab.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b6aa3310b13c428b84fa8f48d7c1e8ef"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sentencepiece.bpe.model:   0%|          | 0.00/2.42M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73649809637a44e58a3ba9b4f407901b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json: 0.00B [00:00, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1751081589d2446b9ab36e0b1dce21bb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/908 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fedb14d303924d57917cc93e20dcf6be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"pytorch_model.bin:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"048ec68d4d5e4cc08d9f72685612beaf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/1.94G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a627b041dcfd45f9bd0a506172a001b5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/512 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc7b0e81139846ca9921622f1d138e13"}},"metadata":{}},{"name":"stderr","text":"The tied weights mapping and config for this model specifies to tie model.shared.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.decoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\nThe tied weights mapping and config for this model specifies to tie model.shared.weight to model.encoder.embed_tokens.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/233 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cfa8a6b0a6b4407db3b2a01058a268dd"}},"metadata":{}},{"name":"stdout","text":"M2M100 model loaded\n","output_type":"stream"}],"execution_count":29},{"cell_type":"code","source":"tokenizer.src_lang = \"en\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:27.946318Z","iopub.execute_input":"2026-04-13T14:24:27.947157Z","iopub.status.idle":"2026-04-13T14:24:28.764842Z","shell.execute_reply.started":"2026-04-13T14:24:27.947130Z","shell.execute_reply":"2026-04-13T14:24:28.763753Z"}},"outputs":[],"execution_count":31},{"cell_type":"code","source":"m2m_predictions = []\n\nbatch_size = 8\n\nfor i in range(0, len(sources), batch_size):\n\n    batch = sources[i:i+batch_size]\n\n    encoded = tokenizer(\n        batch,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True\n    )\n\n    generated_tokens = model.generate(\n        **encoded,\n        forced_bos_token_id=tokenizer.get_lang_id(\"ta\")\n    )\n\n    preds = tokenizer.batch_decode(\n        generated_tokens,\n        skip_special_tokens=True\n    )\n\n    m2m_predictions.extend(preds)\n\nprint(\"Total predictions:\", len(m2m_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:24:39.242373Z","iopub.execute_input":"2026-04-13T14:24:39.243033Z","iopub.status.idle":"2026-04-13T14:38:05.293922Z","shell.execute_reply.started":"2026-04-13T14:24:39.243004Z","shell.execute_reply":"2026-04-13T14:38:05.292653Z"}},"outputs":[{"name":"stdout","text":"Total predictions: 276\n","output_type":"stream"}],"execution_count":32},{"cell_type":"code","source":"m2m_predictions = [preprocess(t) for t in m2m_predictions]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:50:58.063734Z","iopub.execute_input":"2026-04-13T14:50:58.064586Z","iopub.status.idle":"2026-04-13T14:50:58.070956Z","shell.execute_reply.started":"2026-04-13T14:50:58.064557Z","shell.execute_reply":"2026-04-13T14:50:58.070064Z"}},"outputs":[],"execution_count":33},{"cell_type":"code","source":"import evaluate\nimport sacrebleu\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\nfrom bert_score import score\n\nprint(\"Evaluating translations...\\n\")\n\n# CHANGE THIS depending on model\npredictions = m2m_predictions\n\n# Remove empty sentences\nclean_refs = []\nclean_preds = []\n\nfor ref, pred in zip(references, predictions):\n    if ref.strip() != \"\" and pred.strip() != \"\":\n        clean_refs.append(ref)\n        clean_preds.append(pred)\n\nprint(\"Valid samples:\", len(clean_preds))\n\n# BLEU\nbleu = evaluate.load(\"bleu\")\n\nreferences_bleu = [[ref] for ref in clean_refs]\n\nbleu_score = bleu.compute(\n    predictions=clean_preds,\n    references=references_bleu\n)\n\n# chrF\nchrf = sacrebleu.corpus_chrf(\n    clean_preds,\n    [clean_refs]\n)\n\n# BERTScore\nP, R, F1 = score(\n    clean_preds,\n    clean_refs,\n    model_type=\"bert-base-multilingual-cased\"\n)\n\nbert_f1 = F1.mean().item()\n\n# Embedding Similarity\nembed_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\nref_embeddings = embed_model.encode(clean_refs)\npred_embeddings = embed_model.encode(clean_preds)\n\nsimilarities = cosine_similarity(\n    ref_embeddings,\n    pred_embeddings\n).diagonal()\n\nembedding_similarity = np.mean(similarities)\n\n# Print Results\nprint(\"\\n===== Evaluation Results =====\\n\")\n\nprint(\"BLEU Score:\", bleu_score[\"bleu\"])\nprint(\"chrF Score:\", chrf.score)\nprint(\"BERTScore F1:\", bert_f1)\nprint(\"Embedding Similarity:\", embedding_similarity)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-13T14:51:00.596479Z","iopub.execute_input":"2026-04-13T14:51:00.597264Z","iopub.status.idle":"2026-04-13T14:51:04.850673Z","shell.execute_reply.started":"2026-04-13T14:51:00.597236Z","shell.execute_reply":"2026-04-13T14:51:04.849305Z"}},"outputs":[{"name":"stdout","text":"Evaluating translations...\n\nValid samples: 276\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4328a08f769745fdaea8c59818a2b7cb"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: bert-base-multilingual-cased\nKey                                        | Status     |  | \n-------------------------------------------+------------+--+-\ncls.predictions.transform.dense.weight     | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.bias   | UNEXPECTED |  | \ncls.predictions.transform.LayerNorm.weight | UNEXPECTED |  | \ncls.seq_relationship.weight                | UNEXPECTED |  | \ncls.predictions.bias                       | UNEXPECTED |  | \ncls.seq_relationship.bias                  | UNEXPECTED |  | \ncls.predictions.transform.dense.bias       | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a316a81f0dbb4124afd1ae9c72870c2c"}},"metadata":{}},{"name":"stderr","text":"\u001b[1mBertModel LOAD REPORT\u001b[0m from: sentence-transformers/all-MiniLM-L6-v2\nKey                     | Status     |  | \n------------------------+------------+--+-\nembeddings.position_ids | UNEXPECTED |  | \n\n\u001b[3mNotes:\n- UNEXPECTED\u001b[3m\t:can be ignored when loading from different task/architecture; not ok if you expect identical arch.\u001b[0m\n","output_type":"stream"},{"name":"stdout","text":"\n===== Evaluation Results =====\n\nBLEU Score: 0.008726440899402313\nchrF Score: 17.461341772346902\nBERTScore F1: 0.6680243611335754\nEmbedding Similarity: 0.5583976\n","output_type":"stream"}],"execution_count":34}]}