⠏ Waiting for authorization\n\n"
},
"metadata": {}
}
]
}
},
"6e115a3de875472e8dcd1537fdc745bc": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
}
}
}
},
"cells": [
{
"cell_type": "code",
"source": [
"from nltk.corpus import stopwords\n",
"from nltk.tokenize import word_tokenize\n",
"import nltk\n",
"nltk.download('punkt_tab')\n",
"nltk.download('stopwords')\n",
"def clean_text(text: str, lang: str = 'english') -> str:\n",
" # lowercase\n",
" text = text.lower()\n",
"\n",
" # tokenize\n",
" tokens = word_tokenize(text)\n",
"\n",
" # get stopword list for language\n",
" stop_words = set(stopwords.words(lang))\n",
"\n",
" # filter stopwords\n",
" tokens = [t for t in tokens if t not in stop_words]\n",
"\n",
" # join back to string\n",
" return \" \".join(tokens)\n"
],
"metadata": {
"id": "_mVEG6R3rSD8",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "9ac3c2ca-43f5-40be-fdbe-09aa7095db56"
},
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"[nltk_data] Downloading package punkt_tab to /root/nltk_data...\n",
"[nltk_data] Package punkt_tab is already up-to-date!\n",
"[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
"[nltk_data] Package stopwords is already up-to-date!\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import torch\n",
"device='cuda' if torch.cuda.is_available() else 'cpu'\n",
"device"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "B3e_bL5XoO6k",
"outputId": "ce96b0ad-34e9-40d8-fd00-7ff3d6147584"
},
"execution_count": 2,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'cuda'"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 2
}
]
},
{
"cell_type": "code",
"source": [
"import pandas as pd\n",
"\n",
"splits = {'train': 'data/train-00000-of-00001.parquet', 'validation': 'data/validation-00000-of-00001.parquet', 'test': 'data/test-00000-of-00001.parquet'}\n",
"df = pd.read_parquet(\"hf://datasets/cfilt/iitb-english-hindi/\" + splits[\"train\"])"
],
"metadata": {
"id": "zQ5vddsq6wSc",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "c4f9f023-5b33-4e16-86f1-156ccf598670"
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
"The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
"To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
"You will be able to reuse this secret in all of your notebooks.\n",
"Please note that authentication is recommended but still optional to access public models or datasets.\n",
" warnings.warn(\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"len(df)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Vm5bkx9aSdYp",
"outputId": "8c5b1eb4-736d-46d5-d0ab-321d1d6e95af"
},
"execution_count": 4,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"1659083"
]
},
"metadata": {},
"execution_count": 4
}
]
},
{
"cell_type": "code",
"source": [
"# df=df[:1000]"
],
"metadata": {
"id": "1nZqbdb7CYIj"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 423
},
"id": "SG0S31uYAPdm",
"outputId": "7c207893-bbae-4c6b-b4b8-1130699b648e"
},
"execution_count": 6,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" translation\n",
"0 {'en': 'Give your application an accessibility...\n",
"1 {'en': 'Accerciser Accessibility Explorer', 'h...\n",
"2 {'en': 'The default plugin layout for the bott...\n",
"3 {'en': 'The default plugin layout for the top ...\n",
"4 {'en': 'A list of plugins that are disabled by...\n",
"... ...\n",
"1659078 {'en': 'The Prime Minister, Shri Narendra Modi...\n",
"1659079 {'en': 'In a tweet, the Prime Minister said, c...\n",
"1659080 {'en': 'I also congratulate all those who took...\n",
"1659081 {'en': 'The NDA family will work together for ...\n",
"1659082 {'en': 'I assure all possible support from the...\n",
"\n",
"[1659083 rows x 1 columns]"
],
"text/html": [
"\n",
" | \n", " | translation | \n", "
|---|---|
| 0 | \n", "{'en': 'Give your application an accessibility... | \n", "
| 1 | \n", "{'en': 'Accerciser Accessibility Explorer', 'h... | \n", "
| 2 | \n", "{'en': 'The default plugin layout for the bott... | \n", "
| 3 | \n", "{'en': 'The default plugin layout for the top ... | \n", "
| 4 | \n", "{'en': 'A list of plugins that are disabled by... | \n", "
| ... | \n", "... | \n", "
| 1659078 | \n", "{'en': 'The Prime Minister, Shri Narendra Modi... | \n", "
| 1659079 | \n", "{'en': 'In a tweet, the Prime Minister said, c... | \n", "
| 1659080 | \n", "{'en': 'I also congratulate all those who took... | \n", "
| 1659081 | \n", "{'en': 'The NDA family will work together for ... | \n", "
| 1659082 | \n", "{'en': 'I assure all possible support from the... | \n", "
1659083 rows × 1 columns
\n", "| \n", " | English | \n", "Hindi | \n", "
|---|---|---|
| 0 | \n", "Give your application an accessibility workout | \n", "अपने अनुप्रयोग को पहुंचनीयता व्यायाम का लाभ दें | \n", "
| 1 | \n", "Accerciser Accessibility Explorer | \n", "एक्सेर्साइसर पहुंचनीयता अन्वेषक | \n", "
| 2 | \n", "The default plugin layout for the bottom panel | \n", "निचले पटल के लिए डिफोल्ट प्लग-इन खाका | \n", "
| 3 | \n", "The default plugin layout for the top panel | \n", "ऊपरी पटल के लिए डिफोल्ट प्लग-इन खाका | \n", "
| 4 | \n", "A list of plugins that are disabled by default | \n", "उन प्लग-इनों की सूची जिन्हें डिफोल्ट रूप से नि... | \n", "
| ... | \n", "... | \n", "... | \n", "
| 1659078 | \n", "The Prime Minister, Shri Narendra Modi has con... | \n", "प्रधानमंत्री श्री नरेन्द्र मोदी ने बिहार के मु... | \n", "
| 1659079 | \n", "In a tweet, the Prime Minister said, congratul... | \n", "एक ट्वीट में प्रधानमंत्री ने कहा, बिहार के मुख... | \n", "
| 1659080 | \n", "I also congratulate all those who took oath as... | \n", "मैं उन सभी को भी बधाई देता हूं, जिन्होंने बिहा... | \n", "
| 1659081 | \n", "The NDA family will work together for the prog... | \n", "एनडीए परिवार बिहार की प्रगति के लिए साथ मिलकर ... | \n", "
| 1659082 | \n", "I assure all possible support from the Centre ... | \n", "बिहार के कल्याण के लिए केंद्र की ओर से हरसंभव ... | \n", "
1659083 rows × 2 columns
\n", " ❗❗❗ AUTHORIZATION REQUIRED ❗❗❗ \n",
"\n"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Output()"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "4ec604360f5f489ebc058331b7836e20"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"\n",
"Open the following link in your browser to authorize the client:\n",
"https://dagshub.com/login/oauth/authorize?state=7122499f-4a30-48f5-bb81-cebd5dbeb6eb&client_id=32b60ba385aa7cecf24046d8195a71c07dd345d9657977863b52e7748e0f0f28&middleman_request_id=d69209eee37bc01e33d0d52592ae719b884f0b18c66d6e90dc33f13164ded3ba\n",
"\n",
"\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [],
"text/html": [
"\n"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Accessing as vanshsharma7832\n"
],
"text/html": [
"Accessing as vanshsharma7832\n",
"\n"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Initialized MLflow to track repo \u001b[32m\"vanshsharma7832/Sentence-Translator\"\u001b[0m\n"
],
"text/html": [
"Initialized MLflow to track repo \"vanshsharma7832/Sentence-Translator\"\n",
"\n"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Repository vanshsharma7832/Sentence-Translator initialized!\n"
],
"text/html": [
"Repository vanshsharma7832/Sentence-Translator initialized!\n",
"\n"
]
},
"metadata": {}
}
]
},
{
"cell_type": "code",
"source": [
"import mlflow\n",
"with mlflow.start_run():\n",
" mlflow.log_metric('loss', 0.0644)\n",
" # mlflow.log_artifact('model.pth',model)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "awYYBxaP-dBD",
"outputId": "2e2d8477-5e80-4603-b2bc-1bf31781b681"
},
"execution_count": 42,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"🏃 View run enthused-donkey-213 at: https://dagshub.com/vanshsharma7832/Sentence-Translator.mlflow/#/experiments/0/runs/6f6da30dc49d45ae97b252a9788c132c\n",
"🧪 View experiment at: https://dagshub.com/vanshsharma7832/Sentence-Translator.mlflow/#/experiments/0\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "IZz_j4Zw-4Bd",
"outputId": "a26b9105-c6df-45c1-edb4-cd42ff014f36"
},
"execution_count": 41,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"8.11653953244604"
]
},
"metadata": {},
"execution_count": 41
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "2-cR3hdb_GbW"
},
"execution_count": null,
"outputs": []
}
]
}