Delete lang_translator.ipynb

lang_translator.ipynb

@@ -1,205 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "21a4341f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import tqdm as notebook_tqdm"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "38dcf44c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
-    "import tqdm as notebook_tqdm\n",
-    "import torch\n",
-    "# Path where you want to store/cache the model\n",
-    "cache_path = \"./\"\n",
-    "\n",
-    "# device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "model_name = \"facebook/nllb-200-distilled-600M\"\n",
-    "# tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=cache_path)\n",
-    "# model = AutoModelForSeq2SeqLM.from_pretrained(model_name, cache_dir=cache_path).to(device)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "b0860374",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import BitsAndBytesConfig\n",
-    "# Quantization config (8-bit)\n",
-    "bnb_config = BitsAndBytesConfig(\n",
-    "    load_in_8bit=True,\n",
-    "    llm_int8_threshold=6.0\n",
-    ")\n",
-    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "# Load tokenizer\n",
-    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
-    "\n",
-    "# Load quantized model\n",
-    "model = AutoModelForSeq2SeqLM.from_pretrained(\n",
-    "    model_name,\n",
-    "    device_map=\"auto\",\n",
-    "    quantization_config=bnb_config\n",
-    ")\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "3a38e931",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/blue/parisa.rashidi/rohanbagulwar/hackathon/.venv/lib/python3.13/site-packages/transformers/modeling_utils.py:4037: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 200}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n",
-      "  warnings.warn(\n"
-     ]
-    }
-   ],
-   "source": [
-    "SAVE_DIR = \"./nllb-600M-quantized\" \n",
-    "tokenizer.save_pretrained(SAVE_DIR)\n",
-    "model.save_pretrained(SAVE_DIR)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6dac6669",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "नमस्कार, कसे आहात तुम्ही कुठे आहात तुम्ही कुठे जात आहात?\n"
-     ]
-    }
-   ],
-   "source": [
-    "punct_normalizer = MosesPunctNormalizer(lang=\"en\")\n",
-    "\n",
-    "def translate(text: str, src_lang: str, tgt_lang: str):\n",
-    "    src_code = code_mapping[src_lang]   # e.g. \"English\" -> \"eng_Latn\"\n",
-    "    tgt_code = code_mapping[tgt_lang]   # e.g. \"Hindi\"   -> \"hin_Deva\"\n",
-    "\n",
-    "    tokenizer.src_lang = src_code\n",
-    "    tokenizer.tgt_lang = tgt_code\n",
-    "\n",
-    "    # Normalize punctuation\n",
-    "    text = punct_normalizer.normalize(text)\n",
-    "\n",
-    "    # Encode & generate\n",
-    "    inputs = tokenizer(text, return_tensors=\"pt\").to(device)\n",
-    "    outputs = model.generate(\n",
-    "        **inputs,\n",
-    "        forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_code),  # ✅ use FLORES code\n",
-    "        max_length=256,\n",
-    "        num_beams=5,\n",
-    "        no_repeat_ngram_size=4,\n",
-    "    )\n",
-    "    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
-    "\n",
-    "\n",
-    "# Example usage\n",
-    "print(translate(\"Hello, how are you where are you wher are you going?\", \"English\", \"Marathi\"))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8dd925b3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!uv pip install -U bitsandbytes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "648597dd",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from flores import code_mapping\n",
-    "from sacremoses import MosesPunctNormalizer\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "d1f14e91",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "नमस्कार, कसे आहात तुम्ही कुठे आहात तुम्ही कुठे जात आहात?\n"
-     ]
-    }
-   ],
-   "source": [
-    "punct_normalizer = MosesPunctNormalizer(lang=\"en\")\n",
-    "\n",
-    "def translate(text: str, src_lang: str, tgt_lang: str):\n",
-    "    src_code = code_mapping[src_lang]   # e.g. \"English\" -> \"eng_Latn\"\n",
-    "    tgt_code = code_mapping[tgt_lang]   # e.g. \"Hindi\"   -> \"hin_Deva\"\n",
-    "\n",
-    "    tokenizer.src_lang = src_code\n",
-    "    tokenizer.tgt_lang = tgt_code\n",
-    "\n",
-    "    # Normalize punctuation\n",
-    "    text = punct_normalizer.normalize(text)\n",
-    "\n",
-    "    # Encode & generate\n",
-    "    inputs = tokenizer(text, return_tensors=\"pt\").to(device)\n",
-    "    outputs = model.generate(\n",
-    "        **inputs,\n",
-    "        forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_code),  # ✅ use FLORES code\n",
-    "        max_length=256,\n",
-    "        num_beams=5,\n",
-    "        no_repeat_ngram_size=4,\n",
-    "    )\n",
-    "    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
-    "\n",
-    "\n",
-    "# Example usage\n",
-    "print(translate(\"Hello, how are you where are you wher are you going?\", \"English\", \"Marathi\"))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "544890af",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "hackathon",
-   "language": "python",
-   "name": "hackathon"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}