{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🚀 High-Speed Multilingual GPU Indexing (All 15 Indic Languages + English)\n",
    "This notebook indexes the full MSMARCO-XI corpora for all supported languages using **PyTorch FP16 on GPU**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Verify GPU is active\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Clone repo and pull full raw corpora with Git LFS\n",
    "import os\n",
    "!git lfs install\n",
    "if not os.path.exists('/content/RAGINGOA'):\n",
    "    !git clone https://github.com/Anshsurana123/RAGINGOA.git /content/RAGINGOA\n",
    "%cd /content/RAGINGOA\n",
    "!git pull origin main\n",
    "!git lfs pull\n",
    "\n",
    "# Verify raw language data\n",
    "!ls -lh data/raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Install GPU requirements\n",
    "!pip install -q faiss-gpu sentence-transformers onnxruntime pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4A. Index standard 3 languages (English, Hindi, Marathi) ~60s\n",
    "!python retrieval/build_indexes_gpu.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4B. (Optional) Index ALL 15 Indic languages + English ~5-8 mins on T4 GPU\n",
    "# Supported: as, bn, en, gu, hi, kn, ml, mr, ne, or, pa, sa, ta, te, ur\n",
    "# !python retrieval/index_all_languages_gpu.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5. Check index directory size\n",
    "!ls -lh data/indexes/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Zip and Download full index archive\n",
    "!zip -r indexed_data.zip data/indexes/\n",
    "!ls -lh indexed_data.zip\n",
    "\n",
    "from google.colab import files\n",
    "files.download('indexed_data.zip')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}