Add kenLM notebooks

.gitattributes

@@ -1,5 +1,6 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
+text.txt filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 *.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text

src/Create_LM.ipynb

@@ -0,0 +1,362 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5e445ce4-1507-482d-a2a8-03d8802e6311",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1c1820bc-0125-4589-983f-e454801435a5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "117e880c8ae8437e9a16ccdf20b659eb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/1.68k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration chmanoj--ai4bharat__samanantar_processed_te-a0473fa2e2573d48\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset samanantar/te (download: 292.93 MiB, generated: 678.62 MiB, post-processed: Unknown size, total: 971.55 MiB) to /home/manoj/.cache/huggingface/datasets/parquet/chmanoj--ai4bharat__samanantar_processed_te-a0473fa2e2573d48/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bd1bfffa9a424a45b3b7324458818f4a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "22a24004a7a546ea88bf7c3fe1c16e46",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/151M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9e4a161541734dfbb2de2d3dd46d8753",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/156M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "992db97134c94b9284b421c7f3ea0b33",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset parquet downloaded and prepared to /home/manoj/.cache/huggingface/datasets/parquet/chmanoj--ai4bharat__samanantar_processed_te-a0473fa2e2573d48/0.0.0/1638526fd0e8d960534e2155dc54fdff8dce73851f21f031d2fb9c2cf757c121. Subsequent calls will reuse this data.\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(f\"chmanoj/ai4bharat__samanantar_processed_te\", split=\"train\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "62fb01f7-24fe-4384-9940-3c262c321a5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"text.txt\", \"w\") as file:\n",
+    "  file.write(\" \".join(dataset[\"text\"]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4295ab4b-b4d8-4a39-a896-fb86503e4674",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "fcc0b573-516a-45d6-af2a-feace521c16d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/mnt/c/Projects/Speech/xls-R-finetuning/lm_te'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "e1f8f887-6201-4ae0-989e-8bdc57816db1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /mnt/c/Projects/Speech/xls-R-finetuning/lm_te/text.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Unigram tokens 32852369 types 1308846\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:15706152 2:2291295744 3:4296179712\n",
+      "Statistics:\n",
+      "1 1308845 D1=0.726852 D2=1.02775 D3+=1.30996\n",
+      "2 12720239 D1=0.818931 D2=1.12897 D3+=1.32699\n",
+      "3 23789023 D1=0.823705 D2=1.50814 D3+=1.24837\n",
+      "Memory estimate for binary LM:\n",
+      "type     MB\n",
+      "probing 731 assuming -p 1.5\n",
+      "probing 809 assuming -r models -p 1.5\n",
+      "trie    342 without quantization\n",
+      "trie    206 assuming -q 8 -b 8 quantization \n",
+      "trie    316 assuming -a 22 array pointer compression\n",
+      "trie    180 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:15706140 2:203523824 3:475780460\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:15706140 2:203523824 3:475780460\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Name:lmplz\tVmPeak:6613460 kB\tVmRSS:37976 kB\tRSSMax:1975488 kB\tuser:33.1964\tsys:9.29228\tCPU:42.4891\treal:65.5831\n"
+     ]
+    }
+   ],
+   "source": [
+    "!../kenlm/build/bin/lmplz -o 3 <\"text.txt\" > \"3gram.arpa\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afee7f94-f247-4891-822e-1f4edd5abc81",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== 1/5 Counting and sorting n-grams ===\n",
+      "Reading /mnt/c/Projects/Speech/xls-R-finetuning/lm_te/text.txt\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Unigram tokens 32852369 types 1308846\n",
+      "=== 2/5 Calculating and sorting adjusted counts ===\n",
+      "Chain sizes: 1:15706152 2:642680448 3:1205025920 4:1928041344 5:2811727104\n",
+      "Statistics:\n",
+      "1 1308845 D1=0.726852 D2=1.02775 D3+=1.30996\n",
+      "2 12720239 D1=0.818931 D2=1.12897 D3+=1.32699\n",
+      "3 23789023 D1=0.910002 D2=1.27136 D3+=1.38596\n",
+      "4 28332665 D1=0.955371 D2=1.42566 D3+=1.4677\n",
+      "5 30063763 D1=0.898851 D2=1.71714 D3+=1.29889\n",
+      "Memory estimate for binary LM:\n",
+      "type      MB\n",
+      "probing 2032 assuming -p 1.5\n",
+      "probing 2408 assuming -r models -p 1.5\n",
+      "trie    1058 without quantization\n",
+      "trie     613 assuming -q 8 -b 8 quantization \n",
+      "trie     921 assuming -a 22 array pointer compression\n",
+      "trie     476 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
+      "=== 3/5 Calculating and sorting initial probabilities ===\n",
+      "Chain sizes: 1:15706140 2:203523824 3:475780460 4:679983960 5:841785364\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
+      "Chain sizes: 1:15706140 2:203523824 3:475780460 4:679983960 5:841785364\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "####################################################################################################\n",
+      "=== 5/5 Writing ARPA model ===\n",
+      "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
+      "****************************************************************************************************\n",
+      "Name:lmplz\tVmPeak:6620664 kB\tVmRSS:38084 kB\tRSSMax:2239444 kB\tuser:77.3579\tsys:28.8403\tCPU:106.198\treal:159.405\n"
+     ]
+    }
+   ],
+   "source": [
+    "!../kenlm/build/bin/lmplz -o 5 <\"text.txt\" > \"5gram.arpa\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d4f8526-fb6a-40cc-bf02-75c78b4138cd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "33e3c247-1b4b-4e61-a42e-283bef351c4b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 22.7 s, sys: 6.28 s, total: 28.9 s\n",
+      "Wall time: 1min 29s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "with open(\"3gram.arpa\", \"r\") as read_file, open(\"3gram_correct.arpa\", \"w\") as write_file:\n",
+    "  has_added_eos = False\n",
+    "  for line in read_file:\n",
+    "    if not has_added_eos and \"ngram 1=\" in line:\n",
+    "      count=line.strip().split(\"=\")[-1]\n",
+    "      write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
+    "    elif not has_added_eos and \"<s>\" in line:\n",
+    "      write_file.write(line)\n",
+    "      write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
+    "      has_added_eos = True\n",
+    "    else:\n",
+    "      write_file.write(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "0f8ead29-e478-48dd-ace5-46d787d3d68e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1min 25s, sys: 27.2 s, total: 1min 52s\n",
+      "Wall time: 5min 28s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "with open(\"5gram.arpa\", \"r\") as read_file, open(\"5gram_correct.arpa\", \"w\") as write_file:\n",
+    "  has_added_eos = False\n",
+    "  for line in read_file:\n",
+    "    if not has_added_eos and \"ngram 1=\" in line:\n",
+    "      count=line.strip().split(\"=\")[-1]\n",
+    "      write_file.write(line.replace(f\"{count}\", f\"{int(count)+1}\"))\n",
+    "    elif not has_added_eos and \"<s>\" in line:\n",
+    "      write_file.write(line)\n",
+    "      write_file.write(line.replace(\"<s>\", \"</s>\"))\n",
+    "      has_added_eos = True\n",
+    "    else:\n",
+    "      write_file.write(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad4ea204-d61c-4316-bc30-5bbda696d225",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "152fecfe-9a51-4f6d-9640-c810adb5e456",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/Create_dataset_te.ipynb

@@ -0,0 +1,263 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3a55acf6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/workspace/xls-r-300m-te'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "8491f5f9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "fed9879a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dc35d55b7a9444128bb348a38969453f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/3.92k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset samanantar/te to /workspace/.cache/huggingface/datasets/ai4bharat___samanantar/te/0.3.0/556308f80c011cb3c32f3de18199d7b1e4cf9ca707843c92bb0bede0e47a8bd6...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de6defc6eb934d87ab8a18cd4fe2a04d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/4.60G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0 examples [00:00, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset samanantar downloaded and prepared to /workspace/.cache/huggingface/datasets/ai4bharat___samanantar/te/0.3.0/556308f80c011cb3c32f3de18199d7b1e4cf9ca707843c92bb0bede0e47a8bd6. Subsequent calls will reuse this data.\n"
+     ]
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(\"ai4bharat/samanantar\", \"te\", split=\"train\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "5c478941",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chars_to_ignore_regex = '[,?.!\\-\\;\\:\"“%‘”�—’…–]'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "abf69ac9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "\n",
+    "def extract_text(batch):\n",
+    "  text = batch[\"tgt\"]\n",
+    "  batch[\"text\"] = re.sub(chars_to_ignore_regex, \"\", text.lower())\n",
+    "  return batch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "6b4d0c6c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'వర్షాలకు చేతికి వచ్చిన పంట దెబ్బతిన్నదని రైతులు వాపోతున్నారు'"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset[0]['tgt']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "710de6ce",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cc51f1d8191c4118b9281727e6ec4b63",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/4661986 [00:00<?, ?ex/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = dataset.map(extract_text, remove_columns=dataset.column_names)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "bd4c05b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a0a50384591d42489963b8990624ab95",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center>\\n<img src=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import notebook_login\n",
+    "\n",
+    "notebook_login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "791becc3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "02abddfd320f404ba7970f6208f9cc27",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pushing dataset shards to the dataset hub:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset.push_to_hub(f\"ai4bharat__samanantar_processed_te\", split=\"train\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2d34464c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2d8308be",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576b4cccf2cd0a29d989ba3823293e051ded9cc2dd8b70356923a3557a691bb1
+size 697581014