{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "a5d77a87",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Old keys example: ['head.weight', 'head.bias']\n",
"New keys example: ['lm_head.weight', 'lm_head.bias']\n"
]
}
],
"source": [
"import torch\n",
"\n",
"ckpt_path = r\"C:\\Users\\Dinesh\\Downloads\\temple_model_bundle_20260201 2\\temple_model_bundle_20260201\\tamil_clm_15m_instruction.pt\"\n",
"\n",
"state = torch.load(ckpt_path, map_location=\"cpu\")\n",
"\n",
"# --- Rename head.* → lm_head.* ---\n",
"new_state = {}\n",
"\n",
"for k, v in state.items():\n",
" if k.startswith(\"head.\"):\n",
" new_state[k.replace(\"head.\", \"lm_head.\")] = v\n",
" else:\n",
" new_state[k] = v\n",
"\n",
"# Optional sanity check\n",
"print(\"Old keys example:\", [k for k in state.keys() if k.startswith(\"head.\")][:2])\n",
"print(\"New keys example:\", [k for k in new_state.keys() if k.startswith(\"lm_head.\")][:2])\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3dd294ef",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Model loaded with exact weights (strict)\n"
]
}
],
"source": [
"import torch\n",
"from tamil_clm_hf import TamilCLMConfig, TamilCLMForCausalLM\n",
"\n",
"DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
"\n",
"config = TamilCLMConfig()\n",
"model = TamilCLMForCausalLM(config).to(DEVICE)\n",
"\n",
"model.load_state_dict(new_state, strict=True)\n",
"model.eval()\n",
"\n",
"print(\"✅ Model loaded with exact weights (strict)\")\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "0fa1e1d9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"SentencePiece vocab size: 8000\n",
"✅ SentencePiece tokenizer is correct\n"
]
}
],
"source": [
"import sentencepiece as spm\n",
"\n",
"SP_MODEL = r\"C:\\Users\\Dinesh\\Downloads\\temple_model_bundle_20260201 2\\temple_model_bundle_20260201\\tamil_clm.model\"\n",
"\n",
"sp = spm.SentencePieceProcessor()\n",
"sp.load(SP_MODEL)\n",
"\n",
"print(\"SentencePiece vocab size:\", sp.get_piece_size())\n",
"assert sp.get_piece_size() == 8000\n",
"print(\"✅ SentencePiece tokenizer is correct\")\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "ef2cc03e",
"metadata": {},
"outputs": [],
"source": [
"from transformers import PreTrainedTokenizer\n",
"\n",
"class TamilSentencePieceTokenizer(PreTrainedTokenizer):\n",
" model_input_names = [\"input_ids\", \"attention_mask\"]\n",
"\n",
" def __init__(self, sp_model_path, **kwargs):\n",
" self.sp = spm.SentencePieceProcessor()\n",
" self.sp.load(sp_model_path)\n",
" super().__init__(**kwargs)\n",
"\n",
" @property\n",
" def vocab_size(self):\n",
" return self.sp.get_piece_size()\n",
"\n",
" def _tokenize(self, text):\n",
" return self.sp.encode(text, out_type=str)\n",
"\n",
" def _convert_token_to_id(self, token):\n",
" return self.sp.piece_to_id(token)\n",
"\n",
" def _convert_id_to_token(self, index):\n",
" return self.sp.id_to_piece(index)\n",
"\n",
" def get_vocab(self):\n",
" return {self.sp.id_to_piece(i): i for i in range(self.vocab_size)}\n",
"\n",
" def save_vocabulary(self, save_directory, filename_prefix=None):\n",
" return ()\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "048dda1d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tokenizer vocab size: 8000\n",
"✅ Slow tokenizer ready\n"
]
}
],
"source": [
"slow_tokenizer = TamilSentencePieceTokenizer(\n",
" SP_MODEL,\n",
" bos_token=\"\",\n",
" eos_token=\"\",\n",
" unk_token=\"\",\n",
" pad_token=\"\"\n",
")\n",
"\n",
"print(\"Tokenizer vocab size:\", slow_tokenizer.vocab_size)\n",
"assert slow_tokenizer.vocab_size == 8000\n",
"print(\"✅ Slow tokenizer ready\")\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "3048571c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Slow tokenizer saved to HF directory\n"
]
}
],
"source": [
"HF_DIR = r\"C:\\Users\\Dinesh\\hf_tamil_clm_15m\"\n",
"\n",
"slow_tokenizer.save_pretrained(HF_DIR)\n",
"print(\"✅ Slow tokenizer saved to HF directory\")\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "c9eba604",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"protobuf + sentencepiece OK\n"
]
}
],
"source": [
"import sentencepiece, google.protobuf\n",
"print(\"protobuf + sentencepiece OK\")\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "f83b6371",
"metadata": {},
"outputs": [
{
"ename": "ImportError",
"evalue": "\n requires the protobuf library but it was not found in your environment. Check out the instructions on the\ninstallation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones\nthat match your environment. Please note that you may need to restart your runtime after installation.\n",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mValueError\u001b[39m Traceback (most recent call last)",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:2003\u001b[39m, in \u001b[36mPreTrainedTokenizerBase._from_pretrained\u001b[39m\u001b[34m(cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, token, cache_dir, local_files_only, _commit_hash, _is_local, trust_remote_code, *init_inputs, **kwargs)\u001b[39m\n\u001b[32m 2002\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2003\u001b[39m tokenizer = \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43minit_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2004\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m import_protobuf_decode_error():\n",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\tokenization_utils_tokenizers.py:274\u001b[39m, in \u001b[36mTokenizersBackend.__init__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 273\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tokenizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m274\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m 275\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mCouldn\u001b[39m\u001b[33m'\u001b[39m\u001b[33mt instantiate the backend tokenizer from one of: \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 276\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m(1) a `tokenizers` library serialization file, \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 277\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m(2) a slow tokenizer instance to convert or \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 278\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m(3) an equivalent slow tokenizer class to instantiate and convert. \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 279\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mYou need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 280\u001b[39m )\n\u001b[32m 281\u001b[39m \u001b[38;5;66;03m# Only set defaults when creating TokenizersBackend from scratch\u001b[39;00m\n",
"\u001b[31mValueError\u001b[39m: Couldn't instantiate the backend tokenizer from one of: \n(1) a `tokenizers` library serialization file, \n(2) a slow tokenizer instance to convert or \n(3) an equivalent slow tokenizer class to instantiate and convert. \nYou need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[31mImportError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtransformers\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AutoTokenizer, AutoModelForCausalLM\n\u001b[32m 3\u001b[39m HF_DIR = \u001b[33mr\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mC:\u001b[39m\u001b[33m\\\u001b[39m\u001b[33mUsers\u001b[39m\u001b[33m\\\u001b[39m\u001b[33mDinesh\u001b[39m\u001b[33m\\\u001b[39m\u001b[33mhf_tamil_clm_15m\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m tokenizer = \u001b[43mAutoTokenizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 6\u001b[39m \u001b[43m \u001b[49m\u001b[43mHF_DIR\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_fast\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[43m \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[32m 9\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 11\u001b[39m model = AutoModelForCausalLM.from_pretrained(\n\u001b[32m 12\u001b[39m HF_DIR,\n\u001b[32m 13\u001b[39m trust_remote_code=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 14\u001b[39m )\n\u001b[32m 16\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m🎉 LOCAL HF LOAD SUCCESS\u001b[39m\u001b[33m\"\u001b[39m)\n",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\models\\auto\\tokenization_auto.py:702\u001b[39m, in \u001b[36mAutoTokenizer.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[39m\n\u001b[32m 699\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m tokenizer_class \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 700\u001b[39m tokenizer_class = TokenizersBackend\n\u001b[32m--> \u001b[39m\u001b[32m702\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtokenizer_class\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 703\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(config, \u001b[33m\"\u001b[39m\u001b[33mtokenizer_class\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m 704\u001b[39m _class = config.tokenizer_class\n",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1751\u001b[39m, in \u001b[36mPreTrainedTokenizerBase.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, cache_dir, force_download, local_files_only, token, revision, trust_remote_code, *init_inputs, **kwargs)\u001b[39m\n\u001b[32m 1748\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m file_id \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m resolved_vocab_files:\n\u001b[32m 1749\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1751\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_from_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 1752\u001b[39m \u001b[43m \u001b[49m\u001b[43mresolved_vocab_files\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1753\u001b[39m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1754\u001b[39m \u001b[43m \u001b[49m\u001b[43minit_configuration\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1755\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1756\u001b[39m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1757\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1758\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1759\u001b[39m \u001b[43m \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcommit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1760\u001b[39m \u001b[43m \u001b[49m\u001b[43m_is_local\u001b[49m\u001b[43m=\u001b[49m\u001b[43mis_local\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1761\u001b[39m \u001b[43m \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1762\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1763\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:2004\u001b[39m, in \u001b[36mPreTrainedTokenizerBase._from_pretrained\u001b[39m\u001b[34m(cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, token, cache_dir, local_files_only, _commit_hash, _is_local, trust_remote_code, *init_inputs, **kwargs)\u001b[39m\n\u001b[32m 2002\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 2003\u001b[39m tokenizer = \u001b[38;5;28mcls\u001b[39m(*init_inputs, **init_kwargs)\n\u001b[32m-> \u001b[39m\u001b[32m2004\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[43mimport_protobuf_decode_error\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[32m 2005\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 2006\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mUnable to load tokenizer model from SPM, loading from TikToken will be attempted instead.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 2007\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m(Google protobuf error: Tried to load SPM model with non-SPM vocab file).\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 2008\u001b[39m )\n\u001b[32m 2009\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\Dinesh\\anaconda3\\envs\\tamil-llm\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:79\u001b[39m, in \u001b[36mimport_protobuf_decode_error\u001b[39m\u001b[34m(error_message)\u001b[39m\n\u001b[32m 77\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m DecodeError\n\u001b[32m 78\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m79\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(PROTOBUF_IMPORT_ERROR.format(error_message))\n",
"\u001b[31mImportError\u001b[39m: \n requires the protobuf library but it was not found in your environment. Check out the instructions on the\ninstallation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones\nthat match your environment. Please note that you may need to restart your runtime after installation.\n"
]
}
],
"source": [
"from transformers import AutoTokenizer, AutoModelForCausalLM\n",
"\n",
"HF_DIR = r\"C:\\Users\\Dinesh\\hf_tamil_clm_15m\"\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(\n",
" HF_DIR,\n",
" use_fast=False,\n",
" trust_remote_code=True\n",
")\n",
"\n",
"model = AutoModelForCausalLM.from_pretrained(\n",
" HF_DIR,\n",
" trust_remote_code=True\n",
")\n",
"\n",
"print(\"🎉 LOCAL HF LOAD SUCCESS\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fb116309",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "tamil-llm",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}