Update handler.py

handler.py

@@ -1,10 +1,17 @@
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 
+TOKENIZER_NAME = "polyglots/Extended-Sinhala-LLaMA"
+
 class EndpointHandler:
     def __init__(self, path=""):
-        print(f"Loading tokenizer from {path}...")
-        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        print(f"Loading extended Sinhala tokenizer from {TOKENIZER_NAME}...")
+        # Must load from the original extended tokenizer repo, NOT from path
+        # because the model folder doesn't contain the full custom tokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            TOKENIZER_NAME,
+            trust_remote_code=True,
+        )
         if self.tokenizer.pad_token is None:
             self.tokenizer.pad_token = self.tokenizer.eos_token
 
@@ -13,7 +20,11 @@ class EndpointHandler:
             path,
             torch_dtype  = torch.float16,
             device_map   = "auto",
+            trust_remote_code = True,
         )
+        # Resize to match extended vocab (139,336 tokens)
+        self.model.resize_token_embeddings(len(self.tokenizer))
+        self.model.config.pad_token_id = self.tokenizer.eos_token_id
         self.model.eval()
         print("Model ready!")