Update handler.py

- replace bfloat16 to float16
- added basic tokeniser fixes
- Removed prompt

handler.py

@@ -10,16 +10,21 @@ class EndpointHandler:
         self.tokenizer = AutoTokenizer.from_pretrained(path)
         
         # StarCoder2 FIXES
+        # if self.tokenizer.pad_token is None:
+        #     self.tokenizer.pad_token = self.tokenizer.eos_token
+        # self.tokenizer.padding_side = "left"  # Critical for code completion
+
+        # Basic tokenizer fixes only
         if self.tokenizer.pad_token is None:
             self.tokenizer.pad_token = self.tokenizer.eos_token
-        self.tokenizer.padding_side = "left"  # Critical for code completion
         
         print(f"Loading model from {path} on device: {self.device}...")
         self.model = AutoModelForCausalLM.from_pretrained(
             path,
-            torch_dtype=torch.bfloat16,     # ✅ Changed to bfloat16 
+            torch_dtype=torch.float16,     # ✅ back to float16 from bfloat16 
             trust_remote_code=True,
             device_map="auto",
+            low_cpu_mem_usage=True
             # attn_implementation="flash_attention_2"  # ✅ Faster + stable
         )
         self.model.eval()
@@ -32,12 +37,12 @@ class EndpointHandler:
         if not isinstance(inputs, str) or not inputs.strip():
             return {"generated_text": ""}
 
-         # ✅ StarCoder2: Add code context prefix
-        prompt = f"<fim_prefix>{inputs}<fim_suffix><fim_middle>"
+        #  # ✅ StarCoder2: Add code context prefix
+        # prompt = f"<fim_prefix>{inputs}<fim_suffix><fim_middle>"
 
         gen_kwargs = {
             "max_new_tokens": min(parameters.get("max_new_tokens", 256), 512),  # Cap for stability
-            "temperature": parameters.get("temperature", 0.2),
+            "temperature": parameters.get("temperature", 0.3),
             "top_p": parameters.get("top_p", 0.95),
             "top_k": parameters.get("top_k", 50),
             "do_sample": parameters.get("do_sample", True),
@@ -46,13 +51,13 @@ class EndpointHandler:
             "pad_token_id": self.tokenizer.pad_token_id,
         }
         
-        # print(f"Generating with parameters: {gen_kwargs}")
-        print(f"Prompt length: {len(prompt)} | Gen params: {gen_kwargs}")
+        print(f"Generating with parameters: {gen_kwargs}")
+        # print(f"Prompt length: {len(prompt)} | Gen params: {gen_kwargs}")
         
         # StarCoder2 tokenization
         inputs = inputs.strip()
         tokenized = self.tokenizer(
-            prompt,
+            # prompt,
             inputs,
             return_tensors="pt",
             truncation=True,
@@ -77,5 +82,5 @@ class EndpointHandler:
                 clean_up_tokenization_spaces=True
             )
         
-            generated = generated.replace("<fim_middle>", "").replace("<fim_suffix>", "").strip()
+            # generated = generated.replace("<fim_middle>", "").replace("<fim_suffix>", "").strip()
         return {"generated_text": generated.strip()}