Add custom handler for Inference Endpoints

handler.py

@@ -0,0 +1,90 @@
+"""
+Custom handler for cad0 HuggingFace Inference Endpoint.
+
+This loads the Qwen2.5-Coder-7B-Instruct base model with the cad0 LoRA adapter.
+Upload this file to the campedersen/cad0 model repo.
+"""
+
+from typing import Dict, Any
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+
+class EndpointHandler:
+    def __init__(self, path: str = ""):
+        """Load model and tokenizer."""
+        # Base model that cad0 was fine-tuned from
+        base_model = "Qwen/Qwen2.5-Coder-7B-Instruct"
+
+        # Load tokenizer from base model
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            base_model,
+            trust_remote_code=True
+        )
+
+        # Quantization config for efficient inference
+        bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+        )
+
+        # Load the fine-tuned model (path points to the model repo)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path,
+            quantization_config=bnb_config,
+            trust_remote_code=True,
+            device_map="auto",
+            low_cpu_mem_usage=True,
+        )
+
+        self.model.eval()
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Handle inference request.
+
+        Expected input format:
+        {
+            "inputs": "prompt text or chat-formatted text",
+            "parameters": {
+                "max_new_tokens": 256,
+                "temperature": 0.1,
+                "do_sample": true,
+                "return_full_text": false
+            }
+        }
+        """
+        inputs = data.get("inputs", "")
+        parameters = data.get("parameters", {})
+
+        # Default parameters
+        max_new_tokens = parameters.get("max_new_tokens", 256)
+        temperature = parameters.get("temperature", 0.1)
+        do_sample = parameters.get("do_sample", temperature > 0)
+        return_full_text = parameters.get("return_full_text", False)
+
+        # Tokenize
+        encoded = self.tokenizer(inputs, return_tensors="pt").to(self.model.device)
+        input_length = encoded.input_ids.shape[1]
+
+        # Generate
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **encoded,
+                max_new_tokens=max_new_tokens,
+                temperature=temperature if temperature > 0 else 1.0,
+                do_sample=do_sample,
+                pad_token_id=self.tokenizer.eos_token_id,
+                eos_token_id=self.tokenizer.eos_token_id,
+            )
+
+        # Decode
+        if return_full_text:
+            generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        else:
+            generated_text = self.tokenizer.decode(
+                outputs[0][input_length:],
+                skip_special_tokens=True
+            )
+
+        return {"generated_text": generated_text}