Upload handler.py with huggingface_hub

handler.py

@@ -0,0 +1,55 @@
+from typing import Dict, Any
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+class EndpointHandler:
+    def __init__(self, path: str = ""):
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path,
+            torch_dtype=torch.float16,
+            device_map="auto"
+        )
+    
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        # Get inputs
+        inputs = data.get("inputs", "")
+        parameters = data.get("parameters", {})
+        
+        # Extract parameters with defaults
+        max_new_tokens = parameters.get("max_new_tokens", 200)
+        temperature = parameters.get("temperature", 0.7)
+        top_p = parameters.get("top_p", 0.9)
+        
+        # Format prompt if instruction/input provided separately
+        if isinstance(inputs, dict):
+            instruction = inputs.get("instruction", "")
+            product_details = inputs.get("product_details", "")
+            prompt = f"***Instruction: {instruction}\n***Input: {product_details}\n***Response:"
+        else:
+            prompt = inputs
+        
+        # Tokenize
+        input_ids = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        
+        # Generate
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **input_ids,
+                max_new_tokens=max_new_tokens,
+                do_sample=True,
+                temperature=temperature,
+                top_p=top_p,
+                pad_token_id=self.tokenizer.eos_token_id
+            )
+        
+        # Decode
+        full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        
+        # Extract response part
+        if "***Response:" in full_response:
+            response = full_response.split("***Response:")[1].strip()
+        else:
+            response = full_response
+        
+        return {"generated_text": response}