Update handler.py

handler.py

@@ -1,46 +1,48 @@
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import get_peft_model, LoraConfig
-from safetensors.torch import load_file
-from huggingface_hub import hf_hub_download
-import torch
-
-class EndpointHandler:
-    def __init__(self, path=""):
-        self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
-        base_model = AutoModelForCausalLM.from_pretrained(
-            "meta-llama/Llama-2-7b-hf",
-            torch_dtype=torch.float16,
-            device_map="auto"
-        )
-
-        lora_config = LoraConfig(
-            r=8,
-            lora_alpha=32,
-            target_modules=["q_proj"],
-            lora_dropout=0.05,
-            bias="none",
-            task_type="CAUSAL_LM"
-        )
-
-        self.model = get_peft_model(base_model, lora_config)
-        adapter_path = hf_hub_download(
-            repo_id="vignesh0007/Anime-Gen-Llama-2-7B",
-            filename="adapter_model.safetensors",
-            repo_type="model"
-        )
-        lora_state = load_file(adapter_path)
-        self.model.load_state_dict(lora_state, strict=False)
-        self.model.eval()
-
-    def __call__(self, data):
-        inputs = data.get("inputs", "")
-        tokens = self.tokenizer(inputs, return_tensors="pt").to(self.model.device)
-        with torch.no_grad():
-            outputs = self.model.generate(
-                **tokens,
-                max_new_tokens=256,
-                temperature=0.8,
-                top_p=0.95,
-                do_sample=True
-            )
-        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import get_peft_model, LoraConfig
+from safetensors.torch import load_file
+from huggingface_hub import hf_hub_download
+import torch
+
+token = os.getenv("HUGGINGFACE_HUB_TOKEN")
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
+        base_model = AutoModelForCausalLM.from_pretrained(
+            "meta-llama/Llama-2-7b-hf",
+            torch_dtype=torch.float16,
+            device_map="auto"
+        )
+
+        lora_config = LoraConfig(
+            r=8,
+            lora_alpha=32,
+            target_modules=["q_proj"],
+            lora_dropout=0.05,
+            bias="none",
+            task_type="CAUSAL_LM"
+        )
+
+        self.model = get_peft_model(base_model, lora_config)
+        adapter_path = hf_hub_download(
+            repo_id="vignesh0007/Anime-Gen-Llama-2-7B",
+            filename="adapter_model.safetensors",
+            repo_type="model"
+        )
+        lora_state = load_file(adapter_path)
+        self.model.load_state_dict(lora_state, strict=False)
+        self.model.eval()
+
+    def __call__(self, data):
+        inputs = data.get("inputs", "")
+        tokens = self.tokenizer(inputs, return_tensors="pt").to(self.model.device)
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **tokens,
+                max_new_tokens=256,
+                temperature=0.8,
+                top_p=0.95,
+                do_sample=True
+            )
+        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)