add requirement, and handler.py

handler-3.py

@@ -0,0 +1,32 @@
+from transformers import AutoModelForCausalLM,AutoTokenizer,BitsAndBytesConfig
+import torch
+import os
+
+class EndpointHandler():
+
+    def __init__(self, model_id="",HF_TOKEN=""):
+       self.bnb_config = BitsAndBytesConfig(
+          load_in_4bit=True,
+          bnb_4bit_quant_type="nf4",
+          bnb_4bit_compute_dtype=torch.bfloat16,)
+       self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+       self.model = AutoModelForCausalLM.from_pretrained(model_id,
+                                             device_map={"":0},
+                                             quantization_config=self.bnb_config,
+                                             token=HF_TOKEN)
+      self.device = "cuda:0"
+
+    def __call__(self, input:str) -> str:
+        """
+       data args:
+            inputs (:obj: `str` | `PIL.Image` | `np.array`)
+            kwargs
+      Return:
+            A :obj:`list` | `dict`: will be serialized and returned
+        """
+        inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
+        outputs = self.model.generate(**inputs, max_new_tokens=20)
+        result = (self.tokenizer.decode(outputs[0], skip_special_tokens=True))
+        return result
+
+

requirements.txt

@@ -0,0 +1,12 @@
+bitsandbytes==0.42.0
+accelerate==0.27.1
+peft==0.8.2
+trl==0.7.10
+datasets==2.17.0
+transformers==4.38.0
+bitsandbytes==0.42.0
+accelerate==0.27.1
+peft==0.8.2
+trl==0.7.10
+datasets==2.17.0
+transformers==4.38.0