Upload 2 files

inference.py

@@ -0,0 +1,14 @@
+import torch
+from transformers import pipeline
+
+
+def model_fn(model_dir):
+    instruct_pipeline = pipeline(
+        model=model_dir,
+        torch_dtype=torch.bfloat16,
+        trust_remote_code=True,
+        device_map="auto",
+        model_kwargs={"load_in_8bit": True},
+    )
+
+    return instruct_pipeline

requirements.txt

@@ -0,0 +1,3 @@
+accelerate==0.18.0
+transformers==4.27.2
+bitsandbytes==0.38.1