Changes

.gitignore

@@ -0,0 +1 @@
+itaca_mistral7b_qlora_4bit-unsloth.Q4_K_M.gguf

app.py

@@ -1,19 +1,17 @@
 import random
 import gradio as gr
-from transformers import pipeline
+from ctransformers import AutoModelForCausalLM
 
+local_path = r"itaca_mistral7b_qlora_4bit-unsloth.Q4_K_M.gguf"
 
-def random_response(message, history):
-    return random.choice(["Yes", "No"])
+model = AutoModelForCausalLM.from_pretrained(local_path, model_file="itaca_mistral7b_qlora_4bit-unsloth.Q4_K_M.gguf", model_type="mistral", local_files_only=True
+                                             # , gpu_layers=50
 
-
-# model="michelebasilico/itaca-mistral-7b-v2-4bit")
-model = pipeline("text-generation",
-                 model="michelebasilico/itaca-mistral-7b-v2-16bit")
+                                             )
 
 
 def predict(message, history):
-    outputs = model(message)[0]["generated_text"]
+    outputs = model(message)
 
     return outputs
 

requirements.txt

@@ -1,2 +1,2 @@
-transformers
+ctransformers
 torch