Update app.py

app.py

@@ -1,17 +1,71 @@
 import gradio as gr
-from transformers import pipeline
-
-def generate_text(prompt):
-    model_name = "TheBloke/openchat_3.5-GGUF"
-    generator = pipeline('text-generation', model=model_name)
-    return generator(prompt, max_length=50)[0]['generated_text']
-
-iface = gr.Interface(
-    fn=generate_text,
-    inputs=gr.inputs.Textbox(lines=2, placeholder="Enter your prompt here..."),
-    outputs="text",
-    title="GPT-3.5 Text Generation",
-    description="This is a demo for GPT-3.5 text generation model hosted on Hugging Face."
+from gpt4all import GPT4All
+from huggingface_hub import hf_hub_download
+
+model_path = "models"
+model_name = "openchat_3.5-GGUF/blob/main/openchat_3.5.Q4_K_M.gguf"
+hf_hub_download(repo_id="TheBloke/openchat_3.5-GGUF", filename=model_name, local_dir=model_path, local_dir_use_symlinks=False)
+
+model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
+
+model.config["promptTemplate"] = "[INST] {0} [/INST]"
+model.config["systemPrompt"] = ""
+model._is_chat_session_activated = False
+
+max_new_tokens = 2048
+
+def generater(message, history, temperature, top_p, top_k):
+    prompt = "<s>"
+    for user_message, assistant_message in history:
+        prompt += model.config["promptTemplate"].format(user_message)
+        prompt += assistant_message + "</s>"
+    prompt += model.config["promptTemplate"].format(message)
+    outputs = []    
+    for token in model.generate(prompt=prompt, temp=temperature, top_k = top_k, top_p = top_p, max_tokens = max_new_tokens, streaming=True):
+        outputs.append(token)
+        yield "".join(outputs)
+
+chatbot = gr.Chatbot()
+
+additional_inputs=[
+    gr.Slider(
+        label="temperature",
+        value=0.5,
+        minimum=0.0,
+        maximum=2.0,
+        step=0.05,
+        interactive=True,
+        info="Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.",
+    ),
+    gr.Slider(
+        label="top_p",
+        value=1.0,
+        minimum=0.0,
+        maximum=1.0,
+        step=0.01,
+        interactive=True,
+        info="0.1 means only the tokens comprising the top 10% probability mass are considered. Suggest set to 1 and use temperature. 1 means 100% and will disable it",
+    ),
+    gr.Slider(
+        label="top_k",
+        value=40,
+        minimum=0,
+        maximum=1000,
+        step=1,
+        interactive=True,
+        info="limits candidate tokens to a fixed number after sorting by probability. Setting it higher than the vocabulary size deactivates this limit.",
+    )
+]
+
+iface = gr.ChatInterface(
+    fn = generater,
+    title="AI Demo",
+    chatbot=chatbot,
+    additional_inputs=additional_inputs,
 )
 
-iface.launch()
+with gr.Blocks() as demo:
+    iface.render()
+
+if __name__ == "__main__":
+    demo.queue(max_size=3).launch()