app

app.py

@@ -1,7 +1,70 @@
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+import copy
+import random
+import os
+import requests
+import time
+import sys
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+from huggingface_hub import snapshot_download
+from llama_cpp import Llama
+
+repo_name = "kirp/TinyLlama-1.1B-Chat-v0.2-gguf"
+model_name = "ggml-model-q4_k_m.gguf"
+
+snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
+
+model_name = "../model/ggml-model-q2_k.gguf"
+
+model = Llama(
+    model_path=model_name,
+    n_ctx=2000,
+    n_parts=1,
+)
+
+
+template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+
+def generate(
+    input=None,
+    temperature=0.1,
+    top_p=0.75,
+    top_k=40,
+    max_new_tokens=128,
+):
+    prompt = template.format(input)
+    output = model(prompt, 
+                   temperature = temperature,
+                   top_k = top_k,
+                   top_p = top_p,
+                   max_tokens = max_new_tokens + len(input),
+                   stop=["<|im_end|>"], 
+                   echo=True)
+    output = output["choices"][0]['text']
+    return output.split("assistant\n")[1]
+
+g = gr.Interface(
+    fn=generate,
+    inputs=[
+        gr.components.Textbox(
+            lines=2, label="Prompt", placeholder="Tell me about huggingface."
+        ),
+        gr.components.Slider(minimum=0, maximum=1, value=0.7, label="Temperature"),
+        gr.components.Slider(minimum=0, maximum=1, value=0.8, label="Top p"),
+        gr.components.Slider(minimum=0, maximum=100, step=1, value=50, label="Top k"),
+        gr.components.Slider(
+            minimum=1, maximum=1024, step=1, value=256, label="Max tokens"
+        ),
+    ],
+    outputs=[
+        gr.Textbox(
+            lines=10,
+            label="Output",
+        )
+    ],
+    title="tinyllama-1.1b-chat gguf",
+    description=''
+)
+g.queue(concurrency_count=1)
+g.launch()

requirements.txt

@@ -1 +1,2 @@
-llama-cpp-python-kirp
+llama-cpp-python-kirp
+huggingface_hub