.

app.py

@@ -11,8 +11,7 @@ from huggingface_hub import snapshot_download
 from llama_cpp import Llama
 
 repo_name = "kirp/TinyLlama-1.1B-Chat-v0.2-gguf"
-# model_name = "ggml-model-q4_k_m.gguf"
-model_name = "ggml-model-q2_k.gguf"
+model_name = "ggml-model-q4_0.gguf"
 
 
 snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
@@ -31,14 +30,14 @@ def generate(
     temperature=0.1,
     top_p=0.75,
     top_k=40,
-    max_new_tokens=512,
+    max_tokens=512,
 ):
     prompt = template.format(input)
     output = model(prompt, 
                    temperature = temperature,
                    top_k = top_k,
                    top_p = top_p,
-                   max_tokens = max_new_tokens + len(input),
+                   max_tokens = max_tokens,
                    stop=["<|im_end|>"], 
                    echo=True)
     output = output["choices"][0]['text']
@@ -48,7 +47,7 @@ g = gr.Interface(
     fn=generate,
     inputs=[
         gr.components.Textbox(
-            lines=2, label="Prompt", placeholder="Tell me about huggingface."
+            lines=2, label="Prompt", value = "What is huggingface?"
         ),
         gr.components.Slider(minimum=0, maximum=1, value=0.7, label="Temperature"),
         gr.components.Slider(minimum=0, maximum=1, value=0.8, label="Top p"),