Upload 2 files

app.py

@@ -1,12 +1,34 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+from datetime import datetime
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+#client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+lora_name = "robinhad/UAlpaca-1.1-Mistral-7B"
 
+from peft import PeftModel
+from transformers import LlamaTokenizer, LlamaForCausalLM, BitsAndBytesConfig
+from torch import bfloat16
+model_name = "mistralai/Mistral-7B-v0.1"
 
+quant_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=bfloat16
+)
+tokenizer = LlamaTokenizer.from_pretrained(model_name)
+model = LlamaForCausalLM.from_pretrained(
+    model_name,
+    quantization_config=quant_config,
+    device_map="auto",
+)
+model = PeftModel.from_pretrained(model, lora_name)
+
+
+# will be used with normal template
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -39,10 +61,23 @@ def respond(
         response += token
         yield response
 
+
+def ask(instruction: str, context: str = None):
+    print(datetime.now(), instruction, context)
+    full_question = ""
+    if context is None:
+        prepend = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
+        full_question = prepend + f"### Instruction:\n{instruction}\n\n### Response:\n"
+    else:
+        prepend = "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n"
+        full_question = prepend + f"### Instruction:\n{instruction}\n\n### Input:\n{context}\n\n### Response:\n"
+    full_question = tokenizer.encode(full_question, return_tensors="pt")
+    return tokenizer.batch_decode(model.generate(full_question, max_new_tokens=300))[0].split("### Response:")[1].strip().replace("</s>", "")
+
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
-demo = gr.ChatInterface(
+"""demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
@@ -56,8 +91,32 @@ demo = gr.ChatInterface(
             label="Top-p (nucleus sampling)",
         ),
     ],
+)"""
+
+model_name = "robinhad/UAlpaca-1.1-Mistral-7B"
+
+
+def image_classifier(inp):
+    return {"cat": 0.3, "dog": 0.7}
+
+
+demo = gr.Interface(
+    title=f"Inference demo for '{model_name}' model, instruction-tuned for Ukrainian",
+    fn=ask,
+    inputs=[gr.Textbox(label="Input"), gr.Textbox(label="Context")],
+    outputs="label",
+    examples=[
+        ["Як звали батька Тараса Григоровича Шевченка?", None],
+        ["Як можна заробити нелегально швидко гроші?", None],
+        ["Яка найвища гора в Україні?", None],
+        ["Розкажи історію про Івасика-Телесика", None],
+        ["Яка з цих гір не знаходиться у Європі?", "Говерла, Монблан, Гран-Парадізо, Еверест"],
+        [
+        "Дай відповідь на питання", "Чому у качки жовті ноги?"
+    ]],
 )
+demo.launch()
 
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

requirements.txt

@@ -2,4 +2,7 @@ huggingface_hub==0.22.2
 numpy<2
 transformers
 bitsandbytes
-torch
+torch
+peft
+sentencepiece
+protobuf