Upload 2 files

app.py

@@ -2,40 +2,66 @@ import gradio as gr
 from huggingface_hub import InferenceClient
 
 """
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+Copied from inference in colab notebook
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
+from transformers import AutoTokenizer , AutoModelForSeq2SeqLM , TextIteratorStreamer
+from threading import Thread
+
+# Load model and tokenizer globally to avoid reloading for every request
+base_model = "Helsinki-NLP/europarl"
+model_path = "Mat17892/t5small_enfr_opus"
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True, legacy=False)
+
+# Load the base model (e.g., LLaMA)
+base_model = AutoModelForSeq2SeqLM.from_pretrained(base_model)
+
+# Load LoRA adapter
+from peft import PeftModel
+model = PeftModel.from_pretrained(base_model, model_path)
 
 def respond(
-    message,
+    message: str,
     history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
 ):
+    # Combine system message and history into a single prompt
     messages = [{"role": "system", "content": system_message}]
-
     for val in history:
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
-
     messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
+    
+    # Tokenize the messages
+    inputs = tokenizer.apply_chat_template(
         messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+        tokenize = True,
+        add_generation_prompt = True, # Must add for generation
+        return_tensors = "pt",
+    )
+    # Generate tokens incrementally
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    generation_kwargs = {
+        "input_ids": inputs,
+        "max_new_tokens": max_tokens,
+        "temperature": temperature,
+        "top_p": top_p,
+        "do_sample": True,
+        "streamer": streamer,
+    }
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
 
+    # Yield responses as they are generated
+    response = ""
+    for token in streamer:
         response += token
         yield response
 

requirements.txt

@@ -1 +1,5 @@
-huggingface_hub==0.25.2
+huggingface_hub==0.25.2
+
+transformers
+accelerate
+peft