Add application file

app.py

@@ -1,49 +1,26 @@
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
-from threading import Thread
 
-#tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1")
-#model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1", torch_dtype=torch.float16)
+model_name = "aaditya/OpenBioLLM-Llama3-8B-GGUF"
+model_file = "openbiollm-llama3-8b.Q5_K_M.gguf"
 
-tokenizer = AutoTokenizer.from_pretrained("aaditya/OpenBioLLM-Llama3-8B-GGUF")
-model = AutoModelForCausalLM.from_pretrained("aaditya/openbiollm-llama3-8b.Q5_K_M.gguf", torch_dtype=torch.float16)
+model_path = hf_hub_download(model_name,
+                             filename=model_file,
+                             local_dir='/content')
+print("My model path: ", model_path)
+llm = Llama(model_path=model_path,
+            n_gpu_layers=-1)
 
-class StopOnTokens(StoppingCriteria):
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
-        stop_ids = [29, 0]
-        for stop_id in stop_ids:
-            if input_ids[0][-1] == stop_id:
-                return True
-        return False
+def my_inference_function(Question):
+  prompt = f"You are an expert and experienced from the healthcare and biomedical domain with extensive medical knowledge and practical experience. Your name is OpenBioLLM, and you were developed by Saama AI Labs with Open Life Science AI. who's willing to help answer the user's query with explanation. In your explanation, leverage your deep medical expertise such as relevant anatomical structures, physiological processes, diagnostic criteria, treatment guidelines, or other pertinent medical concepts. Use precise medical terminology while still aiming to make the explanation clear and accessible to a general audience. Medical Question: {Question} Medical Answer:"
+  response = llm(prompt, max_tokens=4000)['choices'][0]['text']
+  return response
 
-def predict(message, history):
-    history_transformer_format = history + [[message, ""]]
-    stop = StopOnTokens()
+gradio_interface = gradio.Interface(
+  fn = my_inference_function,
+  inputs = "text",
+  outputs = "text"
+)
+gradio_interface.launch()
 
-    messages = "".join(["".join(["\n<human>:"+item[0], "\n<bot>:"+item[1]])
-                for item in history_transformer_format])
-
-    model_inputs = tokenizer([messages], return_tensors="pt")
-    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
-    generate_kwargs = dict(
-        model_inputs,
-        streamer=streamer,
-        max_new_tokens=1024,
-        do_sample=True,
-        top_p=0.95,
-        top_k=1000,
-        temperature=1.0,
-        num_beams=1,
-        stopping_criteria=StoppingCriteriaList([stop])
-        )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-
-    partial_message = ""
-    for new_token in streamer:
-        if new_token != '<':
-            partial_message += new_token
-            yield partial_message
-
-gr.ChatInterface(predict).launch()