app.py

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

device = 'cuda' if torch.cuda.is_available() else 'cpu'
load8 = True if device == 'cuda' else False
model = AutoModelForCausalLM.from_pretrained("GenAI-CoolCats/WLU-Phi2", load_in_8bit=load8)
model.load_adapter("GenAI-CoolCats/WLU-Phi2", adapter_name="ad1")
tokenizer = AutoTokenizer.from_pretrained("GenAI-CoolCats/WLU-Phi2")


def predict(input_text):
    inputs = tokenizer.encode(input_text, return_tensors="pt", max_length=256).to(device)
    outputs = model.generate(inputs, max_length=128, 
                            temperature=0.5, 
                            pad_token_id=tokenizer.eos_token_id,
                            do_sample=True)
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)

	# remove repeat of the question
    if '?' in response:
        to_q = response.index('?')
        #if len(input_text)-1 <= to_q and response[:to_q] == input_text[:to_q]:
        response = response[to_q+1:]
    return response
	
import gradio as gr

description_html = """
<h2>Ask me questions about Washington and Lee University!</h2>
<img src='https://upload.wikimedia.org/wikipedia/commons/thumb/c/c4/Wash-lee_logo_from_NCAA.svg/300px-Wash-lee_logo_from_NCAA.svg.png' style='width:15%;'>
<p>Enter your prompts below, and click 'submit.'</p>
"""

iface = gr.Interface(fn=predict, 
                     inputs="text", 
                     outputs="text",
                     title="WLU-PHI2",
                     #description="Enter your prompts below, and click 'submit.'",
                     examples=[["What is W&L Known for?"], ["What is W&L like?"], ["Where is W&L located?"]],
                     description=description_html
                    )
    
iface.launch(share=True)