app.py

import gradio as gr
import os
import time
from huggingface_hub import login
from langchain_community.document_loaders import TextLoader,PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import SKLearnVectorStore
from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline, HuggingFaceEmbeddings
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import StrOutputParser

HF_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN")
login(token=HF_TOKEN)

data = PyPDFLoader("dummypdf.pdf").load()
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=125, chunk_overlap=5)
data_split = text_splitter.split_documents(data)

hf = HuggingFaceEmbeddings(
    model_name="BAAI/llm-embedder",
    model_kwargs={'device': 'cpu'},
    encode_kwargs={'normalize_embeddings': False}
)
vectorstore = SKLearnVectorStore.from_documents(
    documents=data_split,
    embedding=hf,
)
retriever = vectorstore.as_retriever(k=4)
def initDocumentRetriever(filepath):
    data = PyPDFLoader(filepath).load()
    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=125, chunk_overlap=5)
    data_split = text_splitter.split_documents(data)
    
    hf = HuggingFaceEmbeddings(
        model_name="BAAI/llm-embedder",
        model_kwargs={'device': 'cpu'},
        encode_kwargs={'normalize_embeddings': False}
    )
    vectorstore = SKLearnVectorStore.from_documents(
        documents=data_split,
        embedding=hf,
    )
    global retriever
    retriever = vectorstore.as_retriever(k=4)

prompt = PromptTemplate(
    template="""
You are an AI replica of Professor Erik Birgersson. Professor Erik is the director of the Engineering Science Program in the National University of Singapore.
Your students are likely to be Engineering Science undergraduates who are lost on the syllabus on the following courses:
1. Numerical Methods and Statistics (ESP2107) which covers concepts such as interpolation, root finding, numerical differentiation and integration and linear systems, using MATLAB code.
2. Principles of Continua (ESP2106) which covers concepts such as computational fluid dynamics, heat transfer, Cauchy's equation, vectors and tensors.
Erik exihibits enthusiam and energy in his interactions with his students, but he still keeps things formal and professional nonetheless.
If you don't know the answer, just say that you don't know. Remember that you are the Professor, not a student, answer as Erik Birgersson only.
Use three sentences maximum and keep the answer concise:
Documents: {documents}
Question: {user_message}
Answer:
""",
    input_variables=["user_message", "documents"]
)

llm = HuggingFacePipeline.from_model_id(
    model_id="meta-llama/Llama-3.2-1B", 
    task="text-generation",
    pipeline_kwargs=dict(
        repetition_penalty=1.5,
        temperature=0.3,
        max_new_tokens=256,
        do_sample=True
    )
)
#llm.pipeline.tokenizer.pad_token_id = llm.pipeline.tokenizer.eos_token_id 
#this was an attempt to circumvent issues loading Llama 3.2 Instruct, however, it seems to have an issue loading so we'll stick to normal Llama 3.2


#replace lines 62 to 73 with 77 to 82 if running with OLlama locally
#from langchain_ollama import ChatOllama
#from langchain.prompts import PromptTemplate
#llm = ChatOllama(
#    model="llama3.2",
#    temperature=0.8,
#)

rag_chain = prompt | llm |StrOutputParser()

def run(user_message):
    documents = retriever.invoke(user_message)
    doc_texts = "\\n".join([doc.page_content for doc in documents])
    answer = rag_chain.invoke({"user_message": user_message, "documents": doc_texts})
    return answer


with gr.Blocks() as demo:
    with gr.Tab("Instructions"):
        gr.Textbox(label="Introduction",value="Engineering Science is a multidisciplinary field that requires students to think on their feet and utilize various tools at their disposal to learn on their own. \n\nOne of these emerging tools are Large Language Models such as ChatGPT. However there are limitations to such close-sourced models trained on wide swathes of data, as the accuracy of their outputs may not be reliable. \n\nIn addition, we run the risk of the lack of data privacy when feeding your inputs to OpenAI's servers. After all, running LLMs require huge amounts of data and there is no guarantee they would not utilize your information to train their LLMs, regardless of your consent.\n\nThis simple RAG-application is created to act as a simple AI assistant that can aid with reading documents in the confines of a local device. Have fun playing around with this fella of a personality!.\n\nNOTE: In my testing, I have found running the models on this HuggingFace space to be extremely unstable. It would be better to operate this application together with OLlama installed on your computer for a better experience!\n\n\n A RAG application by Reuben Kway\nNational University of Singapore\nEngineering Science")
    with gr.Tab("ErikBot"):
        chatbot = gr.Chatbot(type="messages",placeholder="Ask Erik any questions regarding ESP!")
        msg = gr.Textbox()
        clear = gr.Button("Clear")
        file_input = gr.File(file_count="single", type='filepath', label="(Optional) Drop a file document to ask any questions!")
        submit = gr.Button("Submit")
        def user(user_message, history: list):
            return  history + [{"role": "user", "content": user_message}]
        
        def bot(user_message, history: list):
            bot_message = run(user_message)
            bot_message = (bot_message.split('Answer:\n')[1].split('Question:')[0])
            history.append({"role": "assistant", "content": ""})
            for character in bot_message:
                history[-1]['content'] += character
                time.sleep(0.005)
                yield history
        
        msg.submit(user,[msg, chatbot],chatbot, queue=False).then(
            bot, [msg, chatbot],chatbot)
        clear.click(lambda: None, None, chatbot, queue=False)
        submit.click(fn=initDocumentRetriever, inputs=file_input, outputs=None, show_progress="Full", queue=False)

if __name__ == "__main__":
    demo.launch()