adding LLM On doc using on-prem Hugging face models

app.py

@@ -0,0 +1,194 @@
+import os 
+import gradio as gr
+import time 
+from langchain.document_loaders import PDFMinerLoader,CSVLoader ,UnstructuredWordDocumentLoader,TextLoader,OnlinePDFLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings import SentenceTransformerEmbeddings
+from langchain.vectorstores import FAISS
+from langchain import HuggingFaceHub
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+
+
+DEVICE = 'cpu'
+FILE_EXT = ['pdf','text','csv','word']
+DEFAULT_SYSTEM_PROMPT = "As a chatbot you are answering set of questions being requested ."
+MAX_NEW_TOKENS = 4096
+DEFAULT_TEMPERATURE = 0.1
+DEFAULT_MAX_NEW_TOKENS = 2048
+MAX_INPUT_TOKEN_LENGTH = 4000
+
+def loading_file():
+    return "Loading..."
+
+
+def process_documents(documents,data_chunk=1500,chunk_overlap=100):
+    text_splitter = CharacterTextSplitter(chunk_size=data_chunk, chunk_overlap=chunk_overlap,separator='\n')
+    texts = text_splitter.split_documents(documents)
+    return texts
+
+def get_hugging_face_model(model_id,temperature=0.1,max_tokens=4096,API_key=None):
+    chat_llm = HuggingFacePipeline.from_model_id(
+        model_id=model_id,
+        task="text-generation",
+        pipeline_kwargs={"max_new_tokens": max_tokens,"temperature": temperature,},
+    )
+    # chat_llm = HuggingFaceHub(huggingfacehub_api_token=API_key,
+    #                                   repo_id=model_id,
+    #                                   model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens})
+    return chat_llm
+
+def chat_application(temperature=0.1, max_tokens=1024):
+
+    llm = get_hugging_face_model(model_id='tiiuae/falcon-7b-instruct',temperature=temperature, max_tokens=max_tokens)
+    return llm 
+
+
+def document_loader(file_path,doc_type='pdf',temperature=0.1,max_tokens=2048):
+    document = None
+    if doc_type == 'pdf':
+        document = process_pdf_document(document_file=file_path)
+    elif doc_type == 'text':
+        document = process_text_document(document_file=file_path)
+    elif doc_type == 'csv':
+        document = process_csv_document(document_file=file_path)
+    elif doc_type == 'word':
+        document = process_word_document(document_file=file_path)
+    
+    embedding_model = SentenceTransformerEmbeddings(model_name='thenlper/gte-base',model_kwargs={"device": DEVICE})
+    
+    texts = process_documents(documents=document)
+    global vector_db
+    vector_db = FAISS.from_documents(documents=texts, embedding= embedding_model)
+    global qa
+    qa = RetrievalQA.from_chain_type(llm=chat_application(temperature=temperature,
+                                                            max_tokens=max_tokens
+                                                        ),
+                                chain_type='stuff',
+                                retriever=vector_db.as_retriever(),
+                                #  chain_type_kwargs=chain_type_kwargs,
+                                return_source_documents=True
+                            )
+    return "Document Processing completed ..."
+        
+def process_text_document(document_file):
+    loader = TextLoader(document_file.name)
+    document = loader.load()
+    return document
+
+def process_csv_document(document_file):
+    loader = CSVLoader(file_path=document_file.name)
+    document = loader.load()
+    return document
+
+def process_word_document(document_file):
+    loader = UnstructuredWordDocumentLoader(file_path=document_file.name)
+    document = loader.load()
+    return document
+
+def process_pdf_document(document_file):
+    print("Document File Name :",document_file.name)
+    loader = PDFMinerLoader(document_file.name)
+    document = loader.load()
+    return document
+
+def clear_chat():
+    return []
+
+def infer(question, history):
+    # res = []
+    # # for human, ai in history[:-1]:
+    # #     pair = (human, ai)
+    # #     res.append(pair)
+    
+    # chat_history = res
+    print("Question in infer :",question)
+    result = qa({"query": question})
+    matching_docs_score = vector_db.similarity_search_with_score(question)
+    
+    print(" Matching_doc ",matching_docs_score)
+    
+    return result["result"]
+
+def bot(history):
+   
+    response = infer(history[-1][0], history)
+    history[-1][1] = ""
+    
+    for character in response:     
+        history[-1][1] += character
+        time.sleep(0.05)
+        yield history
+
+def add_text(history, text):
+    history = history + [(text, None)]
+    return history, ""
+
+css="""
+#col-container {max-width: 700px; margin-left: auto; margin-right: auto;}
+"""
+
+title = """
+<div style="text-align: center;max-width: 700px;">
+    <h1>Chat with Data • OpenAI/HuggingFace</h1>
+    <p style="text-align: center;">Upload a file from system,UpLoad file and generate embeddings, <br />
+    once status is ready, you can start asking questions about the data you uploaded without chat history <br />
+    and gives you option to use HuggingFace/OpenAI as LLM's, make sure to add your key.
+    </p>
+</div>
+"""
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML(title)
+        
+    with gr.Group():
+        chatbot = gr.Chatbot(height=300)
+    with gr.Row():
+        question = gr.Textbox(label="Type your question !",lines=1)
+        submit_btn = gr.Button(value="Send message", variant="primary", scale = 1)
+        clean_chat_btn =  gr.Button("Delete Chat")
+
+    with gr.Column():
+        with gr.Box():
+            LLM_option = gr.Dropdown(['tiiuae/falcon-7b-instruct'],label='Large Language Model Selection',info='LLM Service')
+                 
+        with gr.Column():
+            with gr.Box():
+                file_extension = gr.Dropdown(FILE_EXT, label="File Extensions", info="Select type of file to upload !")
+                pdf_doc = gr.File(label="Upload File to start QA", file_types=FILE_EXT, type="file")
+                with gr.Accordion(label='Advanced options', open=False):
+                    max_new_tokens = gr.Slider(
+                        label='Max new tokens',
+                        minimum=2048,
+                        maximum=MAX_NEW_TOKENS,
+                        step=1024,
+                        value=DEFAULT_MAX_NEW_TOKENS,
+                        )
+                    temperature = gr.Slider(
+                    label='Temperature',
+                    minimum=0.1,
+                    maximum=4.0,
+                    step=0.1,
+                    value=DEFAULT_TEMPERATURE,
+                    )
+                with gr.Row():
+                    langchain_status = gr.Textbox(label="Status", placeholder="", interactive = False)
+                    load_pdf = gr.Button("Upload File & Generate Embeddings",).style(full_width = False)
+
+        # chatbot = gr.Chatbot()l̥
+        # question = gr.Textbox(label="Question", placeholder="Type your question and hit Enter")
+        # submit_button = gr.Button("Send Message")
+
+    if pdf_doc:
+        load_pdf.click(loading_file, None, langchain_status, queue=False)    
+        load_pdf.click(document_loader, inputs=[pdf_doc,file_extension,temperature,max_new_tokens], outputs=[langchain_status], queue=False)
+
+    question.submit(add_text, inputs=[chatbot, question], outputs=[chatbot, question]).then(bot, chatbot, chatbot)
+    submit_btn.click(add_text, inputs=[chatbot, question], outputs=[chatbot, question]).then(bot, chatbot, chatbot)
+    # submit_btn.then(chatf.highlight_found_text, [chatbot, sources], [sources])
+    clean_chat_btn.click(clear_chat, [], chatbot)
+
+
+demo.launch()

requirements.txt

@@ -0,0 +1,11 @@
+openai
+tiktoken
+chromadb
+langchain
+unstructured
+unstructured[local-inference]
+transformers
+torch
+faiss-cpu
+sentence-transformers
+chromadb