Upload RAG chatbot files

Firest version of granite 3.0 8b

app.py

@@ -0,0 +1,155 @@
+# from ibm_watsonx_ai.foundation_models import ModelInference
+# from ibm_watsonx_ai.metanames import GenTextParamsMetaNames as GenParams
+# from ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames
+# from ibm_watsonx_ai import Credentials
+# from langchain_ibm import WatsonxLLM, WatsonxEmbeddings
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain_community.document_loaders import PyPDFLoader
+# from langchain.chains import RetrievalQA
+from langchain_classic.chains import RetrievalQA
+from langchain_community.llms import HuggingFacePipeline
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_huggingface import HuggingFacePipeline
+import torch
+
+from huggingface_hub import HfFolder
+import gradio as gr
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, Pipeline, BitsAndBytesConfig
+
+## You can use this section to suppress warnings generated by your code:
+def warn(*args, **kwargs):
+    pass
+import warnings
+warnings.warn = warn
+warnings.filterwarnings('ignore')
+
+## LLM
+# def get_llm():
+#     model_id = 'ibm/granite-3-2-8b-instruct'
+#     parameters = {
+#         GenParams.MAX_NEW_TOKENS: 256, # This controls the maximum number of tokens in the generated output
+#         GenParams.TEMPERATURE: 0.5, # This controls randomness or creativity of the model's responses
+#     }
+#     project_id = 'skills-network'
+#     watsonx_llm = WatsonxLLM(
+#         model_id = model_id,
+#         url="https://us-south.ml.cloud.ibm.com",
+#         project_id = project_id,
+#         params = parameters,
+#     )
+#     return watsonx_llm
+
+def get_llm():
+    model_id = 'ibm-granite/granite-3.0-8b-instruct'
+    
+    # Configuration for 4-bit compression
+    quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
+    )
+    
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_id,
+        quantization_config=quantization_config,
+        device_map='cpu',
+        torch_dtype='float32',
+    )
+
+    pipeline = Pipeline(
+        model=model,
+        tokenizer=tokenizer,
+        max_new_tokens=256,
+        temperature= 0.3,
+        do_sample= True
+    )
+    
+    return HuggingFacePipeline(pipeline=pipeline)
+
+## Document loader
+def document_loader(file):
+    loader = PyPDFLoader(file)
+    loaded_document = loader.load()
+    return loaded_document
+## Text splitter
+def text_splitter(data):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size = 1000,
+        chunk_overlap = 50,
+        length_function = len,
+    )
+    chuncks = text_splitter.split_documents(data)
+    return chuncks
+## Embedding model
+
+
+# def watsonx_embedding():
+#     embed_params = {
+#         EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: 256,
+#         EmbedTextParamsMetaNames.RETURN_OPTIONS: {'input_text': True},
+#     }
+    
+#     watsonx_embedding = WatsonxEmbeddings(
+#         model_id = "ibm/slate-125m-english-rtrvr-v2",
+#         url="https://us-south.ml.cloud.ibm.com",
+#         project_id = "skills-network",
+#         params = embed_params,
+#     )
+#     return watsonx_embedding
+def watsonx_embedding():
+    model_id = 'sentence-transformers/all-MiniLM-L6-v2'
+    embedding_model = HuggingFaceEmbeddings(model_name=model_id)
+    return embedding_model
+
+## Vector db
+def vector_database(chunks):
+    
+    chunks = [
+        c for c in chunks
+        if c.page_content and c.page_content.strip() and len(c.page_content.strip()) > 10
+    ]
+    embedding_model = watsonx_embedding()
+    vectordb = Chroma.from_documents(chunks, embedding_model)
+    return vectordb
+## Retriever
+def retriever(file):
+    splits = document_loader(file)
+    chunks = text_splitter(splits)
+    vectordb = vector_database(chunks)
+    retriever = vectordb.as_retriever()
+    
+    return retriever
+## QA Chain
+def retriever_qa(file, query):
+    llm = get_llm()
+    retriever_obj = retriever(file)
+    qa = RetrievalQA.from_chain_type(
+        llm = llm, 
+        chain_type = 'stuff',
+        retriever = retriever_obj,
+        return_source_documents = False
+    )
+    response = qa.invoke(query)
+    return response['result']
+    
+## Create Gradio interface
+rag_application = gr.Interface(
+    fn = retriever_qa,
+    # allow_flagging = 'never',
+    flagging_mode = 'never',
+    inputs = [
+        gr.File(label = 'Upload PDF File', file_count = 'single', file_types = ['.pdf'], type ='filepath'), # Drag and drop file upload
+        gr.Textbox(label= 'Input Query', lines = 2, placeholder = 'Type your question here...')
+    ],
+    outputs = gr.Textbox(label = 'Ouput'),
+    title= 'PDF QA Chatbot',
+    description = 'Upload a PDF document and ask any question. The chatbot will try to answer using the provided document.'
+)
+# Launch the app
+rag_application.launch(server_name= '127.0.0.1', server_port= 7860)

requirements.txt

@@ -0,0 +1,9 @@
+transformers
+torch
+sentence-transformers
+langchain
+langchain-community
+chromadb
+pypdf
+gradio
+accelerate