Initial commit for whole code

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Langchain[[:space:]]Document[[:space:]]Helper[[:space:]]Architecture.png filter=lfs diff=lfs merge=lfs -text

Pipfile

@@ -0,0 +1,26 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+langchain = "*"
+beautifulsoup4 = "*"
+black = "*"
+tiktoken = "*"
+openai = "*"
+pinecone-client = "*"
+unstructured = "*"
+nltk = "*"
+fastapi = "*"
+jinja2 = "*"
+uvicorn = "*"
+streamlit = "*"
+streamlit-chat = "*"
+tqdm = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"
+python_full_version = "3.11.0"

app.py

@@ -0,0 +1,85 @@
+from typing import Set
+
+from backend.core import run_llm
+import streamlit as st
+from streamlit_chat import message
+from PIL import Image
+from io import BytesIO
+import base64
+
+# def add_bg_from_local(image_file):
+#     with open(image_file, "rb") as image_file:
+#         encoded_string = base64.b64encode(image_file.read())
+#     st.markdown(
+#     f"""
+#     <style>
+#     .stApp {{
+#         background-image: url(data:{"jpeg"};base64,{encoded_string.decode()});
+#         background-size: cover
+#     }}
+#     </style>
+#     """,
+#     unsafe_allow_html=True
+#     )
+# background_image = "bg2.jpeg"
+# add_bg_from_local(background_image)
+
+
+st.header("LangChain 🦜🔗 Documentation - Helper ChatBot")
+
+if "user_prompt_history" not in st.session_state:
+    st.session_state["user_prompt_history"] = []
+
+if "chat_answers_history" not in st.session_state:
+    st.session_state["chat_answers_history"] = []
+
+if "chat_history" not in st.session_state:
+    st.session_state["chat_history"] = []
+
+
+def create_sources_string(source_urls: Set[str]) -> str:
+    if not source_urls:
+        return ""
+    sources_list = list(source_urls)
+    sources_list.sort()
+    sources_string = "sources:\n"
+    for i, source in enumerate(sources_list):
+        sources_string += f"{i+1}. {source}\n"
+    return sources_string
+
+
+prompt = st.text_input("Prompt", placeholder="Enter your message here...")
+
+
+if prompt:
+    with st.spinner("Generating response..."):
+        generated_response = run_llm(
+            query=prompt, chat_history=st.session_state["chat_history"]
+        )
+        sources = set(
+            [doc.metadata["source"] for doc in generated_response["source_documents"]]
+        )
+        formatted_response = (
+            f"{generated_response['answer']} \n\n {create_sources_string(sources)}"
+        )
+
+        st.session_state.user_prompt_history.append(prompt)
+        st.session_state.chat_answers_history.append(formatted_response)
+        st.session_state.chat_history.append((prompt, generated_response["answer"]))
+
+if st.session_state["chat_answers_history"]:
+    for generated_response, user_query in zip(
+        st.session_state["chat_answers_history"],
+        st.session_state["user_prompt_history"],
+    ):
+        message(
+            user_query,
+            is_user=True,
+            avatar_style="adventurer",
+            seed=123,
+        )
+        # message(generated_response)
+        st.write(
+            f'<div style="word-wrap: break-word;">{generated_response}</div>',
+            unsafe_allow_html=True,
+        )

backend/core.py

@@ -0,0 +1,34 @@
+import os
+from typing import Any, Dict, List
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain.chains import ConversationalRetrievalChain
+from langchain.vectorstores import Pinecone
+import pinecone
+
+from dotenv import load_dotenv
+
+from consts import INDEX_NAME
+load_dotenv()
+
+pinecone.init(
+    api_key=os.environ.get("PINECONE_API_KEY"),
+    environment=os.environ.get("PINECONE_ENVIRONMENT_REGION"),
+)
+
+def run_llm(query, chat_history):
+    embeddings = OpenAIEmbeddings()
+    docSearch = Pinecone.from_existing_index(index_name = INDEX_NAME, embedding = embeddings)
+    chat = ChatOpenAI(verbose = True, temperature = 0)
+
+    #qa = RetrievalQA.from_chain_type(llm = chat,chain_type = "stuff", retriever = docSearch.as_retriever(), return_source_documents = True)
+    
+    qa = ConversationalRetrievalChain.from_llm(llm = chat, retriever = docSearch.as_retriever(), return_source_documents = True)
+
+    return qa({"question" : query, "chat_history" : chat_history})
+
+# if __name__ == "__main__":
+#     print(run_llm(query = "What is RetrievalQA Chain ? "), )
+
+

consts.py

@@ -0,0 +1 @@
+INDEX_NAME = "langchain-doc-index"

ingestion.py

@@ -0,0 +1,42 @@
+import os
+from langchain.document_loaders import ReadTheDocsLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import Pinecone
+import pinecone
+from dotenv import load_dotenv
+
+from consts import INDEX_NAME
+
+load_dotenv()
+
+pinecone.init(
+    api_key=os.environ.get("PINECONE_API_KEY"),
+    environment=os.environ.get("PINECONE_ENVIRONMENT_REGION"),
+)
+
+
+def ingest_docs() -> None:
+    # openai_api_key=os.environ.get("OPENAI_API_KEY")
+    loader = ReadTheDocsLoader(path="langchain-docs/langchain.readthedocs.io/en/latest")
+    raw_documents = loader.load()
+    print(f"loaded {len(raw_documents) }documents")
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000, chunk_overlap=100, separators=["\n\n", "\n", " ", ""]
+    )
+    documents = text_splitter.split_documents(documents=raw_documents)
+    print(f"Splitted into {len(documents)} chunks")
+
+    for doc in documents:
+        old_path = doc.metadata["source"]
+        new_url = old_path.replace("langchain-docs", "https:/")
+        doc.metadata.update({"source": new_url})
+
+    print(f"Going to insert {len(documents)} to Pinecone")
+    embeddings = OpenAIEmbeddings()
+    Pinecone.from_documents(documents, embeddings, index_name=INDEX_NAME)
+    print("****** Added to Pinecone vectorstore vectors")
+
+
+if __name__ == "__main__":
+    ingest_docs()

requirements.txt

@@ -0,0 +1,110 @@
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+async-timeout==4.0.3
+attrs==23.2.0
+backoff==2.2.1
+beautifulsoup4==4.12.2
+black==23.12.1
+blinker==1.7.0
+cachetools==5.3.2
+certifi==2023.11.17
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+dataclasses-json==0.6.3
+distro==1.9.0
+dnspython==2.4.2
+emoji==2.9.0
+exceptiongroup==1.2.0
+fastapi==0.108.0
+filetype==1.2.0
+frozenlist==1.4.1
+gitdb==4.0.11
+GitPython==3.1.40
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+idna==3.6
+importlib-metadata==6.11.0
+install==1.3.5
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpath-python==1.0.6
+jsonpointer==2.4
+jsonschema==4.20.0
+jsonschema-specifications==2023.12.1
+langchain==0.0.354
+langchain-community==0.0.8
+langchain-core==0.1.5
+langdetect==1.0.9
+langsmith==0.0.77
+loguru==0.7.2
+lxml==5.0.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+mdurl==0.1.2
+multidict==6.0.4
+mypy-extensions==1.0.0
+nltk==3.8.1
+numpy==1.26.3
+openai==1.6.1
+packaging==23.2
+pandas==2.1.4
+pathspec==0.12.1
+pillow==10.2.0
+pinecone-client==2.2.4
+platformdirs==4.1.0
+protobuf==4.25.1
+pyarrow==14.0.2
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydeck==0.8.1b0
+Pygments==2.17.2
+PyMuPDF==1.23.8
+PyMuPDFb==1.23.7
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-iso639==2024.1.2
+python-magic==0.4.27
+pytz==2023.3.post1
+PyYAML==6.0.1
+rapidfuzz==3.6.1
+referencing==0.32.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.16.2
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.25
+starlette==0.32.0.post1
+streamlit==1.29.0
+streamlit-chat==0.1.1
+tabulate==0.9.0
+tenacity==8.2.3
+tiktoken==0.5.2
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+tornado==6.4
+tqdm==4.66.1
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+tzlocal==5.2
+unstructured==0.11.7
+unstructured-client==0.15.1
+urllib3==2.1.0
+uvicorn==0.25.0
+validators==0.22.0
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.17.0