Duplicate from cloud-sean/AOAI-Form-Recognizer

README.md

@@ -0,0 +1,13 @@
+---
+title: AOAI Form Recognizer
+emoji: 💻
+colorFrom: purple
+colorTo: purple
+sdk: gradio
+sdk_version: 3.17.0
+app_file: app.py
+pinned: false
+duplicated_from: cloud-sean/AOAI-Form-Recognizer
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,93 @@
+import gradio as gr
+import openai, os
+import tqdm
+import time
+from langchain.vectorstores import Chroma
+from PyPDF2 import PdfReader
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain import VectorDBQA
+from langchain.llms import AzureOpenAI
+from langchain.chains import RetrievalQA
+from langchain.chat_models import AzureChatOpenAI
+
+# from langchain.chat_models import AzureChatOpenAI
+
+os.environ["OPENAI_API_TYPE"] = openai.api_type = "azure"
+os.environ["OPENAI_API_VERSION"] = openai.api_version = "2023-03-15-preview"
+os.environ["OPENAI_API_BASE"] = openai.api_base = "https://openai-endpoint.openai.azure.com/"
+openai.api_key = os.environ["OPENAI_API_KEY"] 
+
+
+def upload_pdf(file, pdf_text, embeddings, vectorstore, azure_embeddings, qa, progress = gr.Progress(track_tqdm=True)):
+    reader = PdfReader(file)
+    number_of_pages = len(reader.pages)
+    pdf_text = ""
+    for page_number in range(number_of_pages):
+        page = reader.pages[page_number]
+        pdf_text += page.extract_text()
+    text_splitter = RecursiveCharacterTextSplitter(        
+    chunk_size = 1000,
+    chunk_overlap  = 200,
+    length_function = len,)
+    texts = text_splitter.split_text(pdf_text)
+    for text in tqdm.tqdm(texts):
+        try:
+            response = openai.Embedding.create(
+            input=text,
+            engine="text-embedding-ada-002")
+            emb = response['data'][0]['embedding']
+            embeddings.append(emb)
+        except Exception as e:
+            print(e)
+            time.sleep(8)
+            response = openai.Embedding.create(
+            input=text,
+            engine="text-embedding-ada-002")
+            emb = response['data'][0]['embedding']
+            embeddings.append(emb)
+    
+
+    azure_embeddings = OpenAIEmbeddings(document_model_name="text-embedding-ada-002",query_model_name="text-embedding-ada-002")
+    vectorstore = Chroma("collection", embedding_function=azure_embeddings)
+
+    vectorstore._collection.add(
+        ids= [f"doc_{i}" for i in range(len(texts))],
+        documents=texts,
+        embeddings=embeddings,
+        metadatas=[{"source": "source"} for text in texts])
+    qa = RetrievalQA.from_chain_type(llm= AzureChatOpenAI(deployment_name="Bartos", model_name='gpt-35-turbo' ), chain_type="stuff", retriever=vectorstore.as_retriever())
+    # qa = RetrievalQA.from_chain_type(llm= AzureOpenAI(deployment_name="davinci003", model_name="text-davinci-003"), chain_type="stuff", vectorstore=vectorstore)
+
+    return pdf_text, pdf_text, embeddings, vectorstore, azure_embeddings, qa, gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)
+
+
+def add_text(chatstate, query, qa):
+    # chain.run(input_documents=docs, question=query)
+    chatstate = chatstate + [(query, qa.run(query))]
+
+    return chatstate, chatstate, qa
+
+with gr.Blocks(css="footer {visibility: hidden}", title='PDF - Q&A') as demo:
+    qa = pdf_text = embeddings = vectorstore = azure_embeddings = gr.State([])
+
+    with gr.Row(visible=False) as chat_row:
+        chatbot = gr.Chatbot()
+    with gr.Row(visible=False) as submit_row:
+        text = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False)
+    chatstate = gr.State([])
+    text.submit(add_text, [chatstate, text, qa], [chatbot, chatstate, qa])
+
+    # set state
+    with gr.Column() as upload_column:
+
+        file = gr.File()
+        upload_btn = gr.Button("Upload")
+        output_text = gr.TextArea()
+        upload_btn.click(upload_pdf, inputs=[file, pdf_text, embeddings, vectorstore, azure_embeddings, qa], outputs=[output_text, pdf_text, embeddings, vectorstore, azure_embeddings, qa, chat_row, submit_row, upload_column])
+
+    with gr.Row():
+        gr.Markdown("`now with GPT-3.5 Turbo`")
+
+
+demo.launch(enable_queue=True)

requirements.txt

@@ -0,0 +1,159 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+appnope==0.1.3
+argilla==1.3.0
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==22.2.0
+backcall==0.2.0
+backoff==2.2.1
+blinker==1.5
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.0.1
+chromadb==0.3.21
+click==8.1.3
+clickhouse-connect==0.5.12
+comm==0.1.2
+contourpy==1.0.7
+cycler==0.11.0
+dataclasses-json==0.5.7
+debugpy==1.6.6
+decorator==5.1.1
+Deprecated==1.2.13
+duckdb==0.7.1
+entrypoints==0.4
+et-xmlfile==1.1.0
+executing==1.2.0
+fastapi==0.85.2
+ffmpy==0.3.0
+filelock==3.9.0
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2023.1.0
+gitdb==4.0.10
+GitPython==3.1.31
+gptcache==0.1.11
+gradio==3.18.0
+h11==0.14.0
+hnswlib==0.7.0
+httpcore==0.16.3
+httptools==0.5.0
+httpx==0.23.3
+huggingface-hub==0.12.1
+idna==3.4
+importlib-metadata==6.0.0
+importlib-resources==5.12.0
+ipykernel==6.21.2
+ipython==8.10.0
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter-client==8.0.3
+jupyter-core==5.2.0
+kiwisolver==1.4.4
+langchain==0.0.139
+linkify-it-py==1.0.3
+lxml==4.9.2
+lz4==4.3.2
+markdown-it-py==2.1.0
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.0
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.4
+mdurl==0.1.2
+monotonic==1.6
+multidict==6.0.4
+mypy-extensions==1.0.0
+nest-asyncio==1.5.6
+nltk==3.8.1
+numpy==1.21.6
+openai==0.27.4
+openapi-schema-pydantic==1.2.4
+openpyxl==3.1.1
+orjson==3.8.6
+packaging==23.0
+pandas==1.5.3
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.4.0
+platformdirs==3.0.0
+posthog==2.5.0
+prompt-toolkit==3.0.36
+protobuf==3.20.3
+psutil==5.9.4
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==11.0.0
+pycryptodome==3.17
+pydantic==1.10.5
+pydeck==0.8.0
+pydub==0.25.1
+Pygments==2.14.0
+Pympler==1.0.1
+pyparsing==3.0.9
+pypdf2==3.0.1
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==0.21.1
+python-magic==0.4.27
+python-multipart==0.0.5
+python-pptx==0.6.21
+pytz==2022.7.1
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+pyzmq==25.0.0
+regex==2022.10.31
+requests==2.28.2
+rfc3986==1.5.0
+rich==13.3.1
+scikit-learn==1.2.1
+scipy==1.10.0
+semver==2.13.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.97
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+SQLAlchemy==1.4.46
+stack-data==0.6.2
+starlette==0.20.4
+streamlit==1.18.1
+tenacity==8.2.1
+threadpoolctl==3.1.0
+tiktoken==0.3.3
+tokenizers==0.13.2
+toml==0.10.2
+toolz==0.12.0
+torch==1.13.1
+torchvision==0.14.1
+tornado==6.2
+tqdm==4.64.1
+traitlets==5.9.0
+transformers==4.26.1
+typing-extensions==4.5.0
+typing-inspect==0.8.0
+tzdata==2022.7
+tzlocal==4.2
+uc-micro-py==1.0.1
+unstructured==0.4.11
+urllib3==1.26.14
+uvicorn==0.18.3
+uvloop==0.17.0
+validators==0.20.0
+watchfiles==0.18.1
+wcwidth==0.2.6
+websockets==10.4
+wrapt==1.14.1
+XlsxWriter==3.0.8
+yarl==1.8.2
+zipp==3.14.0
+zstandard==0.19.0