initial commit

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+.streamlit/

README.md

@@ -1,8 +1,8 @@
 ---
-title: Index Demo
-emoji: 🏃
-colorFrom: pink
-colorTo: green
+title: Indexing Demo
+emoji: 📈
+colorFrom: green
+colorTo: pink
 sdk: streamlit
 sdk_version: 1.10.0
 app_file: app.py

app.py

@@ -0,0 +1,106 @@
+import streamlit as st
+from transformers import pipeline
+from sentence_transformers import CrossEncoder
+import requests
+from bs4 import BeautifulSoup
+from functools import reduce
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelWithLMHead, pipeline
+import openai
+
+all_documents = {}
+
+def qa_gpt3(question, context):
+    openai.api_key = st.secrets["openai_key"]
+
+    response = openai.Completion.create(
+      model="text-davinci-002",
+      prompt=f"Given this context, answer a question. If you cannot find an answer say \"Unknown\".\n\nContext: {context}\n\nQuestion: {query}",
+      temperature=0.7,
+      max_tokens=256,
+      top_p=1,
+      frequency_penalty=0,
+      presence_penalty=0
+    )
+    print(response)
+    return {'answer': response['choices'][0]['text'].strip()}
+
+st.title('Document Question Answering System')
+
+qa_model = None
+
+crawl_urls = st.checkbox('Crawl?', value=False)
+
+document_text = st.text_area(
+label="Links (Comma separated)", height=100,
+value='https://www.databricks.com/blog/2022/11/15/values-define-databricks-culture.html, https://databricks.com/product/databricks-runtime-for-machine-learning/faq'
+)
+query = st.text_input("Query")
+
+qa_option = st.selectbox('Q/A Answerer', ('gpt3', 'a-ware/bart-squadv2'))
+
+if qa_option == 'gpt3':
+    qa_model = qa_gpt3
+else:
+    qa_model = pipeline("question-answering", qa_option)
+st.write(f'Using {qa_option} as the Q/A model')
+
+encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+
+def get_relevent_passage(question, documents):
+    query_paragraph_list = [(question, para) for para in list(documents.keys()) if len(para.strip()) > 0]
+
+    scores = encoder.predict(query_paragraph_list)
+    top_5_indices = scores.argsort()[-5:]
+    top_5_query_paragraph_list = [query_paragraph_list[i] for i in top_5_indices]
+    top_5_query_paragraph_list.reverse()
+    return top_5_query_paragraph_list[0][1]
+
+
+def answer_question(query, context):
+    answer = qa_model(question=query, context=context)['answer']
+    return answer
+
+
+def get_documents(document_text, crawl=crawl_urls):
+    urls = document_text.split(',')
+    for url in urls:
+        st.write(f'Crawling {url}')
+        if url in set(all_documents.values()):
+            continue
+        html = requests.get(url).text
+        soup = BeautifulSoup(html, 'html.parser')
+
+        if crawl:
+            st.write('Give me a sec, crawling..')
+            import re
+
+            more_urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', html)
+            more_urls = list(set([m for m in more_urls if m[-4] != '.' and m[-3] != '.' and m.split('/')[:3] == url.split('/')[:3]]))
+            for more_url in more_urls:
+                all_documents.update(get_documents(more_url, crawl=False))
+
+        body = soup.get_text()
+
+        document_paragraphs = [body]  # TODO change this to tokenize docs differently
+        for document_paragraph in document_paragraphs:
+            all_documents[document_paragraph] = url
+
+    return all_documents
+
+
+if len(document_text.strip()) > 0 and len(query.strip()) > 0 and qa_model and encoder:
+    st.write('Hmmm let me think about that..')
+    document_text = document_text.strip()
+    documents = get_documents(document_text)
+    st.write(f'I am looking through {len(set(documents.values()))} sites')
+
+    query = query.strip()
+    context = get_relevent_passage(query, documents)
+    answer = answer_question(query, context)
+
+    relevant_url = documents[context]
+
+    st.write('Check the answer below...with reference text')
+    st.header("ANSWER: "+answer)
+    st.subheader("REFERENCE: "+context)
+    st.subheader("REFERENCE URL: "+relevant_url)

requirements.txt

@@ -0,0 +1,8 @@
+numpy
+torch
+transformers
+sentence-transformers
+streamlit
+openai
+bs4
+requests