File size: 4,781 Bytes
13874d9 48b2c9b 13874d9 48b2c9b f887d37 48b2c9b 13874d9 5a5d377 13874d9 48b2c9b 13874d9 48b2c9b 13874d9 2525a71 13874d9 135090c 13874d9 618813d 13874d9 618813d 13874d9 1835782 1f69a30 1835782 1f69a30 13874d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | import operations as op
import textwrap
# from dotenv import load_dotenv
import streamlit as st
import os
import nltk
from nltk.tokenize import sent_tokenize
import openai
nltk.download('punkt')
openai.organization = os.environ['org']
openai.api_key = os.environ['api_key']
st.title("Mobius")
# st.write("AI Powered Smart Search System")
st.markdown("""
<style>
.big-font {
font-size:14px;
}
</style>
""", unsafe_allow_html=True)
st.markdown("""<p class="big-font">Mobius is the perfect way for companies to easily organize, access and leverage their data. With Mobius, companies can quickly and intuitively search their data to answer any question, while still staying organized and on track. Mobius' powerful semantic search capabilities cut through all the noise and access only the data that's necessary to get the answers companies need. With Mobius, companies can be smarter and more efficient in the way they work with data.</p>""", unsafe_allow_html=True)
# st.subheader(
# "With Mobius, companies can be smarter and more efficient in the way they work with data.")
top_match_sentences = []
@ st.cache_data
def process_pdf_data(uploaded_files):
"""
The function accepts an uploaded PDF file as input and then proceeds to extract, preprocess, and vectorize the text. It ultimately returns a list of filtered sentences and their respective embeddings.
Parameters:
uploaded_files (list): List of uploaded files
Returns:
filt1_list (list): List of filtered sentences
embeddings (list): List of embeddings of the sentences
"""
filt1_list = []
embeddings = []
text_ext = []
for i in uploaded_files:
if i.type == "application/pdf":
# Reading the pdf file and extracting the text
text_ext += op.read_pdf(i)
# Applying sent_tokenize to the text and storing the result in a list
sent_toks = []
for i in text_ext:
sent_toks.append(sent_tokenize(i))
concat_list = [j for i in sent_toks for j in i]
# Removing the new line characters from the list
for i in concat_list:
a = (i.replace('\n', ' '))
filt1_list.append(a)
# Creating embeddings for the sentences
embeddings = op.create_content_embeddings(filt1_list)
return filt1_list, embeddings
# Streamlit code to upload files
uploaded_files = st.file_uploader(
"Upload files - ", accept_multiple_files=True, type=['pdf'])
if st.button("Process!"):
if len(uploaded_files) != 0:
# Calling the function process_pdf_data to process the uploaded files
filt1_list, embeddings = process_pdf_data(uploaded_files)
st.write("Process Completed")
else:
st.warning("Please upload a PDF file.")
# Streamlit code to take user input after vectorization of the documents
query = st.text_input('Ask me anything!', placeholder='Type.....')
try:
if st.button("Confirm!"):
cached_data = process_pdf_data(uploaded_files)
filt1_list = cached_data[0]
embeddings = cached_data[1]
# Creating embeddings for the query
query_embedding = op.create_query_embeddings(query)
# Calculating cosine similarity between the query and the sentences
cosine_lis = op.calculate_cosine(
query_embedding, embeddings, filt1_list)
# Fetching the top 15 sentences with the highest cosine similarity
indexes_final = op.fetch_top_rank_ans(cosine_lis, 15)
# Fetching the most relevant sentence from the top 15 sentences, and providing it as the context to the GPT-3 model
most_relevant = op.fetch_most_relevant(
indexes_final, filt1_list, cosine_lis, query)
# Calling the GPT-3 model to generate the answer
# COMPLETIONS_API_PARAMS = {
# "temperature": 0.0,
# "max_tokens": 300,
# "model": "text-davinci-003",
# }
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are an educational assistant. Answer the questions with detail and in 200-300 characters only"},
{"role": "user", "content": most_relevant},
]
)
# print("\n\n", textwrap.fill(
# response["choices"][0]["text"].strip(" \n")))
# Displaying the answer to the user
# print(response['choices'][0].message.content)
st.write(textwrap.fill(response['choices'][0].message.content))
# import streamlit_scrollable_textbox as stx
# long_text = response['choices'][0].message.content
# stx.scrollableTextbox(long_text, height=300)
except Exception as e:
print(e)
st.warning("Something went wrong. Please try again.")
|