File size: 5,035 Bytes
13874d9 618813d 135090c 13874d9 135090c 13874d9 135090c 13874d9 618813d 13874d9 618813d 13874d9 32afc1c 13874d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | from numpy.linalg import norm
import numpy as np
from sentence_transformers import SentenceTransformer
import PyPDF2
from nltk.tokenize import sent_tokenize
def read_pdf(fname):
"""
This function reads the pdf file and extracts the text from it.
Parameters:
fname (str): Name of the pdf file
Returns:
text_ext (list): List of extracted text from the pdf file
"""
reader = PyPDF2.PdfReader(fname)
text_ext = []
for i in range(len(reader.pages)):
pageObj = reader.pages[i]
# extracting text from page
text_ext.append(pageObj.extract_text())
return text_ext
def sent_tokenize(text_ext):
"""
This function apply sent_tokenize to the text and stores the result in a list.
Parameters:
text_ext (list): List of extracted text from the pdf file
Returns:
sent_toks (list): List of tokenized sentences
"""
sent_toks = []
for i in text_ext:
sent_toks.append(sent_tokenize(i))
print("len(sent_toks) ", len(sent_toks))
return sent_toks
def create_content_embeddings(concat_list):
"""
This function creates embeddings for the document sentences.
Parameters:
concat_list (list): List of tokenized sentences
Returns:
embeddings (list): List of embeddings of the sentences
"""
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
embeddings = model.encode(concat_list)
return embeddings
def create_query_embeddings(query_text):
"""
This function creates embeddings for the query.
Parameters:
query_text (str): Query entered by the user
Returns:
query_embedding (list): List of embeddings of the query
"""
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
query_embedding = model.encode(query_text)
return query_embedding
def calculate_cosine(query_embedding, embeddings, concat_list):
"""
This function calculates cosine similarity between the query and the sentences.
Parameters:
query_embedding (list): List of embeddings of the query
embeddings (list): List of embeddings of the sentences
concat_list (list): List of tokenized sentences
Returns:
cosine_lis (list): List of cosine similarity values
"""
cosine_lis = []
for i in range(len(concat_list)):
cosine = np.dot(query_embedding,
embeddings[i]) / (norm(query_embedding)*norm(embeddings[i]))
cosine_lis.append(cosine)
# print("cosine_lis ", cosine_lis)
return (cosine_lis)
def fetch_top_rank_ans(cosine_lis, N):
"""
This function fetches the top N ranked sentences.
Parameters:
cosine_lis (list): List of cosine similarity values
N (int): Number of sentences to be ranked
Returns:
indexes_final (list): List of top N ranked sentences
"""
list1 = cosine_lis
indexes_final = sorted(
range(len(list1)), key=lambda i: list1[i], reverse=True)[:N]
print("indexes_final ", indexes_final, len(indexes_final))
indices = len(list1)
sorted_indices = sorted(range(indices),
key=lambda i: list1[i], reverse=True)
print(indices, indices)
if indices < N:
N = indices
indexes_final = []
for i in range(N):
indexes_final.append(sorted_indices[i])
len(indexes_final)
return indexes_final
def fetch_most_relevant(indexes_final, concat_list, list1, query):
"""
This function fetches the most relevant sentences, pass it as a context to GPT-3 prompt along with user's query.
Parameters:
indexes_final (list): List of top N ranked sentences
concat_list (list): List of tokenized sentences
list1 (list): List of cosine similarity values
query (str): Query entered by the user
Returns:
prompt (str): GPT-3 prompt
"""
dicts = {}
keys = indexes_final
# print(indexes_final)
for i in keys:
dicts[i] = concat_list[i]
most_relevant_document_sections = [dicts]
len(most_relevant_document_sections)
chosen_sections = []
chosen_sections_len = 0
chosen_sections_indexes = []
indices = range(len(list1))
sorted_indices = sorted(indices, key=lambda i: list1[i], reverse=True)
print(indexes_final, len(indexes_final))
for section_index in range(len(indexes_final)):
if chosen_sections_len > 500:
break
chosen_sections.append(
concat_list[sorted_indices[section_index]].replace("\n", " "))
chosen_sections_indexes.append(str(section_index))
# Useful diagnostic information
print(f"Selected {len(chosen_sections)} document sections:")
header = """Answer the question as a human in natural language conversation using the provided context, and if the answer is not contained within the text below, say "I don't have that information"\n\nContext:\n"""
# print(query)
prompt = header + "".join(chosen_sections) + "\n\n Q: " + query + "\n A:"
return prompt
|