operations.py

from numpy.linalg import norm
import numpy as np
from sentence_transformers import SentenceTransformer
import PyPDF2
from nltk.tokenize import sent_tokenize


def read_pdf(fname):
    """
    This function reads the pdf file and extracts the text from it.

    Parameters:
    fname (str): Name of the pdf file

    Returns:
    text_ext (list): List of extracted text from the pdf file
    """
    reader = PyPDF2.PdfReader(fname)
    text_ext = []
    for i in range(len(reader.pages)):
        pageObj = reader.pages[i]
        # extracting text from page
        text_ext.append(pageObj.extract_text())

    return text_ext


def sent_tokenize(text_ext):
    """
    This function apply sent_tokenize to the text and stores the result in a list.

    Parameters:
    text_ext (list): List of extracted text from the pdf file

    Returns:
    sent_toks (list): List of tokenized sentences
    """
    sent_toks = []

    for i in text_ext:
        sent_toks.append(sent_tokenize(i))
    print("len(sent_toks) ", len(sent_toks))

    return sent_toks


def create_content_embeddings(concat_list):
    """
    This function creates embeddings for the document sentences.

    Parameters:
    concat_list (list): List of tokenized sentences

    Returns:
    embeddings (list): List of embeddings of the sentences
    """
    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
    embeddings = model.encode(concat_list)

    return embeddings


def create_query_embeddings(query_text):
    """
    This function creates embeddings for the query.
    Parameters:
    query_text (str): Query entered by the user

    Returns:
    query_embedding (list): List of embeddings of the query
    """

    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
    query_embedding = model.encode(query_text)
    return query_embedding


def calculate_cosine(query_embedding, embeddings, concat_list):
    """

    This function calculates cosine similarity between the query and the sentences.

    Parameters:
    query_embedding (list): List of embeddings of the query
    embeddings (list): List of embeddings of the sentences
    concat_list (list): List of tokenized sentences

    Returns:
    cosine_lis (list): List of cosine similarity values
    """
    cosine_lis = []

    for i in range(len(concat_list)):
        cosine = np.dot(query_embedding,
                        embeddings[i]) / (norm(query_embedding)*norm(embeddings[i]))
        cosine_lis.append(cosine)

    # print("cosine_lis ", cosine_lis)
    return (cosine_lis)


def fetch_top_rank_ans(cosine_lis, N):
    """
    This function fetches the top N ranked sentences.

    Parameters:
    cosine_lis (list): List of cosine similarity values
    N (int): Number of sentences to be ranked

    Returns:
    indexes_final (list): List of top N ranked sentences
    """

    list1 = cosine_lis
    indexes_final = sorted(
        range(len(list1)), key=lambda i: list1[i], reverse=True)[:N]

    print("indexes_final ", indexes_final, len(indexes_final))
    indices = len(list1)

    sorted_indices = sorted(range(indices),
                            key=lambda i: list1[i], reverse=True)
    print(indices, indices)

    if indices < N:
        N = indices
    indexes_final = []

    for i in range(N):
        indexes_final.append(sorted_indices[i])
    len(indexes_final)
    return indexes_final


def fetch_most_relevant(indexes_final, concat_list, list1, query):
    """
    This function fetches the most relevant sentences, pass it as a context to GPT-3 prompt along with user's query.

    Parameters:
    indexes_final (list): List of top N ranked sentences
    concat_list (list): List of tokenized sentences
    list1 (list): List of cosine similarity values
    query (str): Query entered by the user

    Returns:
    prompt (str): GPT-3 prompt
    """

    dicts = {}

    keys = indexes_final
    # print(indexes_final)
    for i in keys:
        dicts[i] = concat_list[i]

    most_relevant_document_sections = [dicts]

    len(most_relevant_document_sections)

    chosen_sections = []
    chosen_sections_len = 0
    chosen_sections_indexes = []

    indices = range(len(list1))
    sorted_indices = sorted(indices, key=lambda i: list1[i], reverse=True)
    print(indexes_final, len(indexes_final))

    for section_index in range(len(indexes_final)):

        if chosen_sections_len > 500:
            break
        chosen_sections.append(
            concat_list[sorted_indices[section_index]].replace("\n", " "))
        chosen_sections_indexes.append(str(section_index))

    # Useful diagnostic information
    print(f"Selected {len(chosen_sections)} document sections:")

    header = """Answer the question as a human in natural language conversation using the provided context, and if the answer is not contained within the text below, say "I don't have that information"\n\nContext:\n"""

    # print(query)
    prompt = header + "".join(chosen_sections) + "\n\n Q: " + query + "\n A:"
    return prompt