Upload 4 files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dataset.tsv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,18 @@
+import streamlit as st
+from question_processing import process_question
+
+st.title("Question Answering System")
+st.write("Enter your question and get an answer from the pre-trained model.")
+
+# Input field for the user's question
+question = st.text_input("Please enter your question:")
+
+# Process the question and display the answer(s) when the user clicks the "Submit" button
+if st.button("Submit"):
+    if question:
+        answers = process_question(question)
+        for answer in answers:
+            st.write("Answer:", answer)
+            st.write("---")
+    else:
+        st.write("Please enter a question.")

dataset.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e88b7f47f3494171367face846d1dcaf2710854870b076d6d419b8bae720bf1
+size 28877451

question_processing.py

@@ -0,0 +1,91 @@
+# Import necessary libraries
+import os
+import textwrap
+import pandas as pd
+from langchain import HuggingFaceHub
+from langchain.document_loaders import TextLoader
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.chains.question_answering import load_qa_chain
+from transformers import AutoTokenizer
+
+def wrap_text_preserve_newlines(text, width=110):
+    lines = text.split('\n')
+    wrapped_lines = [textwrap.fill(line, width=width) for line in lines]
+    wrapped_text = '\n'.join(wrapped_lines)
+    return wrapped_text
+
+def split_into_chunks(text, tokenizer, max_tokens=500):
+    tokens = tokenizer.encode(text, return_tensors="pt").squeeze()
+    token_chunks = []
+
+    current_chunk = []
+    current_chunk_len = 0
+    for token in tokens:
+        token_len = len(tokenizer.decode(token.item()))
+        if current_chunk_len + token_len + 1 > max_tokens:
+            token_chunks.append(tokenizer.decode(current_chunk))
+            current_chunk = []
+            current_chunk_len = 0
+        current_chunk.append(token.item())
+        current_chunk_len += token_len + 1
+
+    if current_chunk:
+        token_chunks.append(tokenizer.decode(current_chunk))
+
+    return token_chunks
+
+tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
+
+class TextDocument:
+    def __init__(self, content, id, metadata=None):
+        self.page_content = content
+        self.metadata = metadata if metadata is not None else {}
+        self.metadata['id'] = id
+
+os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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"
+
+data_frame = pd.read_csv("dataset.tsv", sep="\t", nrows=1000)
+data = data_frame.to_dict(orient="records")
+documents = [TextDocument(content=str(item["answer"]), id=item["id"]) for item in data]
+text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=0)
+docs = text_splitter.split_documents(documents)
+embeddings = HuggingFaceEmbeddings()
+db = FAISS.from_documents(docs, embeddings)
+
+llm = HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature": 0.75, "max_length": 2048})
+chain = load_qa_chain(llm, chain_type="refine")
+
+def truncate_answer(answer, question, tokenizer, max_total_tokens=1000):
+    special_tokens = 2
+    question_tokens = len(tokenizer.encode(question, return_tensors="pt").squeeze())
+    max_answer_tokens = max_total_tokens - question_tokens - special_tokens
+    answer_tokens = tokenizer.encode(answer, return_tensors="pt").squeeze()
+    truncated_answer = tokenizer.decode(answer_tokens[:max_answer_tokens])
+    return truncated_answer
+
+def combined_length_exceeds_limit(question, answer, tokenizer, model_token_limit=1024):
+    special_tokens = 2
+    question_tokens = len(tokenizer.encode(question, return_tensors="pt").squeeze())
+    answer_tokens = len(tokenizer.encode(answer, return_tensors="pt").squeeze())
+    return question_tokens + answer_tokens > (model_token_limit - special_tokens)
+
+def process_question(query):
+    answers = []
+
+    docs = db.similarity_search(query)
+    most_similar_doc = docs[0]
+    print(f"Most similar answer: \n{wrap_text_preserve_newlines(str(most_similar_doc.page_content))}\n")
+
+    query_chunks = split_into_chunks(query, tokenizer, max_tokens=500)
+
+    for query_chunk in query_chunks:
+        if combined_length_exceeds_limit(query_chunk, str(docs[0].page_content), tokenizer):
+            print("The combined length of the question and answer exceeds the model's token limit.")
+        else:
+            truncated_answer = truncate_answer(str(docs[0].page_content), query_chunk, tokenizer, max_total_tokens=500)
+            result = chain.run(input_documents=[TextDocument(content=truncated_answer, id=docs[0].metadata['id'])], question=query_chunk)
+            answers.append(result)
+
+    return answers

requirements.txt

@@ -0,0 +1,5 @@
+transformers==4.27.1
+torch>=1.13.1
+datasets==2.10.1
+
+tqdm==4.65.0