Update app.py
Browse files
app.py
CHANGED
|
@@ -38,7 +38,10 @@ def load_documents(docs_path):
|
|
| 38 |
documents.append(text)
|
| 39 |
return documents
|
| 40 |
|
|
|
|
| 41 |
documents = load_documents(docs_path)
|
|
|
|
|
|
|
| 42 |
|
| 43 |
# Load model and tokenizer for encoding documents
|
| 44 |
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
|
|
@@ -51,15 +54,23 @@ def encode_documents(documents):
|
|
| 51 |
embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
|
| 52 |
return embeddings
|
| 53 |
|
|
|
|
| 54 |
embeddings = encode_documents(documents)
|
|
|
|
|
|
|
| 55 |
|
|
|
|
| 56 |
# Index embeddings using FAISS
|
| 57 |
index = faiss.IndexFlatL2(embeddings.shape[1])
|
| 58 |
index.add(embeddings)
|
|
|
|
| 59 |
|
|
|
|
| 60 |
# Load T5 model and tokenizer for question generation
|
| 61 |
t5_tokenizer = T5Tokenizer.from_pretrained("t5-small")
|
| 62 |
t5_model = T5ForConditionalGeneration.from_pretrained("t5-small")
|
|
|
|
|
|
|
| 63 |
|
| 64 |
def generate_questions(text):
|
| 65 |
input_text = f"generate question: {text}"
|
|
@@ -73,9 +84,11 @@ def retrieve_documents(query, index, documents):
|
|
| 73 |
D, I = index.search(query_embedding, k=5)
|
| 74 |
return [documents[i] for i in I[0]]
|
| 75 |
|
|
|
|
| 76 |
# Load GPT-2 model and tokenizer for answer generation
|
| 77 |
gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
|
| 78 |
gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2")
|
|
|
|
| 79 |
|
| 80 |
def generate_answer(question, context):
|
| 81 |
input_text = f"Question: {question}\nContext: {context}\nAnswer:"
|
|
|
|
| 38 |
documents.append(text)
|
| 39 |
return documents
|
| 40 |
|
| 41 |
+
print ("loading documents")
|
| 42 |
documents = load_documents(docs_path)
|
| 43 |
+
print ("done loading documents")
|
| 44 |
+
|
| 45 |
|
| 46 |
# Load model and tokenizer for encoding documents
|
| 47 |
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
|
|
|
|
| 54 |
embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
|
| 55 |
return embeddings
|
| 56 |
|
| 57 |
+
print ("encoding documents")
|
| 58 |
embeddings = encode_documents(documents)
|
| 59 |
+
print ("done encoding documents")
|
| 60 |
+
|
| 61 |
|
| 62 |
+
print ("vector faiss documents")
|
| 63 |
# Index embeddings using FAISS
|
| 64 |
index = faiss.IndexFlatL2(embeddings.shape[1])
|
| 65 |
index.add(embeddings)
|
| 66 |
+
print ("done vector documents")
|
| 67 |
|
| 68 |
+
print ("start tokenizing documents")
|
| 69 |
# Load T5 model and tokenizer for question generation
|
| 70 |
t5_tokenizer = T5Tokenizer.from_pretrained("t5-small")
|
| 71 |
t5_model = T5ForConditionalGeneration.from_pretrained("t5-small")
|
| 72 |
+
print ("done tokenizing documents")
|
| 73 |
+
|
| 74 |
|
| 75 |
def generate_questions(text):
|
| 76 |
input_text = f"generate question: {text}"
|
|
|
|
| 84 |
D, I = index.search(query_embedding, k=5)
|
| 85 |
return [documents[i] for i in I[0]]
|
| 86 |
|
| 87 |
+
print ("loading gpt-2")
|
| 88 |
# Load GPT-2 model and tokenizer for answer generation
|
| 89 |
gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
|
| 90 |
gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2")
|
| 91 |
+
print ("done gpt-2")
|
| 92 |
|
| 93 |
def generate_answer(question, context):
|
| 94 |
input_text = f"Question: {question}\nContext: {context}\nAnswer:"
|