laitkor commited on
Commit
f8d5779
·
verified ·
1 Parent(s): 1b2c612

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +13 -0
app.py CHANGED
@@ -38,7 +38,10 @@ def load_documents(docs_path):
38
  documents.append(text)
39
  return documents
40
 
 
41
  documents = load_documents(docs_path)
 
 
42
 
43
  # Load model and tokenizer for encoding documents
44
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
@@ -51,15 +54,23 @@ def encode_documents(documents):
51
  embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
52
  return embeddings
53
 
 
54
  embeddings = encode_documents(documents)
 
 
55
 
 
56
  # Index embeddings using FAISS
57
  index = faiss.IndexFlatL2(embeddings.shape[1])
58
  index.add(embeddings)
 
59
 
 
60
  # Load T5 model and tokenizer for question generation
61
  t5_tokenizer = T5Tokenizer.from_pretrained("t5-small")
62
  t5_model = T5ForConditionalGeneration.from_pretrained("t5-small")
 
 
63
 
64
  def generate_questions(text):
65
  input_text = f"generate question: {text}"
@@ -73,9 +84,11 @@ def retrieve_documents(query, index, documents):
73
  D, I = index.search(query_embedding, k=5)
74
  return [documents[i] for i in I[0]]
75
 
 
76
  # Load GPT-2 model and tokenizer for answer generation
77
  gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
78
  gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2")
 
79
 
80
  def generate_answer(question, context):
81
  input_text = f"Question: {question}\nContext: {context}\nAnswer:"
 
38
  documents.append(text)
39
  return documents
40
 
41
+ print ("loading documents")
42
  documents = load_documents(docs_path)
43
+ print ("done loading documents")
44
+
45
 
46
  # Load model and tokenizer for encoding documents
47
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
 
54
  embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
55
  return embeddings
56
 
57
+ print ("encoding documents")
58
  embeddings = encode_documents(documents)
59
+ print ("done encoding documents")
60
+
61
 
62
+ print ("vector faiss documents")
63
  # Index embeddings using FAISS
64
  index = faiss.IndexFlatL2(embeddings.shape[1])
65
  index.add(embeddings)
66
+ print ("done vector documents")
67
 
68
+ print ("start tokenizing documents")
69
  # Load T5 model and tokenizer for question generation
70
  t5_tokenizer = T5Tokenizer.from_pretrained("t5-small")
71
  t5_model = T5ForConditionalGeneration.from_pretrained("t5-small")
72
+ print ("done tokenizing documents")
73
+
74
 
75
  def generate_questions(text):
76
  input_text = f"generate question: {text}"
 
84
  D, I = index.search(query_embedding, k=5)
85
  return [documents[i] for i in I[0]]
86
 
87
+ print ("loading gpt-2")
88
  # Load GPT-2 model and tokenizer for answer generation
89
  gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
90
  gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2")
91
+ print ("done gpt-2")
92
 
93
  def generate_answer(question, context):
94
  input_text = f"Question: {question}\nContext: {context}\nAnswer:"