added packages for linux

packages.txt

@@ -0,0 +1,2 @@
+poppler-utils
+xpdf

search.py

@@ -0,0 +1,60 @@
+
+
+import pinecone
+index_name = "abstractive-question-answering"
+
+# check if the abstractive-question-answering index exists
+if index_name not in pinecone.list_indexes():
+    # create the index if it does not exist
+    pinecone.create_index(
+        index_name,
+        dimension=768,
+        metric="cosine"
+    )
+
+# connect to abstractive-question-answering index we created
+index = pinecone.Index(index_name)
+
+# we will use batches of 64
+batch_size = 64
+
+for i in tqdm(range(0, len(df), batch_size)):
+    # find end of batch
+    i_end = min(i+batch_size, len(df))
+    # extract batch
+    batch = df.iloc[i:i_end]
+    # generate embeddings for batch
+    emb = retriever.encode(batch["passage_text"].tolist()).tolist()
+    # get metadata
+    meta = batch.to_dict(orient="records")
+    # create unique IDs
+    ids = [f"{idx}" for idx in range(i, i_end)]
+    # add all to upsert list
+    to_upsert = list(zip(ids, emb, meta))
+    # upsert/insert these records to pinecone
+    _ = index.upsert(vectors=to_upsert)
+
+# check that we have all vectors in index
+index.describe_index_stats()
+
+# from transformers import BartTokenizer, BartForConditionalGeneration
+
+# # load bart tokenizer and model from huggingface
+# tokenizer = BartTokenizer.from_pretrained('vblagoje/bart_lfqa')
+# generator = BartForConditionalGeneration.from_pretrained('vblagoje/bart_lfqa')
+
+# def query_pinecone(query, top_k):
+#     # generate embeddings for the query
+#     xq = retriever.encode([query]).tolist()
+#     # search pinecone index for context passage with the answer
+#     xc = index.query(xq, top_k=top_k, include_metadata=True)
+#     return xc
+
+# def format_query(query, context):
+#     # extract passage_text from Pinecone search result and add the  tag
+#     context = [f" {m['metadata']['passage_text']}" for m in context]
+#     # concatinate all context passages
+#     context = " ".join(context)
+#     # contcatinate the query and context passages
+#     query = f"question: {query} context: {context}"
+#     return query