Upload 2 files

app.py

@@ -49,45 +49,45 @@ with open(data_path, "r") as f:
 # Pre-compute corpus embeddings
 import re
 
-def split_into_sentences(text):
-    """Splits a paragraph into sentences based on capitalization and punctuation."""
-    # This regex looks for a capital letter, followed by anything that's not a period, 
-    # exclamation mark, or question mark, and then ends with one of those punctuation marks.
-    sentences = re.findall(r'([A-Z][^.!?]*[.!?])', text)
-    return sentences
-
-def create_overlapped_chunks(corpus_documents, chunk_size=2, overlap=1):
-    chunked_corpus = []
-    for doc_idx, doc_text in enumerate(corpus_documents):
-        sentences = split_into_sentences(doc_text)
-        if not sentences:
-            continue
-
-        # If there are fewer sentences than chunk_size, just use the whole document as one chunk
-        if len(sentences) < chunk_size:
-            chunked_corpus.append({
-                "text": doc_text,
-                "original_doc_idx": doc_idx,
-                "start_sentence_idx": 0,
-                "end_sentence_idx": len(sentences) - 1
-            })
-            continue
-
-        for i in range(0, len(sentences) - chunk_size + 1, chunk_size - overlap):
-            chunk_sentences = sentences[i : i + chunk_size]
-            chunk_text = " ".join(chunk_sentences)
-            chunked_corpus.append({
-                "text": chunk_text,
-                "original_doc_idx": doc_idx,
-                "start_sentence_idx": i,
-                "end_sentence_idx": i + chunk_size - 1
-            })
-    return chunked_corpus
-
-def process_documents_for_chunking(documents):
-    chunked_corpus_data = create_overlapped_chunks(documents)
-    flat_corpus_chunks = [item["text"] for item in chunked_corpus_data]
-    return chunked_corpus_data, flat_corpus_chunks
+# def split_into_sentences(text):
+#     """Splits a paragraph into sentences based on capitalization and punctuation."""
+#     # This regex looks for a capital letter, followed by anything that's not a period, 
+#     # exclamation mark, or question mark, and then ends with one of those punctuation marks.
+#     sentences = re.findall(r'([A-Z][^.!?]*[.!?])', text)
+#     return sentences
+
+# def create_overlapped_chunks(corpus_documents, chunk_size=2, overlap=1):
+#     chunked_corpus = []
+#     for doc_idx, doc_text in enumerate(corpus_documents):
+#         sentences = split_into_sentences(doc_text)
+#         if not sentences:
+#             continue
+
+#         # If there are fewer sentences than chunk_size, just use the whole document as one chunk
+#         if len(sentences) < chunk_size:
+#             chunked_corpus.append({
+#                 "text": doc_text,
+#                 "original_doc_idx": doc_idx,
+#                 "start_sentence_idx": 0,
+#                 "end_sentence_idx": len(sentences) - 1
+#             })
+#             continue
+
+#         for i in range(0, len(sentences) - chunk_size + 1, chunk_size - overlap):
+#             chunk_sentences = sentences[i : i + chunk_size]
+#             chunk_text = " ".join(chunk_sentences)
+#             chunked_corpus.append({
+#                 "text": chunk_text,
+#                 "original_doc_idx": doc_idx,
+#                 "start_sentence_idx": i,
+#                 "end_sentence_idx": i + chunk_size - 1
+#             })
+#     return chunked_corpus
+
+# def process_documents_for_chunking(documents):
+#     chunked_corpus_data = create_overlapped_chunks(documents)
+#     flat_corpus_chunks = [item["text"] for item in chunked_corpus_data]
+#     return chunked_corpus_data, flat_corpus_chunks
 
 # Pre-compute corpus embeddings
 original_corpus = [item["positive"] for item in dataset]
@@ -147,8 +147,8 @@ def find_similar(prompt, top_k):
     end_time = time.time()
 
     results = []
-    # for doc, score in  doc_score_pairs[:top_k]:
-    for doc, score in  doc_score_pairs:
+    for doc, score in  doc_score_pairs[:top_k]:
+    # for doc, score in  doc_score_pairs:
         results.append((score, doc))
 
     return results, f"{(end_time - start_time) * 1000:.2f} ms"

requirements.txt

@@ -1,4 +1,5 @@
 gradio
 transformers
+sentence_transformers
 torch
 huggingface_hub