Update tools.py

tools.py

@@ -150,9 +150,11 @@ def load_scopus_csv(filepath: str) -> str:
     abstract_sentences = list(
         filter(None, sum(map(_split_sentences, df["Abstract"].dropna().tolist()), []))
     )
-    title_sentences = list(
-        filter(None, sum(map(_split_sentences, df["Title"].dropna().tolist()), []))
-    )
+    # Titles are atomic units — count each non-empty title as one unit (no sent_tokenize)
+    title_sentences = list(filter(
+        lambda s: len(s.strip()) >= 5,
+        list(map(lambda t: _clean_sentence(str(t)), df["Title"].dropna().tolist()))
+    ))
 
     stats = {
         "papers": paper_count,
@@ -168,7 +170,7 @@ def load_scopus_csv(filepath: str) -> str:
         f"✅ CSV loaded successfully.\n"
         f"📄 Papers: {paper_count}\n"
         f"📝 Abstract sentences (after cleaning): {len(abstract_sentences)}\n"
-        f"🔤 Title sentences (after cleaning): {len(title_sentences)}\n"
+        f"🔤 Title records (after cleaning): {len(title_sentences)}\n"
         f"📅 Year range: {stats['year_range']}\n"
         f"📊 Columns: {', '.join(stats['columns'])}\n\n"
         f"Data is ready. Please type **'run abstract'** to begin Phase 2 BERTopic analysis on abstracts."
@@ -187,10 +189,17 @@ def run_bertopic_discovery(run_key: str, threshold: float = 0.7) -> str:
     texts = sum(
         list(map(lambda col: df[col].dropna().tolist(), columns)), []
     )
-    sentences = list(
-        filter(lambda s: len(s) > 20,
-               sum(list(map(_split_sentences, texts)), []))
-    )
+
+    # Titles are already single semantic units — do NOT split into sentences.
+    # Abstracts get split into sentences for finer-grained clustering.
+    # Min-length: 5 chars for titles, 20 chars for abstract sentences.
+    sentences = list(filter(
+        lambda s: len(s.strip()) >= 5,
+        list(map(lambda t: _clean_sentence(str(t)), texts))
+    )) if run_key == "title" else list(filter(
+        lambda s: len(s) > 20,
+        sum(list(map(_split_sentences, texts)), [])
+    ))
 
     model = SentenceTransformer("all-MiniLM-L6-v2")
     embeddings = model.encode(sentences, normalize_embeddings=True, show_progress_bar=False)