Bug fix.

app.py

@@ -58,7 +58,7 @@ import datetime
 import gradio as gr
 from openai import OpenAI
 from langchain_community.embeddings import HuggingFaceEmbeddings
-from create_rag_retriever import load_retriever_components
+from create_rag_retriever import load_retriever_components, DPRetriever, HybridRetriever
 import numpy as np
 import hashlib
 from langchain_core.documents import Document
@@ -214,14 +214,14 @@ SCENARIO_FALLBACK_TWEET_PATHS = {
 # Default fallback tweet dataset (empty; populated per-scenario at runtime).
 FALLBACK_TWEET_DATA = []
 
-RAG_K       = 60
+RAG_K       = 50
 RAG_FETCH_K = 10000
 RAG_LINKAGE_STORED_EXCERPT_LENGTH             = 500
 ATTR_INFERENCE_EVIDENCE_STORED_EXCERPT_LENGTH = 500
 
 # ── RAG Linkage Similarity Thresholds ────────────────────────
-ROUGE_L_THRESHOLD    = 0.15   # Minimum ROUGE-L score (0.0-1.0)
-COSINE_SIM_THRESHOLD = 0.15   # Minimum cosine similarity (0.0-1.0)
+ROUGE_L_THRESHOLD    = 0.16   # Minimum ROUGE-L score (0.0-1.0)
+COSINE_SIM_THRESHOLD = 0.16   # Minimum cosine similarity (0.0-1.0)
 MIN_COMBINED_SCORE   = 0.2    # Minimum combined similarity score
 ROUGE_WEIGHT         = 0.3    # Weight for ROUGE-L in combined score
 COSINE_WEIGHT        = 0.7    # Weight for cosine similarity in combined score
@@ -229,6 +229,9 @@ MIN_NGRAM_LENGTH     = 4      # Minimum characters for n-gram matching
 MIN_NGRAM_WORDS      = 4      # Minimum words in a phrase to highlight
 MAX_NGRAM_WORDS      = 7      # Maximum words in a phrase to highlight
 
+RAG_MIN_SIMILARITY_THRESHOLD = 0.25   # baseline threshold when DP is off
+RAG_DP_THRESHOLD_SCALE       = 1.5   # how fast the threshold rises as ε falls
+
 # ── Privacy / Differential Privacy ───────────────────────────
 MAX_POSSIBLE_EPS         = 100.0
 INFERENCE_LIFT_THRESHOLD = 0
@@ -1584,8 +1587,6 @@ def build_retriever_from_csv(csv_file_path, progress=None, user_id_col="user id"
         HybridRetriever instance or None if failed
     """
     try:
-        from create_rag_retriever import HybridRetriever
-
         logger.info(f"Building retriever from CSV: {csv_file_path}")
         if progress:
             progress(0.25, desc="Loading CSV data...")
@@ -1868,108 +1869,22 @@ def strip_special_chars(text):
     """Strip digits and special characters."""
     return re.sub(r'[^a-zA-Z]', '', text.lower())
 
-
-class HybridRetriever:
-    """Hybrid retriever (semantic + BM25 + entity)."""
-
-    def __init__(self, vectorstore, bm25=None, entity_index=None, documents=None,
-                 semantic_weight=0.4, bm25_weight=0.3, entity_weight=0.3):
-        self.vectorstore = vectorstore
-        self.bm25 = bm25
-        self.entity_index = entity_index or defaultdict(list)
-        self.documents = documents or []
-        self.semantic_weight = semantic_weight
-        self.bm25_weight = bm25_weight
-        self.entity_weight = entity_weight
-
-    def retrieve(self, query, k=10, fetch_k=100):
-        """Hybrid retrieval."""
-        # Simplified: just use semantic search
-        # Full implementation would combine semantic + BM25 + entity
-        semantic_results = self.vectorstore.similarity_search(query, k=fetch_k)
-        return semantic_results[:k]
-
-
-class DPRetriever:
-    """Differentially private retriever using exponential mechanism.
-
-    Sensitivity is computed empirically from the actual utility range of the
-    candidate documents rather than a fixed constant.  This data-adaptive
-    approach (Dwork & Roth 2014; Koga et al. 2024 "Privacy-preserving RAG
-    with DP") gives a tighter bound, improving the privacy-utility trade-off
-    without weakening the formal ε-DP guarantee.
-
-    For the exponential mechanism the sensitivity Δu of the utility function u
-    is defined as:
-        Δu = max_{d, d'} |u(d, q) - u(d', q)|
-    When u(d, q) = max_dist - dist(d, q)  (distance-to-similarity inversion),
-    Δu equals the range of utilities across the candidate pool.  We clamp it
-    to a configurable minimum (sensitivity_cap) to guard against degenerate
-    cases where all candidates are equidistant from the query.
-
-    Parameters
-    ----------
-    base_retriever : HybridRetriever
-        The underlying retriever whose vectorstore provides FAISS scores.
-    epsilon : float
-        DP privacy budget.  Smaller = stronger privacy.
-    sensitivity_cap : float
-        Floor value for the computed sensitivity.  Set to a small positive
-        number (default 1e-3) so the mechanism stays well-defined even when
-        all utility values are identical.  For most corpora the empirical
-        range will be far larger than this cap, so it has no practical effect.
+def epsilon_adjusted_threshold(epsilon, base=RAG_MIN_SIMILARITY_THRESHOLD, scale=RAG_DP_THRESHOLD_SCALE):
     """
+    Compute a similarity threshold that tightens as epsilon falls.
 
-    def __init__(self, base_retriever, epsilon=1.0, sensitivity_cap=1e-3):
-        self.base_retriever = base_retriever
-        self.epsilon = epsilon
-        self.sensitivity_cap = sensitivity_cap
-
-    def retrieve(self, query, k=10, fetch_k=100, seed=None):
-        """Retrieve documents with differential privacy."""
-        if seed is not None:
-            np.random.seed(seed)
-
-        # Get candidates
-        try:
-            candidates = self.base_retriever.vectorstore.similarity_search_with_score(
-                query, k=fetch_k
-            )
-        except:
-            # Fallback if vectorstore doesn't support scores
-            docs = self.base_retriever.vectorstore.similarity_search(query, k=fetch_k)
-            return docs[:k]
-
-        if not candidates:
-            return []
-
-        docs, base_scores = zip(*candidates)
-        base_scores = np.array(base_scores)
+    At epsilon=inf (no DP)  → threshold = base            (e.g. 0.30)
+    At epsilon=1            → threshold ≈ base + 0.12     (e.g. 0.42)
+    At epsilon=0.1          → threshold ≈ base + 0.29     (e.g. 0.59)
 
-        # Convert L2 distances to similarities (higher is better)
-        if base_scores[0] >= base_scores[-1]:
-            utilities = base_scores.copy()  # similarity scores – higher is better
-        else:
-            max_dist = base_scores.max() + 1e-10
-            utilities = max_dist - base_scores  # distance scores – lower is better
-
-        # ── Empirical sensitivity (Dwork & Roth 2014; Koga et al. 2024) ────
-        # Sensitivity = range of the utility function across the candidate pool.
-        # This is the tightest valid bound for the exponential mechanism when
-        # the utility function is score-based and bounded by the data at hand.
-        u_range = float(utilities.max() - utilities.min())
-        sensitivity = max(u_range, self.sensitivity_cap)
-
-        # Exponential mechanism: P(d) ∝ exp(ε · u(d) / 2Δu)
-        probabilities = np.exp((self.epsilon * utilities) / (2.0 * sensitivity))
-        probabilities = probabilities / probabilities.sum()
-
-        # Sample k documents without replacement
-        selected_indices = np.random.choice(
-            len(docs), size=min(k, len(docs)), replace=False, p=probabilities
-        )
-
-        return [docs[i] for i in selected_indices]
+    The log1p curve means the boost is largest at very small epsilon
+    (strong privacy) and negligible at large epsilon (no privacy).
+    The cap at 0.92 prevents the threshold from becoming unreachable.
+    """
+    if epsilon == float("inf"):
+        return base
+    boost = scale * math.log1p(1.0 / max(epsilon, 1e-6))
+    return min(base + boost, 0.92)
 
 
 # ============================================================
@@ -4229,21 +4144,25 @@ def process_message(user_input, use_rag, epsilon, model_str, show_tips, show_rag
     logger.info(f"[LOCAL] RAG retrieval use_rag={use_rag}; retriever={retriever}")
     if use_rag and retriever is not None:
         try:
-            # if epsilon != float('inf'):
-            #     logger.info(f"  Using DP retrieval (ε={epsilon})")
-            #     dp_retriever = DPRetriever(retriever, epsilon=epsilon)
-            #     retrieved = dp_retriever.retrieve(user_input, k=RAG_K, fetch_k=RAG_FETCH_K)
-            # else:
-            #     retrieved = retriever.retrieve(user_input, k=RAG_K)
-            retrieved = retriever.retrieve(llm_input, k=RAG_K)
-            logger.info(f"  Using standard retrieval (Retrieval DP disabled)")
-            logger.info(f">>>>>>> RAG RETRIEVED: {retrieved}")
-            u_rag = []  # computed concurrently inside make_concurrent_llm_calls_optimized
-            retrieval_elapsed = time.time() - retrieval_start
-            # for r in retrieved:
-            #     print(r)
-            #     print("\n")
-            logger.info(f"  ✓ Retrieved {len(retrieved)} docs in {retrieval_elapsed:.2f}s (LOCAL)")
+            min_sim = epsilon_adjusted_threshold(epsilon)
+            logger.info("  Using threshold-filtered retrieval (min_sim=%.3f, ε=%s)", min_sim, epsilon)
+            if epsilon != float('inf'):
+                # Layer 1: threshold filter
+                # Layer 2: exponential-mechanism sampling (Retrieval DP)
+                pre_filtered = retriever.retrieve(llm_input, k=RAG_FETCH_K, fetch_k=RAG_FETCH_K, min_similarity=min_sim)
+                logger.info(f">>>>>>> RAG RETRIEVED pre_filtered: {retrieved}")
+                if pre_filtered:
+                    dp_retriever = DPRetriever(retriever, epsilon=epsilon)
+                    # Pass pre-filtered docs only — DPRetriever re-samples from them
+                    retrieved = dp_retriever.retrieve(llm_input, k=RAG_K, fetch_k=len(pre_filtered))
+                    logger.info(f">>>>>>> RAG RETRIEVED post_filtered: {retrieved}")
+                else:
+                    retrieved = []
+            else:
+                retrieved = retriever.retrieve(llm_input, k=RAG_K, fetch_k=RAG_FETCH_K, min_similarity=min_sim)
+                logger.info(f">>>>>>> RAG RETRIEVED: {retrieved}")
+                retrieval_elapsed = time.time() - retrieval_start
+                logger.info(f"  ✓ Retrieved {len(retrieved)} docs in {retrieval_elapsed:.2f}s (LOCAL)")
         except Exception as e:
             logger.info(f"  ✗ RAG retrieval error: {e}")
             retrieved = []

create_rag_retriever.py

@@ -1,4 +1,5 @@
 import logging
+import math
 import sys
 import os
 import re
@@ -216,8 +217,18 @@ def load_panorama_data():
 
 class HybridRetriever:
     """
-    Hybrid retriever combining semantic, lexical, and entity-based search.
+    Hybrid retriever combining semantic (FAISS), lexical (BM25), and
+    entity-based search with Reciprocal Rank Fusion (RRF).
+
+    The ``retrieve`` method accepts an optional ``min_similarity``
+    threshold.  When set, only documents whose normalised semantic
+    similarity to the query meets or exceeds that value are eligible for
+    retrieval.  This makes the system sensitive to Input-DP perturbation:
+    a heavily noised query will drift away from the persona corpus and
+    return fewer — or zero — documents, so the system's output correctly
+    reflects the privacy protection that is active.
     """
+
     def __init__(
             self,
             documents,
@@ -270,6 +281,15 @@ class HybridRetriever:
                         self.entity_index[clean_token].append(i)
         print(f"  ✓ Entity index built ({len(self.entity_index)} unique entities)")
 
+        # ── Content-to-corpus-index lookup (used by threshold filtering) ──────
+        # Maps page_content → list of corpus indices so that FAISS results
+        # (which are Document objects, not indices) can be mapped back to
+        # their position in self.documents.  Duplicate page_content values
+        # are handled by storing all matching indices.
+        self._content_to_indices = defaultdict(list)
+        for i, doc in enumerate(self.documents):
+            self._content_to_indices[doc.page_content].append(i)
+
     def _reciprocal_rank_fusion(self, rankings, k=60):
         """Combine multiple rankings using RRF."""
         scores = defaultdict(float)
@@ -286,41 +306,115 @@ class HybridRetriever:
         """Retrieve using only semantic search (for comparison)."""
         return self.vectorstore.similarity_search(query, k=k)
 
-    def retrieve(self, query, k=10, fetch_k=100):
-        """Hybrid retrieval combining semantic, BM25, and entity matching."""
-        # 1. Semantic search
-        semantic_results = self.vectorstore.similarity_search(query, k=fetch_k)
-        semantic_ranking = list(range(len(semantic_results)))
+    def retrieve(self, query, k=10, fetch_k=100, min_similarity=None):
+        """Hybrid retrieval combining semantic, BM25, and entity matching.
+
+        Parameters
+        ----------
+        query : str
+            The search query (may be DP-perturbed).
+        k : int
+            Maximum number of documents to return.
+        fetch_k : int
+            Candidate pool size passed to FAISS.
+        min_similarity : float or None
+            When set, only documents whose normalised semantic similarity
+            to the query meets or exceeds this value are eligible.
+            Documents below the threshold are excluded from ALL three
+            ranking components (semantic, BM25, entity) before RRF.
+            Pass None (default) to disable filtering and preserve the
+            original behaviour.
+
+        Returns
+        -------
+        list[Document]
+            Up to k documents, potentially fewer (or empty) when
+            min_similarity is strict relative to the query.
+        """
+        # ── 1. Semantic search WITH scores ───────────────────────────────────
+        try:
+            candidates = self.vectorstore.similarity_search_with_score(
+                query, k=fetch_k
+            )
+        except Exception:
+            # Fallback: vectorstore does not expose scores → no threshold
+            docs = self.vectorstore.similarity_search(query, k=fetch_k)
+            candidates = [(doc, 0.0) for doc in docs]
+
+        if not candidates:
+            return []
+
+        raw_docs, raw_scores = zip(*candidates)
+        raw_scores = list(raw_scores)
 
-        # 2. BM25 search
+        # ── Normalise scores to similarity ∈ [0, 1] ─────────────────────────
+        # FAISS/IndexFlatL2 returns L2 distances (ascending: closer = smaller).
+        # IndexFlatIP on normalised vectors returns cosine scores (descending).
+        if len(raw_scores) >= 2 and raw_scores[0] <= raw_scores[-1]:
+            # L2 distances: invert so that higher = more similar
+            max_dist = max(raw_scores) + 1e-10
+            similarities = [1.0 - s / max_dist for s in raw_scores]
+        else:
+            # Already similarity scores; clamp to [0, 1]
+            similarities = [max(0.0, min(1.0, s)) for s in raw_scores]
+
+        # ── Apply similarity threshold ────────────────────────────────────────
+        if min_similarity is not None:
+            passing_pairs = [
+                (doc, sim)
+                for doc, sim in zip(raw_docs, similarities)
+                if sim >= min_similarity
+            ]
+            if not passing_pairs:
+                logger.info(
+                    "  HybridRetriever: 0/%d candidates above threshold %.3f"
+                    " — returning []",
+                    len(raw_docs), min_similarity,
+                )
+                return []
+
+            # Build the set of *corpus* indices that passed the threshold.
+            # This is used to restrict BM25 and entity rankings to the
+            # same document subset, ensuring all three components only
+            # vote for threshold-passing documents.
+            valid_corpus_indices = set()
+            for doc, _ in passing_pairs:
+                for idx in self._content_to_indices.get(doc.page_content, []):
+                    valid_corpus_indices.add(idx)
+
+            # Semantic ranking: use actual corpus indices for filtered docs
+            # so they are consistent with BM25/entity namespace.
+            semantic_ranking = []
+            for doc, _ in passing_pairs:
+                for idx in self._content_to_indices.get(doc.page_content, []):
+                    semantic_ranking.append(idx)
+                    break  # one representative index per doc is enough for RRF
+        else:
+            # No threshold: preserve original behaviour (range-based indices).
+            valid_corpus_indices = None
+            semantic_ranking = list(range(len(raw_docs)))
+
+        # ── 2. BM25 search ───────────────────────────────────────────────────
         query_tokens = word_tokenize(query.lower())
         bm25_scores = self.bm25.get_scores(query_tokens)
         bm25_ranking = np.argsort(bm25_scores)[::-1][:fetch_k].tolist()
+        if valid_corpus_indices is not None:
+            bm25_ranking = [i for i in bm25_ranking if i in valid_corpus_indices]
 
-        # 3. Entity matching with stripped characters
+        # ── 3. Entity matching ───────────────────────────────────────────────
         query_lower = query.lower()
         entity_matches = set()
-
-        # Extract potential entities from query and strip special chars
-        query_tokens_for_entity = re.split(r'[;,\s.!?]+', query_lower)
-        for query_token in query_tokens_for_entity:
+        for query_token in re.split(r'[;,\s.!?]+', query_lower):
             clean_query_token = strip_special_chars(query_token)
             if len(clean_query_token) > 2:
-                # Check if this cleaned token matches any entity
                 for entity, doc_ids in self.entity_index.items():
                     if clean_query_token in entity or entity in clean_query_token:
                         entity_matches.update(doc_ids)
-
         entity_ranking = list(entity_matches)[:fetch_k]
+        if valid_corpus_indices is not None:
+            entity_ranking = [i for i in entity_ranking if i in valid_corpus_indices]
 
-        # Combine with RRF
-        rankings = {
-            "semantic": semantic_ranking,
-            "bm25": bm25_ranking,
-            "entity": entity_ranking,
-        }
-
-        # Weight rankings
+        # ── 4. Reciprocal Rank Fusion ─────────────────────────────────────────
         weighted_rankings = {}
         if self.semantic_weight > 0:
             weighted_rankings["semantic"] = semantic_ranking
@@ -330,11 +424,17 @@ class HybridRetriever:
             weighted_rankings["entity"] = entity_ranking
 
         scores, sources = self._reciprocal_rank_fusion(weighted_rankings)
-
-        # Sort by score and return top-k documents
-        sorted_doc_ids = sorted(scores.keys(), key=lambda x: scores[x], reverse=True)[:k]
-
-        return [self.documents[i] for i in sorted_doc_ids]
+        sorted_doc_ids = sorted(
+            scores.keys(), key=lambda x: scores[x], reverse=True
+        )[:k]
+
+        result = [self.documents[i] for i in sorted_doc_ids]
+        logger.info(
+            "  HybridRetriever: returning %d/%d docs (min_similarity=%s)",
+            len(result), len(candidates),
+            f"{min_similarity:.3f}" if min_similarity is not None else "None",
+        )
+        return result
 
 
 # ==============================================================================
@@ -342,54 +442,101 @@ class HybridRetriever:
 # ==============================================================================
 
 class DPRetriever:
-    """Differentially private retriever using exponential mechanism."""
+    """Differentially private retriever using the Exponential Mechanism.
+
+    Sensitivity is computed *empirically* from the actual utility range of
+    the candidate pool rather than a fixed constant.  This data-adaptive
+    approach (Dwork & Roth 2014; Koga et al. 2024) gives a tighter bound,
+    improving the privacy-utility trade-off without weakening the formal
+    ε-DP guarantee.
+
+    For the exponential mechanism the sensitivity Δu of utility function u is:
+        Δu = max_{d, d'} |u(d, q) − u(d', q)|
+    When u(d, q) = max_dist − dist(d, q) (distance-to-similarity inversion),
+    Δu equals the range of utilities across the candidate pool.  We clamp it
+    to ``sensitivity_cap`` to guard against degenerate cases where all
+    candidates are equidistant from the query.
+
+    Parameters
+    ----------
+    base_retriever : HybridRetriever
+        The underlying retriever whose vectorstore provides FAISS scores.
+    epsilon : float
+        DP privacy budget.  Smaller ε → stronger privacy.
+    sensitivity_cap : float
+        Floor value for the empirical sensitivity (default 1e-3).  For
+        most corpora the empirical range will be far larger than this cap
+        so it has no practical effect.
+    """
 
-    def __init__(self, base_retriever, epsilon=1.0):
+    def __init__(self, base_retriever, epsilon=1.0, sensitivity_cap=1e-3):
         self.base_retriever = base_retriever
         self.epsilon = epsilon
-
-    def retrieve(self, query, k=10, fetch_k=100, method="exponential", seed=None,
-                 epsilon_total=None, track_budget=False, return_diagnostics=False):
-        """
-        Retrieve documents with differential privacy.
-
-        Uses exponential mechanism to add noise to retrieval scores.
+        self.sensitivity_cap = sensitivity_cap
+
+    def retrieve(self, query, k=10, fetch_k=100, seed=None):
+        """Retrieve up to k documents with differential privacy.
+
+        Uses the Exponential Mechanism: each candidate document is sampled
+        with probability proportional to exp(ε · u(d) / 2Δu), where u is
+        the normalised similarity utility and Δu is the empirical range.
+
+        Parameters
+        ----------
+        query : str
+            Search query (may be DP-perturbed).
+        k : int
+            Number of documents to return.
+        fetch_k : int
+            Candidate pool size fetched from FAISS before sampling.
+        seed : int or None
+            Optional random seed for reproducibility.
+
+        Returns
+        -------
+        list[Document]
+            k documents sampled with DP-weighted probabilities.
         """
-        if epsilon_total is not None:
-            epsilon = epsilon_total
-        else:
-            epsilon = self.epsilon
-
         if seed is not None:
             np.random.seed(seed)
 
-        # Get candidate pool
-        candidates = self.base_retriever.vectorstore.similarity_search_with_score(
-            query, k=fetch_k
-        )
+        # Fetch candidate pool
+        try:
+            candidates = self.base_retriever.vectorstore.similarity_search_with_score(
+                query, k=fetch_k
+            )
+        except Exception:
+            # Fallback: vectorstore does not expose scores
+            docs = self.base_retriever.vectorstore.similarity_search(query, k=fetch_k)
+            return docs[:k]
 
         if not candidates:
             return []
 
-        # Apply exponential mechanism
         docs, base_scores = zip(*candidates)
         base_scores = np.array(base_scores)
 
-        # Convert distances to similarities (lower distance = higher similarity)
-        max_dist = base_scores.max() + 1e-10
-        utilities = max_dist - base_scores
+        # ── Convert raw scores to a utility where higher = better ────────────
+        # FAISS with L2: scores are distances (ascending) → invert
+        # FAISS with IP: scores are similarities (descending) → use as-is
+        if base_scores[0] >= base_scores[-1]:
+            utilities = base_scores.copy()          # already similarity scores
+        else:
+            max_dist = base_scores.max() + 1e-10
+            utilities = max_dist - base_scores       # invert L2 distances
 
-        # Add noise using exponential mechanism
-        sensitivity = 2.0  # Conservative estimate
+        # ── Empirical sensitivity (Dwork & Roth 2014; Koga et al. 2024) ─────
+        u_range = float(utilities.max() - utilities.min())
+        sensitivity = max(u_range, self.sensitivity_cap)
 
-        probabilities = np.exp((epsilon * utilities) / (2 * sensitivity))
+        # ── Exponential mechanism: P(d) ∝ exp(ε · u(d) / 2Δu) ──────────────
+        probabilities = np.exp((self.epsilon * utilities) / (2.0 * sensitivity))
         probabilities = probabilities / probabilities.sum()
 
         # Sample k documents without replacement
         selected_indices = np.random.choice(
             len(docs), size=min(k, len(docs)), replace=False, p=probabilities
         )
-
         return [docs[i] for i in selected_indices]
 
 
@@ -506,6 +653,11 @@ def build_documents(merged, users_df):
 
 
 if __name__ == "__main__":
+    # retriever = load_retriever_components("./faiss_persona_sarah_chen_retriever_components.pkl")
+    retriever = load_retriever_components("./faiss_panorama_retriever_components.pkl")
+    # save_retriever_components(retriever, "./faiss_persona_sarah_chen_retriever_components.pkl")
+    save_retriever_components(retriever, "./faiss_panorama_retriever_components.pkl")
+
     RETRIEVER_SAVE_PATH = r"C:\Users\user\Datasets\Panorama Synthetic Data RAG\faiss_panorama_retriever_components.pkl"
     if os.path.exists(RETRIEVER_SAVE_PATH):
         retriever = load_retriever_components(RETRIEVER_SAVE_PATH)

dp_vocab.txt

@@ -16,7 +16,6 @@ resident
 citizen
 participant
 contributor
-representative
 colleague
 client
 customer
@@ -35,7 +34,6 @@ associate
 
 # ── Age / life-stage ─────────────────────────────────────────────────────────
 young
-adult
 senior
 graduate
 student
@@ -43,8 +41,6 @@ retired
 veteran
 experienced
 junior
-entry-level
-mid-career
 established
 seasoned
 upcoming
@@ -54,12 +50,9 @@ elderly
 beginner
 intermediate
 advanced
-early-career
-late-career
 apprentice
 trainee
 postgraduate
-undergraduate
 freshman
 sophomore
 retiree
@@ -92,7 +85,6 @@ western
 central
 domestic
 overseas
-international
 remote
 nearby
 provincial
@@ -154,12 +146,9 @@ prosperous
 struggling
 secure
 insecure
-well-off
-underprivileged
 affluent
 cash
 fund
-capital
 asset
 liability
 revenue
@@ -172,7 +161,6 @@ benefit
 pension
 dividend
 interest
-tax
 insurance
 
 # ── Marital / relationship ────────────────────────────────────────────────────
@@ -186,9 +174,7 @@ companion
 household
 relationship
 personal
-close
 romantic
-domestic
 familial
 mutual
 shared
@@ -199,7 +185,6 @@ public
 exclusive
 casual
 serious
-stable
 supportive
 caring
 devoted
@@ -209,7 +194,6 @@ free
 # ── Work / profession ─────────────────────────────────────────────────────────
 working
 employed
-professional
 analyst
 researcher
 developer
@@ -229,7 +213,6 @@ technician
 advisor
 operator
 executive
-administrator
 instructor
 trainer
 officer
@@ -276,7 +259,6 @@ mental
 therapy
 medication
 recovery
-rehabilitation
 nutrition
 exercise
 diet
@@ -285,7 +267,6 @@ symptom
 diagnosis
 prescription
 procedure
-specialist
 consultation
 examination
 test
@@ -312,7 +293,6 @@ managed
 education
 learning
 training
-qualification
 certificate
 diploma
 degree
@@ -339,7 +319,6 @@ tuition
 enrollment
 graduation
 admission
-examination
 assignment
 project
 laboratory
@@ -353,7 +332,6 @@ problem
 solution
 issue
 topic
-subject
 matter
 experience
 perspective
@@ -371,7 +349,6 @@ change
 difference
 impact
 effect
-benefit
 risk
 option
 choice
@@ -380,22 +357,17 @@ step
 process
 method
 approach
-environment
 context
 background
 field
-sector
 industry
 role
 position
 task
-project
 goal
 target
 opportunity
 challenge
-lifestyle
-routine
 schedule
 activity
 factor
@@ -403,21 +375,18 @@ aspect
 element
 feature
 requirement
-condition
 circumstance
 resource
 support
 service
 system
 network
-community
 group
 organization
 institution
 
 # ── General adjectives ────────────────────────────────────────────────────────
 good
-bad
 great
 difficult
 easy
@@ -428,21 +397,15 @@ available
 common
 specific
 general
-personal
-public
-private
 current
 recent
 positive
 negative
 significant
-major
-minor
 various
 unique
 typical
 standard
-advanced
 basic
 complex
 simple
@@ -459,8 +422,6 @@ uncertain
 confident
 prepared
 qualified
-experienced
-knowledgeable
 skilled
 capable
 able
@@ -494,11 +455,9 @@ accessible
 # ── General verbs ─────────────────────────────────────────────────────────────
 work
 help
-support
 manage
 handle
 find
-use
 need
 want
 think
@@ -507,15 +466,12 @@ know
 understand
 consider
 believe
-try
 make
-get
 give
 take
 keep
 stay
 move
-change
 improve
 reduce
 increase
@@ -589,11 +545,9 @@ integrate
 connect
 link
 relate
-associate
 involve
 include
 exclude
-consider
 ignore
 focus
 prioritize
@@ -602,3 +556,1337 @@ adjust
 modify
 customize
 personalize
+
+
+# ===========================================================================
+# AUGMENTED SECTION  –  stop-word-cleaned  (1834 total)
+# Generated : 2026-03-17  (stop words removed)
+# Filter    : 4-12 chars, alpha only, deduplicated
+# ===========================================================================
+
+# ── Extended medical / mental health ────────────────────────────────────
+anxiety
+depression
+stress
+disorder
+psychiatric
+psychiatrist
+counseling
+psychologist
+dosage
+refill
+referral
+prognosis
+symptoms
+procedures
+cardiologist
+neurologist
+pediatrician
+gynecologist
+oncologist
+radiologist
+orthopedic
+urologist
+nephrologist
+allergist
+internist
+pathologist
+obstetrician
+radiographer
+occupational
+speech
+respiratory
+cognitive
+behavioral
+telehealth
+telemedicine
+outpatient
+inpatient
+ambulatory
+surgical
+noninvasive
+palliative
+integrative
+alternative
+mindfulness
+meditation
+relaxation
+breathing
+insomnia
+fatigue
+migraine
+headache
+nausea
+dizziness
+vertigo
+arthritis
+diabetes
+hypertension
+asthma
+obesity
+anemia
+thyroid
+infection
+inflammation
+allergy
+allergies
+autoimmune
+neurological
+emotional
+recurring
+intermittent
+progressive
+degenerative
+remission
+relapse
+healing
+prevention
+intervention
+urine
+stool
+saliva
+biopsy
+imaging
+ultrasound
+mammogram
+vaccine
+vaccination
+immunization
+booster
+antibody
+antibiotics
+antivirals
+ward
+emergency
+intensive
+obstetric
+pediatric
+geriatric
+intake
+discharge
+aftercare
+copay
+deductible
+premium
+formulary
+appeal
+grievance
+denial
+approval
+claim
+stroke
+tumor
+cyst
+polyp
+ulcer
+fracture
+sprain
+strain
+concussion
+laceration
+bruising
+swelling
+redness
+rash
+burn
+wound
+scar
+retention
+edema
+abscess
+hernia
+rupture
+prolapse
+dislocation
+hemorrhage
+bleeding
+clotting
+thrombosis
+embolism
+infarction
+seizure
+epilepsy
+dementia
+alzheimer
+parkinson
+sclerosis
+fibromyalgia
+lupus
+psoriasis
+eczema
+rosacea
+vitiligo
+osteoporosis
+rheumatoid
+gout
+tendinitis
+bursitis
+neuropathy
+sciatica
+scoliosis
+kyphosis
+herniated
+tonsillitis
+sinusitis
+bronchitis
+pneumonia
+pleurisy
+pericarditis
+myocarditis
+endocarditis
+tachycardia
+bradycardia
+colonoscopy
+endoscopy
+gastroscopy
+laparoscopy
+arthroscopy
+aspiration
+drainage
+irrigation
+debridement
+suturing
+casting
+splinting
+traction
+bracing
+prosthetics
+orthotics
+chemotherapy
+radiation
+hormone
+targeted
+biological
+robotic
+intravenous
+subcutaneous
+topical
+transdermal
+inhaled
+nebulized
+sublingual
+
+# ── Extended occupations and roles ──────────────────────────────────────
+fisherman
+fisher
+farmer
+rancher
+shepherd
+herder
+forester
+logger
+miner
+steelworker
+welder
+fabricator
+machinist
+millwright
+carpenter
+joiner
+glazier
+roofer
+tiler
+bricklayer
+stonemason
+plasterer
+painter
+decorator
+upholsterer
+floorer
+landscaper
+housekeeper
+janitor
+custodian
+cleaner
+launderer
+tailor
+seamstress
+cobbler
+jeweler
+watchmaker
+optician
+audiologist
+podiatrist
+chiropractor
+osteopath
+naturopath
+homeopath
+midwife
+paramedic
+phlebotomist
+firefighter
+security
+warden
+ranger
+inspector
+customs
+immigration
+caseworker
+probation
+parole
+advocate
+mediator
+interpreter
+translator
+editor
+proofreader
+publisher
+illustrator
+photographer
+videographer
+animator
+filmmaker
+producer
+screenwriter
+playwright
+lyricist
+composer
+musician
+performer
+conductor
+baker
+confectioner
+pastry
+butcher
+fishmonger
+greengrocer
+grocer
+barista
+bartender
+waiter
+waitress
+host
+hostess
+concierge
+actuary
+broker
+dealer
+investor
+recruiter
+headhunter
+marketer
+advertiser
+publicist
+influencer
+strategist
+fundraiser
+politician
+legislator
+diplomat
+ambassador
+consul
+professor
+lecturer
+tutor
+mentor
+facilitator
+principal
+provost
+chancellor
+
+# ── Family and relationships ────────────────────────────────────────────
+spouse
+husband
+wife
+girlfriend
+boyfriend
+fiancee
+parent
+mother
+father
+daughter
+sibling
+brother
+sister
+child
+grandparent
+grandmother
+grandfather
+grandchild
+grandson
+uncle
+aunt
+nephew
+niece
+cousin
+relative
+caregiver
+guardian
+stepparent
+stepmother
+stepfather
+stepchild
+stepsibling
+foster
+adoptive
+extended
+nuclear
+blended
+dependent
+beneficiary
+divorced
+widowed
+separated
+remarried
+cohabiting
+engaged
+friendship
+mentorship
+roommate
+housemate
+flatmate
+
+# ── Common adverbs and qualifiers ───────────────────────────────────────
+currently
+previously
+formerly
+typically
+regularly
+consistently
+occasionally
+partly
+gradually
+carefully
+thoroughly
+properly
+directly
+indirectly
+actively
+passively
+openly
+privately
+publicly
+formally
+informally
+personally
+academically
+financially
+medically
+legally
+locally
+nationally
+regionally
+globally
+remotely
+immediately
+urgently
+eventually
+ultimately
+initially
+obviously
+seemingly
+reportedly
+roughly
+precisely
+adequately
+sufficiently
+strongly
+weakly
+moderately
+increasingly
+effectively
+efficiently
+successfully
+voluntarily
+separately
+individually
+collectively
+
+# ── Extended location and geography ─────────────────────────────────────
+street
+avenue
+boulevard
+highway
+trail
+apartment
+suite
+building
+campus
+compound
+workplace
+factory
+warehouse
+depot
+terminal
+station
+airport
+seaport
+harbor
+suburban
+exurban
+parish
+province
+canton
+department
+continent
+hemisphere
+equatorial
+tropical
+subtropical
+temperate
+arctic
+alpine
+inland
+riverside
+lakeside
+hillside
+uptown
+midtown
+periphery
+fringe
+northeast
+northwest
+southeast
+southwest
+midwestern
+pacific
+atlantic
+caribbean
+scandinavian
+
+# ── Technology and digital ──────────────────────────────────────────���───
+computer
+laptop
+tablet
+smartphone
+device
+keyboard
+internet
+wireless
+broadband
+fiber
+cable
+satellite
+software
+hardware
+platform
+application
+database
+server
+cloud
+storage
+backup
+encryption
+privacy
+firewall
+website
+browser
+search
+email
+messaging
+online
+virtual
+hybrid
+distributed
+asynchronous
+productivity
+coordination
+workflow
+account
+profile
+login
+password
+username
+credential
+notification
+alert
+reminder
+calendar
+spreadsheet
+presentation
+archive
+video
+audio
+podcast
+streaming
+broadcast
+recording
+playback
+algorithm
+automation
+analytics
+dashboard
+reporting
+insight
+upgrade
+update
+patch
+version
+release
+deployment
+maintenance
+subscription
+license
+membership
+
+# ── Insurance legal and finance extension ───────────────────────────────
+coverage
+coinsurance
+qualifying
+provider
+generic
+gatekeeper
+medicaid
+medicare
+marketplace
+exchange
+annuity
+portfolio
+equity
+bond
+yield
+allocation
+rebalancing
+withdrawal
+deposit
+contribution
+retirement
+checking
+overdraft
+statement
+transaction
+merchant
+vendor
+payee
+credit
+debit
+outstanding
+amortization
+refinance
+consolidate
+bankruptcy
+foreclosure
+garnishment
+lien
+collateral
+compliance
+regulation
+statute
+ordinance
+jurisdiction
+negligence
+indemnity
+warranty
+guarantee
+remedy
+arbitration
+mediation
+litigation
+judgment
+contract
+clause
+amendment
+addendum
+disclosure
+consent
+waiver
+notarize
+certify
+authenticate
+witness
+
+# ── Social and community ────────────────────────────────────────────────
+nonprofit
+charity
+foundation
+committee
+council
+coalition
+alliance
+partnership
+outreach
+engagement
+advocacy
+awareness
+campaign
+initiative
+hotline
+helpline
+diversity
+inclusion
+belonging
+cultural
+linguistic
+religious
+ethnic
+marginalized
+underserved
+vulnerable
+empowerment
+resilience
+capacity
+civic
+democracy
+policy
+reform
+legislation
+grassroots
+stakeholder
+
+# ── Work environment and employment ─────────────────────────────────────
+bonus
+incentive
+commission
+overtime
+fulltime
+permanent
+seasonal
+freelance
+onsite
+promotion
+demotion
+transfer
+reassignment
+rotation
+performance
+evaluation
+appraisal
+feedback
+milestone
+objective
+metric
+competency
+assessment
+onboarding
+orientation
+shadowing
+offboarding
+resignation
+termination
+redundancy
+layoff
+severance
+union
+bargaining
+dispute
+harassment
+retaliation
+ethics
+conduct
+integrity
+transparency
+governance
+culture
+leadership
+hierarchy
+division
+squad
+forecast
+headcount
+procurement
+supplier
+outsourcing
+audit
+quality
+intellectual
+patent
+trademark
+copyright
+licensing
+royalty
+
+# ── Education extended ──────────────────────────────────────────────────
+literacy
+numeracy
+creativity
+teamwork
+media
+kindergarten
+elementary
+vocational
+polytechnic
+flipped
+mastery
+syllabus
+rubric
+formative
+summative
+diagnostic
+authentic
+tutorial
+discussion
+textbook
+workbook
+journal
+article
+fellowship
+recognition
+articulation
+registration
+deferral
+suspension
+expulsion
+commencement
+alumni
+career
+placement
+internship
+cooperative
+volunteering
+
+# ── Common general nouns ────────────────────────────────────────────────
+ability
+absence
+accident
+achievement
+action
+addition
+advantage
+affair
+agreement
+argument
+assistance
+attention
+attitude
+authority
+barrier
+behavior
+board
+body
+branch
+category
+cause
+ceiling
+chapter
+character
+class
+climate
+comment
+commitment
+component
+conference
+conflict
+conversation
+corporation
+deadline
+debate
+demand
+description
+destination
+detail
+direction
+display
+distance
+distribution
+domain
+draft
+duration
+duty
+emphasis
+employment
+energy
+entrance
+entry
+exception
+existence
+expansion
+explanation
+extension
+failure
+figure
+format
+framework
+function
+funding
+generation
+guidance
+habit
+history
+identity
+importance
+impression
+indicator
+information
+inspection
+integration
+intention
+interview
+involvement
+language
+layer
+lesson
+limitation
+management
+manner
+margin
+meaning
+mechanism
+meeting
+mention
+mission
+mixture
+modification
+moment
+movement
+nature
+obligation
+observation
+occasion
+overview
+ownership
+pattern
+period
+phase
+practice
+presence
+priority
+production
+progress
+provision
+quantity
+range
+rate
+reaction
+reason
+reference
+relevance
+response
+revision
+selection
+shortage
+space
+status
+strategy
+strength
+structure
+substance
+summary
+timeline
+transition
+trend
+uncertainty
+validation
+value
+variety
+vision
+volume
+
+# ── Common general adjectives ───────────────���───────────────────────────
+absolute
+acceptable
+accurate
+actual
+additional
+adverse
+annual
+applicable
+approximate
+aware
+balanced
+brief
+broad
+careful
+clear
+consistent
+continuous
+correct
+critical
+decisive
+detailed
+distinct
+diverse
+dominant
+dynamic
+essential
+exact
+existing
+expected
+extensive
+external
+extreme
+fair
+familiar
+frequent
+fundamental
+genuine
+global
+gradual
+growing
+helpful
+historical
+immediate
+inclusive
+initial
+integrated
+internal
+logical
+meaningful
+minimal
+national
+natural
+notable
+original
+partial
+potential
+primary
+proper
+reasonable
+remaining
+responsive
+similar
+structured
+successful
+sustainable
+systemic
+thorough
+timely
+transparent
+valid
+vital
+voluntary
+widespread
+neutral
+open
+profound
+rapid
+realistic
+regular
+related
+robust
+sensitive
+sharp
+solid
+strict
+superior
+systematic
+tender
+total
+universal
+updated
+useful
+verbal
+vertical
+visible
+visual
+vocal
+written
+functional
+interactive
+legitimate
+measurable
+operational
+parallel
+
+# ── Common general verbs ────────────────────────────────────────────────
+accept
+acknowledge
+acquire
+activate
+affect
+affirm
+align
+allocate
+allow
+announce
+approve
+assign
+assist
+attain
+authorize
+avoid
+begin
+calculate
+check
+clarify
+classify
+collect
+confirm
+control
+convert
+convey
+cooperate
+define
+depend
+designate
+direct
+enable
+establish
+expand
+extend
+follow
+formulate
+fulfill
+generate
+guide
+highlight
+incorporate
+indicate
+inform
+introduce
+launch
+leverage
+listen
+locate
+meet
+minimize
+negotiate
+notify
+operate
+outline
+prepare
+proceed
+promote
+protect
+publish
+pursue
+raise
+recognize
+recruit
+refer
+reflect
+retain
+select
+send
+serve
+solve
+specify
+standardize
+strengthen
+streamline
+validate
+verify
+welcome
+capture
+commit
+continue
+discover
+educate
+empower
+enhance
+examine
+express
+gather
+invest
+justify
+motivate
+observe
+overcome
+prevent
+produce
+reach
+realize
+regulate
+reinforce
+represent
+restore
+seek
+simplify
+stimulate
+sustain
+tackle
+utilize
+
+# ── Broad common English words ──────────────────────────────────────────
+abroad
+admit
+adopt
+advance
+advice
+afraid
+agree
+alarm
+almost
+amount
+anger
+animal
+appear
+arise
+arrive
+assume
+attach
+attack
+attend
+attract
+basis
+birth
+block
+break
+bring
+burst
+calm
+carry
+cease
+certain
+chain
+chair
+charge
+chart
+cheap
+chief
+clean
+cloth
+coast
+color
+compete
+content
+copy
+core
+count
+cover
+crime
+cross
+crowd
+daily
+damage
+dead
+deal
+dear
+deep
+delay
+differ
+divide
+doubt
+draw
+drive
+early
+earth
+effort
+enter
+error
+event
+example
+exist
+face
+fall
+false
+favor
+fight
+file
+fire
+fixed
+force
+form
+front
+given
+glass
+growth
+happy
+heard
+heavy
+hold
+house
+ideal
+image
+large
+legal
+light
+limit
+little
+lower
+model
+night
+north
+noted
+phone
+press
+prove
+pull
+push
+recall
+remain
+remove
+return
+right
+river
+road
+room
+round
+route
+save
+seem
+sense
+short
+sleep
+small
+south
+speak
+spend
+staff
+stand
+store
+style
+term
+thick
+train
+trial
+true
+trust
+usual
+visit
+voice
+watch
+water
+whole
+word
+write
+zero
+alone
+deeply
+hard
+likely
+sometimes
+back
+base
+best
+care
+case
+date
+done
+fact
+fail
+fast
+fill
+firm
+fish
+flow
+food
+full
+gain
+head
+high
+home
+hope
+hour
+kind
+land
+late
+lead
+life
+like
+line
+live
+look
+mark
+mean
+mind
+miss
+name
+news
+note
+page
+pain
+part
+past
+play
+poor
+read
+real
+rule
+safe
+self
+show
+sign
+slow
+sort
+talk
+team
+tell
+turn
+type
+wait
+walk
+warm
+weak
+week
+wish
+year