Upload app.py

app.py

@@ -3788,13 +3788,15 @@ UI_CONFIG = {
     "ONLINE_RERANK_MAX_CPU": 12,
     # Smarter selection for the same 12 expensive BGE slots: accuracy up, latency flat.
     "CHEAP_CANDIDATES_PER_BOOK": 3,
-    # Set-valued questions often need several sibling records from the same book.
-    # Keep the normal per-book recall small, but deepen only the strongest books.
-    # This expands the internal answer pool without increasing the visible UI limit
-    # or the expensive CPU BGE rerank budget.
-    "SET_COVERAGE_CANDIDATES_PER_BOOK": 10,
-    "SET_COVERAGE_DEEP_BOOKS": 6,
-    "SET_COVERAGE_MAX_CANDIDATES": 144,
+    # Set-valued questions may distribute one answer across nearby source records.
+    # Do not deepen whole books blindly. Instead, reserve a few reranker slots for
+    # semantically qualified sibling records near a source-local set seed.
+    "SET_RECALL_MAX_SEEDS": 8,
+    "SET_RECALL_SEED_PER_BOOK": 2,
+    "SET_RECALL_PAGE_RADIUS": 4,
+    "SET_RECALL_MAX_SIBLINGS": 12,
+    "SET_RECALL_SIBLINGS_PER_BOOK": 8,
+    "SET_RECALL_RERANK_RESERVE": 6,
     "MAX_RERANKED_PER_BOOK": 2,
     "DIVERSE_CANDIDATE_SELECTION": True,
     "MIN_RETRIEVER_AGREEMENT": 2,
@@ -6278,74 +6280,170 @@ class ProfessionalEvidenceEngine:
         per_book_limit=max(1,min(4,int(filters.get("evidence_count",1) or 1))); allowed_df=self.df.iloc[answer_allowed]
 
         # The visible evidence_count is a presentation limit, not an answer-completeness
-        # limit. Set-valued requests may distribute one complete answer across several
-        # sibling records in the same book, so deepen recall only for the strongest
-        # books while leaving normal questions at the original small per-book budget.
+        # limit. Ordinary questions keep the original top-k-per-book behavior.
         query_structure = analyze_direct_answer_intent_ui(query, lang)
         generic_request_type = clean_ui(query_structure.get("generic_request_type", "")).casefold()
+        generic_subject_terms = [
+            clean_ui(value)
+            for value in (query_structure.get("generic_subject_terms", []) or [])
+            if clean_ui(value)
+        ]
         set_valued_request = generic_request_type in {"list", "components", "conditions", "pillars", "duties"}
 
         base_cheap_per_book=max(
             per_book_limit,
             int(UI_CONFIG.get("CHEAP_CANDIDATES_PER_BOOK",3)),
         )
-        deep_cheap_per_book=max(
-            base_cheap_per_book,
-            int(UI_CONFIG.get("SET_COVERAGE_CANDIDATES_PER_BOOK",10)),
-        )
-        deep_book_limit=max(
-            1,
-            int(UI_CONFIG.get("SET_COVERAGE_DEEP_BOOKS",6)),
-        )
+
+        def _set_recall_page_number(row) -> int | None:
+            match = re.search(r"\d+", str(getattr(row, "page_number", "") or ""))
+            return int(match.group()) if match else None
+
+        def _set_recall_operator_cue(row) -> bool:
+            if not set_valued_request:
+                return False
+            structural = " ".join(
+                str(value or "")
+                for value in (
+                    getattr(row, "ruling", "") if ar else getattr(row, "ruling_en", ""),
+                    getattr(row, "title", "") if ar else getattr(row, "title_en", ""),
+                    getattr(row, "question", "") if ar else getattr(row, "question_en", ""),
+                    getattr(row, "chapter", "") if ar else getattr(row, "chapter_en", ""),
+                )
+            )
+            normalized = norm_ar_ui(structural) if ar else norm_en_ui(structural)
+            if generic_request_type == "conditions":
+                patterns = (
+                    (r"(?:^|\s)(?:شرط|شروط|اشتراط|يشترط|الاستطاعة|القدرة)(?:\s|$)",)
+                    if ar
+                    else (r"\b(?:condition|conditions|requirement|requirements|capacity|ability)\b",)
+                )
+            elif generic_request_type == "pillars":
+                patterns = (
+                    (r"(?:^|\s)(?:ركن|اركان|أركان)(?:\s|$)",)
+                    if ar
+                    else (r"\b(?:pillar|pillars)\b",)
+                )
+            elif generic_request_type == "duties":
+                patterns = (
+                    (r"(?:^|\s)(?:واجب|واجبات)(?:\s|$)",)
+                    if ar
+                    else (r"\b(?:duty|duties|required|obligatory)\b",)
+                )
+            else:
+                patterns = (
+                    (r"(?:^|\s)(?:مكونات|اجزاء|أجزاء|عناصر|قائمة)(?:\s|$)",)
+                    if ar
+                    else (r"\b(?:components?|parts?|items?|list)\b",)
+                )
+            return any(re.search(pattern, normalized, re.I) for pattern in patterns)
+
+        def _set_recall_subject_bound(row) -> bool:
+            if not generic_subject_terms:
+                return True
+            scope = " ".join(
+                str(value or "")
+                for value in (
+                    getattr(row, "question", "") if ar else getattr(row, "question_en", ""),
+                    getattr(row, "title", "") if ar else getattr(row, "title_en", ""),
+                    getattr(row, "chapter", "") if ar else getattr(row, "chapter_en", ""),
+                    getattr(row, "answer", "") if ar else getattr(row, "answer_en", ""),
+                    getattr(row, "answer_short", "") if ar else getattr(row, "answer_short_en", ""),
+                    getattr(row, "answer_detailed", "") if ar else getattr(row, "answer_detailed_en", ""),
+                    getattr(row, "answer_evidence", "") if ar else getattr(row, "answer_evidence_en", ""),
+                )
+            )
+            probe = " ".join(generic_subject_terms)
+            return topic_overlap(probe, scope, lang) >= 0.46
 
         grouped_books=[]
         for bid, group in allowed_df.groupby("book_id", sort=False):
             local=np.asarray(group.index.tolist(), dtype=int)
             if not len(local):
                 continue
-            best_score=float(np.max(pre[local]))
-            grouped_books.append((str(bid), local, best_score))
-
-        deep_books=set()
-        if set_valued_request:
-            deep_books={
-                bid
-                for bid, _, _ in sorted(
-                    grouped_books,
-                    key=lambda row: row[2],
-                    reverse=True,
-                )[:deep_book_limit]
-            }
+            grouped_books.append((str(bid), local, float(np.max(pre[local]))))
 
         candidate_pool=set(exact_indices)
         preliminary_book_best=[]
         answer_take_by_book={}
         for bid, local, _ in grouped_books:
-            take_limit=(
-                deep_cheap_per_book
-                if set_valued_request and bid in deep_books
-                else base_cheap_per_book
-            )
-            take=min(len(local),take_limit)
-            answer_take_by_book[bid]=take_limit
+            take=min(len(local),base_cheap_per_book)
+            answer_take_by_book[bid]=base_cheap_per_book
             if take:
                 best=local[np.argsort(-pre[local])[:take]]
                 candidate_pool.update(map(int,best))
                 preliminary_book_best.append(int(best[0]))
-
         candidate_pool.update(global_set)
-        max_pool=max(
-            int(UI_CONFIG.get("MAX_CANDIDATES",96)),
-            len(preliminary_book_best),
-        )
+
+        set_seed_indices=[]
         if set_valued_request:
-            max_pool=max(
-                max_pool,
-                min(
-                    int(UI_CONFIG.get("SET_COVERAGE_MAX_CANDIDATES",144)),
-                    len(candidate_pool),
-                ),
-            )
+            seed_cap=max(1,int(UI_CONFIG.get("SET_RECALL_MAX_SEEDS",8)))
+            seed_per_book=max(1,int(UI_CONFIG.get("SET_RECALL_SEED_PER_BOOK",2)))
+            seed_counts=Counter()
+            qualified=[]
+            for idx in answer_allowed:
+                idx=int(idx)
+                row=self.df.iloc[idx]
+                if not _set_recall_operator_cue(row):
+                    continue
+                if not _set_recall_subject_bound(row):
+                    continue
+                qualified.append(idx)
+            for idx in sorted(qualified,key=lambda i:float(pre[int(i)]),reverse=True):
+                bid=str(self.df.iloc[int(idx)].book_id)
+                if seed_counts[bid]>=seed_per_book:
+                    continue
+                set_seed_indices.append(int(idx))
+                seed_counts[bid]+=1
+                if len(set_seed_indices)>=seed_cap:
+                    break
+            candidate_pool.update(set_seed_indices)
+
+        set_sibling_indices=[]
+        set_sibling_distance={}
+        if set_valued_request and set_seed_indices:
+            page_radius=max(0,int(UI_CONFIG.get("SET_RECALL_PAGE_RADIUS",4)))
+            sibling_cap=max(1,int(UI_CONFIG.get("SET_RECALL_MAX_SIBLINGS",12)))
+            sibling_per_book=max(1,int(UI_CONFIG.get("SET_RECALL_SIBLINGS_PER_BOOK",8)))
+            seed_pages_by_book=defaultdict(list)
+            for idx in set_seed_indices:
+                row=self.df.iloc[int(idx)]
+                page=_set_recall_page_number(row)
+                if page is not None:
+                    seed_pages_by_book[str(row.book_id)].append(page)
+
+            sibling_candidates=[]
+            for bid, local, _ in grouped_books:
+                seed_pages=seed_pages_by_book.get(str(bid), [])
+                if not seed_pages:
+                    continue
+                for idx in local:
+                    idx=int(idx)
+                    if idx in set_seed_indices:
+                        continue
+                    row=self.df.iloc[idx]
+                    if not _set_recall_operator_cue(row):
+                        continue
+                    page=_set_recall_page_number(row)
+                    if page is None:
+                        continue
+                    distance=min(abs(page-seed_page) for seed_page in seed_pages)
+                    if distance>page_radius:
+                        continue
+                    sibling_candidates.append((distance,-float(pre[idx]),idx,bid))
+
+            per_book_siblings=Counter()
+            for distance,_,idx,bid in sorted(sibling_candidates):
+                if per_book_siblings[str(bid)]>=sibling_per_book:
+                    continue
+                set_sibling_indices.append(int(idx))
+                set_sibling_distance[int(idx)]=int(distance)
+                per_book_siblings[str(bid)]+=1
+                if len(set_sibling_indices)>=sibling_cap:
+                    break
+            candidate_pool.update(set_sibling_indices)
+
+        max_pool=max(int(UI_CONFIG.get("MAX_CANDIDATES",96)),len(preliminary_book_best))
         ranked_pool=sorted(candidate_pool,key=lambda i:float(pre[i]),reverse=True)
         graph_seeds=ranked_pool[:min(12, len(ranked_pool))]
         graph_candidates, graph_reasons = self.kb_graph.expand(
@@ -6354,7 +6452,19 @@ class ProfessionalEvidenceEngine:
         )
         candidate_pool.update(graph_candidates)
         ranked_pool=sorted(candidate_pool,key=lambda i:float(pre[i]),reverse=True)
-        candidates=np.asarray(sorted(set(preliminary_book_best)|set(exact_indices)|set(ranked_pool[:max_pool])|set(graph_candidates),key=lambda i:float(pre[i]),reverse=True),dtype=int)
+        candidates=np.asarray(
+            sorted(
+                set(preliminary_book_best)
+                | set(exact_indices)
+                | set(ranked_pool[:max_pool])
+                | set(graph_candidates)
+                | set(set_seed_indices)
+                | set(set_sibling_indices),
+                key=lambda i:float(pre[i]),
+                reverse=True,
+            ),
+            dtype=int,
+        )
         rerank_cap=int(UI_CONFIG.get("ONLINE_RERANK_MAX_CPU",12)) if self._model_device=="cpu" else len(candidates)
         arrays_by_name={"word":sw,"char":sc,"bm25":bm,"dense":dn,"fusion":pre}
         promotion_order,promotion_reasons=self._diverse_promotion_order(candidates,exact_indices,arrays_by_name,rerank_cap)
@@ -6384,6 +6494,40 @@ class ProfessionalEvidenceEngine:
         for idx, reasons in graph_reasons.items():
             promotion_reasons.setdefault(int(idx), [])
             promotion_reasons[int(idx)] = sorted(set(promotion_reasons[int(idx)]) | set(reasons))
+
+        set_recall_ranked=[]
+        if set_valued_request and set_sibling_indices:
+            reserve=min(
+                max(0,int(UI_CONFIG.get("SET_RECALL_RERANK_RESERVE",6))),
+                max(0,rerank_cap//2),
+            )
+            set_recall_ranked=sorted(
+                set_sibling_indices,
+                key=lambda i:(
+                    -int(set_sibling_distance.get(int(i),999)),
+                    float(pre[int(i)]),
+                ),
+                reverse=True,
+            )[:reserve]
+            protected=set(map(int,exact_indices))|set(map(int,graph_ranked))|set(map(int,set_recall_ranked))
+            for idx in set_recall_ranked:
+                idx=int(idx)
+                if idx in promotion_order:
+                    promotion_reasons.setdefault(idx,[])
+                    promotion_reasons[idx]=sorted(set(promotion_reasons[idx])|{"set_local_sibling"})
+                    continue
+                if len(promotion_order)>=rerank_cap:
+                    removable=next(
+                        (j for j in reversed(promotion_order) if int(j) not in protected),
+                        None,
+                    )
+                    if removable is None:
+                        continue
+                    promotion_order.remove(removable)
+                promotion_order.append(idx)
+                promotion_reasons.setdefault(idx,[])
+                promotion_reasons[idx]=sorted(set(promotion_reasons[idx])|{"set_local_sibling"})
+
         rerank_indices=np.asarray(promotion_order,dtype=int); reranked_set=set(map(int,rerank_indices))
         top_sets={name:set(map(int,self._top_indices(arr,allowed,min(per_retriever,24)))) for name,arr in arrays_by_name.items()}
 
@@ -6487,14 +6631,33 @@ class ProfessionalEvidenceEngine:
             if not items:
                 missing_books.append(bid); continue
 
-            # Keep an internal pool for answer completeness. For ordinary questions
-            # this remains the original small budget. For set-valued questions only
-            # the strongest books receive deeper sibling-record coverage.
+            # Keep the ordinary top-k internal pool, then append only source-local
+            # semantic siblings that were explicitly selected above.
             answer_take=int(answer_take_by_book.get(str(bid),base_cheap_per_book))
-            for answer_rank, item in enumerate(items[:answer_take], start=1):
+            answer_items=list(items[:answer_take])
+            if set_valued_request and set_sibling_indices:
+                sibling_ids={
+                    str(self.df.iloc[int(i)].record_id)
+                    for i in set_sibling_indices
+                    if str(self.df.iloc[int(i)].book_id)==str(bid)
+                }
+                existing_ids={clean_ui(item.get("record_id","")) for item in answer_items}
+                for item in items:
+                    rid=clean_ui(item.get("record_id",""))
+                    if rid in sibling_ids and rid not in existing_ids:
+                        answer_items.append(item)
+                        existing_ids.add(rid)
+
+            for answer_rank, item in enumerate(answer_items, start=1):
                 answer_item = dict(item)
                 answer_item["book_rank"] = answer_rank
                 answer_item["book_records_searched"] = int(self.book_record_counts.get(bid,0))
+                answer_item["set_local_sibling"] = bool(
+                    clean_ui(item.get("record_id","")) in {
+                        str(self.df.iloc[int(i)].record_id)
+                        for i in set_sibling_indices
+                    }
+                )
                 answer_pool.append(answer_item)
 
             chosen = items[:per_book_limit]
@@ -6529,7 +6692,10 @@ class ProfessionalEvidenceEngine:
                 "index_only_candidates_skipped":int(index_only_rows_skipped),"reranked_candidates":int(len(rerank_indices)),
                 "cheap_candidates_per_book":int(base_cheap_per_book),"answer_pool_count":len(answer_pool),"diverse_candidate_selection":True,"kb_native":self.kb_native,
                 "generic_request_type":generic_request_type,"set_valued_retrieval_expansion":bool(set_valued_request),
-                "set_coverage_deep_books":len(deep_books),"set_coverage_candidates_per_deep_book":int(deep_cheap_per_book if set_valued_request else base_cheap_per_book),
+                "set_recall_semantic_seeds":len(set_seed_indices),
+                "set_recall_local_siblings":len(set_sibling_indices),
+                "set_recall_reranked_siblings":len(set_recall_ranked),
+                "set_recall_page_radius":int(UI_CONFIG.get("SET_RECALL_PAGE_RADIUS",4)),
                 "top_score":float(max((x["score"] for x in all_items),default=0.0)),
                 "latency":time.perf_counter()-started,"array_latency":arrays_sec,"rerank_latency":rerank_sec,
                 "cache_hit":False,"minimum_score":requested,"retrieval_mode":mode,