v3.1 (#3)

- Elastic config updates (468a9ee9b833e3d2ced7be5e7c461345dbb22b86)
- Update ES sources config (64b9f1e6dc6b24f397665d22bc12443295b3bcce)
- Update KB service to use new ES indices (08718d549b2a1271c092a0f73eab727a34fc5f84)

ask_candid/base/config/connections.py

@@ -20,10 +20,11 @@ class BaseElasticAPIKeyCredential:
     api_key: str | None = field(default_factory=str)
 
 
-SEMANTIC_ELASTIC_QA = BaseElasticAPIKeyCredential(
+SEMANTIC_ELASTIC = BaseElasticAPIKeyCredential(
     cloud_id=_load_value("SEMANTIC_ELASTIC_CLOUD_ID"),
     api_key=_load_value("SEMANTIC_ELASTIC_API_KEY"),
 )
+ELSER_INFERENCE_ID = _load_value("ELSER_INFERENCE_ID") or "elser_model_2_linux-x86_64_search"
 
 SEMANTIC_ELASTIC_QA_WRITER = BaseElasticAPIKeyCredential(
     cloud_id=_load_value("SEMANTIC_ELASTIC_WRITER_CLOUD_ID"),

ask_candid/base/retrieval/sources.py

@@ -3,7 +3,7 @@ from ask_candid.base.retrieval.schemas import ElasticSourceConfig
 
 CandidBlogConfig = ElasticSourceConfig(
     index_name="search-semantic-blog",
-    semantic_fields=("semantic_title_summary_tags_text", "semantic_authors_text","semantic_content"),
+    semantic_fields=("semantic_title_summary_tags_text", "semantic_authors_text", "semantic_content"),
     text_fields=("title", "summary", "content", "authors_text"),
     highlight_fields=("semantic_content",),
     excluded_fields=("content",)
@@ -11,14 +11,27 @@ CandidBlogConfig = ElasticSourceConfig(
 
 
 CandidHelpConfig = ElasticSourceConfig(
-    index_name="search-semantic-candid-help-elser_ve1",
-    semantic_fields=("content", "combined_article_description")
+    index_name="search-semantic-help",
+    semantic_fields=("semantic_content", "semantic_title_summary_question_category"),
+    text_fields=("title", "summary", "content_question"),
+    highlight_fields=("semantic_content",),
+    excluded_fields=("content_html", "content")
 )
 
 
 CandidLearningConfig = ElasticSourceConfig(
-    index_name="search-semantic-candid-learning_ve1",
-    semantic_fields=("content", "title", "training_topics", "staff_recommendations")
+    index_name="search-semantic-learning",
+    semantic_fields=("semantic_title_short_description", "semantic_lessons_description","semantic_lessons_content"),
+    text_fields=("title", "short_description", "lesson_list.description", "lessson_content.content"),
+    highlight_fields=("semantic_lessons_content",),
+    excluded_fields=(
+        "lesson_content.content_html",
+        "lesson_list.description_html",
+        "semantic_lessons_content",
+        "semantic_lessons_description",
+        "lesson_content.content",
+        "lesson_list.description"
+    )
 )
 
 

ask_candid/services/knowledge_base.py

@@ -6,7 +6,7 @@ import logging
 from langchain_core.documents import Document
 
 from ask_candid.base.retrieval.elastic import (
-    build_sparse_vector_query,
+    # build_sparse_vector_query,
     build_sparse_vector_and_text_query,
     news_query_builder,
     issuelab_query_builder,
@@ -15,7 +15,7 @@ from ask_candid.base.retrieval.elastic import (
 from ask_candid.base.retrieval.sparse_lexical import SpladeEncoder
 from ask_candid.base.retrieval.schemas import ElasticHitsResult
 import ask_candid.base.retrieval.sources as S
-from ask_candid.base.config.connections import SEMANTIC_ELASTIC_QA, NEWS_ELASTIC
+from ask_candid.base.config.connections import SEMANTIC_ELASTIC, ELSER_INFERENCE_ID, NEWS_ELASTIC
 from ask_candid.services.small_lm import CandidSmallLanguageModel
 
 SourceNames = Literal[
@@ -32,7 +32,6 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 
-# TODO remove
 def get_context(field_name: str, hit: ElasticHitsResult, context_length: int = 1024, add_context: bool = True) -> str:
     """Pads the relevant chunk of text with context before and after
 
@@ -123,18 +122,31 @@ def generate_queries(
                 semantic_fields=S.CandidBlogConfig.semantic_fields,
                 text_fields=S.CandidBlogConfig.text_fields,
                 highlight_fields=S.CandidBlogConfig.highlight_fields,
-                excluded_fields=S.CandidBlogConfig.excluded_fields
+                excluded_fields=S.CandidBlogConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
             )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidBlogConfig.index_name}, q])
         elif source_name == "Candid Help":
-            q = build_sparse_vector_query(query=query, fields=S.CandidHelpConfig.semantic_fields)
-            q["_source"] = {"excludes": ["embeddings"]}
+            q = build_sparse_vector_and_text_query(
+                query=query,
+                semantic_fields=S.CandidHelpConfig.semantic_fields,
+                text_fields=S.CandidHelpConfig.text_fields,
+                highlight_fields=S.CandidHelpConfig.highlight_fields,
+                excluded_fields=S.CandidHelpConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
+            )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidHelpConfig.index_name}, q])
         elif source_name == "Candid Learning":
-            q = build_sparse_vector_query(query=query, fields=S.CandidLearningConfig.semantic_fields)
-            q["_source"] = {"excludes": ["embeddings"]}
+            q = build_sparse_vector_and_text_query(
+                query=query,
+                semantic_fields=S.CandidLearningConfig.semantic_fields,
+                text_fields=S.CandidLearningConfig.text_fields,
+                highlight_fields=S.CandidLearningConfig.highlight_fields,
+                excluded_fields=S.CandidLearningConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
+            )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidLearningConfig.index_name}, q])
         elif source_name == "Candid News":
@@ -166,7 +178,8 @@ def generate_queries(
                 semantic_fields=S.YoutubeConfig.semantic_fields,
                 text_fields=S.YoutubeConfig.text_fields,
                 highlight_fields=S.YoutubeConfig.highlight_fields,
-                excluded_fields=S.YoutubeConfig.excluded_fields
+                excluded_fields=S.YoutubeConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
             )
             q["size"] = 5
             vector_queries.extend([{"index": S.YoutubeConfig.index_name}, q])
@@ -215,7 +228,7 @@ def run_search(
 
     results = []
     if vector_searches is not None and len(vector_searches) > 0:
-        hits = multi_search_base(queries=vector_searches, credentials=SEMANTIC_ELASTIC_QA)
+        hits = multi_search_base(queries=vector_searches, credentials=SEMANTIC_ELASTIC)
         for hit in _msearch_response_generator(responses=hits):
             results.append(hit)
     if non_vector_searches is not None and len(non_vector_searches) > 0:
@@ -368,6 +381,7 @@ def process_hit(hit: ElasticHitsResult) -> Document:
         )
     elif "blog" in hit.index:
         highlight = hit.highlight or {}
+        blog_url = hit.source.get("link", "")
         doc = Document(
             page_content='\n\n'.join([
                 hit.source.get("title_summary_tags_text", ""),
@@ -378,35 +392,35 @@ def process_hit(hit: ElasticHitsResult) -> Document:
                 "title": hit.source.get("title", ""),
                 "source": "Candid Blog",
                 "source_id": hit.source["id"],
-                "url": hit.source["link"]
+                "url": blog_url
             }
         )
-    elif "candid-learning" in hit.index:
+    elif "learning" in hit.index:
+        highlight = hit.highlight or {}
         doc = Document(
             page_content='\n\n'.join([
-                hit.source.get("title", ""),
-                hit.source.get("staff_recommendations", ""),
-                hit.source.get("training_topics", ""),
-                get_context("content", hit, context_length=12)
+                hit.source.get("semantic_title_short_description", ""),
+                ' '.join(highlight.get("semantic_lessons_content", []))
             ]),
             metadata={
                 "title": hit.source["title"],
                 "source": "Candid Learning",
-                "source_id": hit.source["post_id"],
-                "url": hit.source.get("url", "")
+                "source_id": hit.source["course_id"],
+                "url": hit.source.get("course_url", "")
             }
         )
-    elif "candid-help" in hit.index:
+    elif "help" in hit.index:
+        highlight = hit.highlight or {}
         doc = Document(
             page_content='\n\n'.join([
-                hit.source.get("combined_article_description", ""),
-                get_context("content", hit, context_length=12)
+                hit.source.get("semantic_title_summary_question_category", ""),
+                ' '.join(highlight.get("semantic_content", []))
             ]),
             metadata={
                 "title": hit.source.get("title", ""),
                 "source": "Candid Help",
-                "source_id": hit.source["id"],
-                "url": hit.source.get("link", "")
+                "source_id": hit.source["article_id"],
+                "url": f"""https://help.candid.org/s/article/{hit.source.get("url", "")}"""
             }
         )
     elif "news" in hit.index: