Added topics endpoint

backend/app/main.py

@@ -10,6 +10,7 @@ import asyncio
 import logging
 import os
 from backend.app.crawler import DomainCrawler
+from backend.app.vectorstore import get_all_unique_source_of_docs_in_collection_DUMB
 
 app = FastAPI()
 
@@ -41,8 +42,12 @@ class FeedbackResponse(BaseModel):
     feedback: List[str]
 
 
-@app.post("/api/crawl/")
-async def crawl_documentation(input_data: UrlInput):
+class TopicsResponse(BaseModel):
+    sources: List[str]
+
+
+@app.post("/api/ingest/")
+async def ingest_documentation(input_data: UrlInput):
     print(f"Received url {input_data.url}")
     return {"status": "received"}
 
@@ -85,6 +90,12 @@ async def get_feedback(request: FeedbackRequest):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@app.post("/api/topics", response_model=TopicsResponse)
+async def get_topics():
+    sources = get_all_unique_source_of_docs_in_collection_DUMB()
+    return {"sources": sources}
+
+
 # Serve static files
 app.mount("/static", StaticFiles(directory="/app/static/static"), name="static")
 

backend/app/vectorstore.py

@@ -43,38 +43,6 @@ _embedding_model: Optional[Union[OpenAIEmbeddings, HuggingFaceEmbeddings]] = Non
 _embedding_model_id: str = None
 
 
-def _get_qdrant_client():
-    global _qdrant_client_instance
-
-    if _qdrant_client_instance is None:
-        if (
-            os.environ.get("QDRANT_URL") is None
-            or os.environ.get("QDRANT_API_KEY") is None
-        ):
-            logger.warning(
-                "QDRANT_URL or QDRANT_API_KEY is not set. Defaulting to local memory vector store."
-            )
-
-            os.makedirs(LOCAL_QDRANT_PATH, exist_ok=True)
-            _qdrant_client_instance = QdrantClient(path=LOCAL_QDRANT_PATH)
-            # _qdrant_client_instance = QdrantClient(":memory:")
-            return _qdrant_client_instance
-
-        logger.info(
-            f"Attempting to connect to Qdrant at {os.environ.get("QDRANT_URL")}"
-        )
-        try:
-            _qdrant_client_instance = QdrantClient(
-                url=os.environ.get("QDRANT_URL"),
-                api_key=os.environ.get("QDRANT_API_KEY"),
-            )
-            logger.info("Successfully connected to Qdrant Cloud")
-        except Exception as e:
-            logger.error(f"Failed to connect to Qdrant Cloud: {str(e)}")
-            raise e
-    return _qdrant_client_instance
-
-
 def _initialize_vector_db():
     os.makedirs("static/data", exist_ok=True)
 
@@ -112,10 +80,44 @@ def _initialize_vector_db():
     )
 
 
-def get_all_unique_source_docs_in_collection(
-    collection_name: str, client: QdrantClient, limit: int = 1000, offset: int = 0
+def get_qdrant_client():
+    global _qdrant_client_instance
+
+    if _qdrant_client_instance is None:
+        if (
+            os.environ.get("QDRANT_URL") is None
+            or os.environ.get("QDRANT_API_KEY") is None
+        ):
+            logger.warning(
+                "QDRANT_URL or QDRANT_API_KEY is not set. Defaulting to local memory vector store."
+            )
+
+            os.makedirs(LOCAL_QDRANT_PATH, exist_ok=True)
+            _qdrant_client_instance = QdrantClient(path=LOCAL_QDRANT_PATH)
+            # _qdrant_client_instance = QdrantClient(":memory:")
+            return _qdrant_client_instance
+
+        logger.info(
+            f"Attempting to connect to Qdrant at {os.environ.get("QDRANT_URL")}"
+        )
+        try:
+            _qdrant_client_instance = QdrantClient(
+                url=os.environ.get("QDRANT_URL"),
+                api_key=os.environ.get("QDRANT_API_KEY"),
+            )
+            logger.info("Successfully connected to Qdrant Cloud")
+        except Exception as e:
+            logger.error(f"Failed to connect to Qdrant Cloud: {str(e)}")
+            raise e
+    return _qdrant_client_instance
+
+
+def get_all_unique_source_of_docs_in_collection(
+    collection_name: str = PROBLEMS_REFERENCE_COLLECTION_NAME,
+    limit: int = 1000,
+    offset: int = 0,
 ) -> List[Document]:
-    response = client.scroll(
+    response = get_qdrant_client().scroll(
         collection_name=collection_name,
         limit=limit,
         offset=offset,
@@ -128,7 +130,7 @@ def get_all_unique_source_docs_in_collection(
             if "source" in point.payload:
                 result.add(point.payload["source"])
         offset = response[1]
-        response = client.scroll(
+        response = get_qdrant_client().scroll(
             collection_name=collection_name,
             limit=limit,
             offset=offset + limit,
@@ -136,6 +138,23 @@ def get_all_unique_source_docs_in_collection(
     return list(result)
 
 
+# TODO This is a dumb hack to get around Qdrant client restrictions when using local file storage.
+# Instead of using the client directly, we use QdrantVectorStore's similarity search 
+# with a dummy query to get all documents, then extract unique sources.
+def get_all_unique_source_of_docs_in_collection_DUMB(
+    collection_name: str = PROBLEMS_REFERENCE_COLLECTION_NAME,
+) -> List[str]:
+    vector_store = get_vector_db()
+    # Use a very generic query that should match everything
+    docs = vector_store.similarity_search("",k=1000)
+    
+    sources = set()
+    for doc in docs:
+        if doc.metadata and "title" in doc.metadata:
+            sources.add(doc.metadata["title"])
+    return list(sources)
+
+
 def store_documents(
     documents: List[Document],
     collection_name: str,
@@ -145,7 +164,7 @@ def store_documents(
     assert _vector_db_instance is not None, "Vector database instance not initialized"
 
     embedding_model = get_embedding_model(embedding_model_id)
-    client = _get_qdrant_client()
+    client = get_qdrant_client()
 
     _vector_db_instance.add_documents(
         documents=documents,
@@ -181,7 +200,7 @@ def get_vector_db(embedding_model_id: str = None) -> QdrantVectorStore:
         need_to_initialize_db = False
         embedding_model = get_embedding_model(embedding_model_id)
 
-        client = _get_qdrant_client()
+        client = get_qdrant_client()
 
         if not check_collection_exists(client, PROBLEMS_REFERENCE_COLLECTION_NAME):
             client.create_collection(

backend/tests/test_api.py

@@ -6,7 +6,7 @@ client = TestClient(app)
 
 
 def test_crawl_endpoint():
-    response = client.post("/api/crawl/", json={"url": "https://example.com"})
+    response = client.post("/api/ingest/", json={"url": "https://example.com"})
     assert response.status_code == 200
     assert response.json() == {"status": "received"}
 
@@ -61,3 +61,14 @@ def test_successful_feedback():
     for feedback in result["feedback"]:
         assert feedback.strip().startswith(("Correct", "Incorrect"))
         assert len(feedback.split(". ")) >= 2
+
+
+def test_topics_endpoint():
+    """Test that topics endpoint returns expected sources"""
+    response = client.post("/api/topics")
+    assert response.status_code == 200
+    result = response.json()
+    
+    assert "sources" in result
+    assert len(result["sources"]) == 1
+    assert result["sources"][0] == "LangChain RAG Tutorial"

backend/tests/test_vectorstore.py

@@ -4,7 +4,7 @@ import pytest
 import requests
 
 from langchain.schema import Document
-from backend.app.vectorstore import get_vector_db, _get_qdrant_client
+from backend.app.vectorstore import get_vector_db, get_qdrant_client
 
 
 def test_directory_creation():
@@ -72,7 +72,7 @@ def test_qdrant_cloud_connection():
         print(f"Port: {parsed_url.port}")
         print(f"Path: {parsed_url.path}")
 
-        client = _get_qdrant_client()
+        client = get_qdrant_client()
         client.get_collections()
         assert True, "Connection successful"
     except Exception as e: