Update app/policy_vector_db.py

app/policy_vector_db.py

@@ -10,7 +10,6 @@ import logging
 logger = logging.getLogger("app")
 
 class PolicyVectorDB:
-    # ... (the class code is correct and does not need changes) ...
     def __init__(self, persist_directory: str, top_k_default: int = 5, relevance_threshold: float = 0.5):
         self.persist_directory = persist_directory
         self.client = chromadb.PersistentClient(path=persist_directory, settings=Settings(allow_reset=True))
@@ -55,7 +54,16 @@ class PolicyVectorDB:
             batch = new_chunks[i:i + batch_size]
             texts = [chunk['text'] for chunk in batch]
             ids = [str(chunk['id']) for chunk in batch]
-            metadatas = [self._flatten_metadata(chunk['metadata']) if chunk.get('metadata') else {} for chunk in batch]
+
+            # --- FIX: Ensure metadata is never an empty dictionary ---
+            metadatas = []
+            for chunk in batch:
+                meta = chunk.get('metadata')
+                if not meta:  # Handles cases where metadata is missing or empty
+                    # Provide a default, non-empty metadata dictionary
+                    meta = {"description": "General information chunk."}
+                metadatas.append(self._flatten_metadata(meta))
+            
             embeddings = self.embedding_model.encode(texts, show_progress_bar=False).tolist()
             collection.add(ids=ids, embeddings=embeddings, documents=texts, metadatas=metadatas)
             logger.info(f"Added batch {i//batch_size + 1}/{(len(new_chunks) + batch_size - 1) // batch_size}")
@@ -90,9 +98,12 @@ def ensure_db_populated(db_instance: PolicyVectorDB, chunks_file_path: str):
                 logger.error(f"Chunks file not found at {chunks_file_path}. Cannot populate DB.")
                 return False
             
-            # --- REVERTED: This is the correct method for a standard .json file ---
+            chunks_to_add = []
             with open(chunks_file_path, 'r', encoding='utf-8') as f:
-                chunks_to_add = json.load(f)
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        chunks_to_add.append(json.loads(line))
             
             if not chunks_to_add:
                 logger.warning(f"Chunks file at {chunks_file_path} is empty. No data to add to DB.")