Update create_granular_chunks.py

create_granular_chunks.py

@@ -24,7 +24,6 @@ def create_chunk(context: Dict, text: str) -> Dict:
         "title": context.get("title"),
         "source_description": context.get("description"),
     }
-    # Add any other relevant context keys to metadata
     for key, value in context.items():
         if key not in metadata and isinstance(value, (str, int, float, bool)):
             metadata[key] = value
@@ -36,14 +35,20 @@ def create_chunk(context: Dict, text: str) -> Dict:
     }
 
 def format_delegation_text(delegation: Any) -> str:
-    """Formats a delegation dictionary or string into a readable string."""
+    """
+    Formats a delegation dictionary or string into a readable string.
+    --- ACCURACY FIX ---
+    This function now explicitly includes "NIL" or "---" values instead of skipping them.
+    This is crucial for the model to correctly answer questions about roles with no power.
+    """
     if not isinstance(delegation, dict):
         return str(delegation)
-    parts = [f"the limit for {auth} is {limit}" for auth, limit in delegation.items() if limit and str(limit).lower() != 'nil']
+    # Use "is NIL" for None or "---", otherwise use "is [limit]"
+    parts = [f"the limit for {auth} is {limit if limit and str(limit) != '---' else 'NIL'}" for auth, limit in delegation.items()]
     return ", ".join(parts) if parts else "No specific delegation provided."
 
 def format_remarks(remarks: Any) -> str:
-    """Safely formats the 'remarks' field, handling strings, lists of strings, and lists of dictionaries."""
+    """Safely formats the 'remarks' field, handling various data types."""
     if isinstance(remarks, list):
         remark_parts = []
         for item in remarks:
@@ -57,22 +62,18 @@ def format_remarks(remarks: Any) -> str:
 
 def build_descriptive_text(context: Dict) -> str:
     """
-    BEST PRACTICE: Intelligently builds a single, descriptive, natural language sentence 
-    by combining all relevant fields from the context. This creates the most semantically
-    rich chunk possible for the embedding model.
+    Intelligently builds a single, descriptive, natural language sentence 
+    by combining all relevant fields from the context.
     """
     text_parts = []
     
-    # Start with the highest-level identifiers
     if context.get("title"):
         text_parts.append(f"Regarding the policy for '{context['title']}'")
 
-    # Add the specific description, method, or sub-clause title
-    specific_desc = context.get('description') or context.get('method') or context.get('title')
+    specific_desc = context.get('description') or context.get('method')
     if specific_desc and specific_desc != context.get('title'):
          text_parts.append(f"specifically for '{specific_desc}'")
 
-    # Add the core rule or delegation information
     if "delegation" in context:
         delegation_text = format_delegation_text(context["delegation"])
         text_parts.append(f", the financial delegations are: {delegation_text}.")
@@ -85,7 +86,6 @@ def build_descriptive_text(context: Dict) -> str:
                     composition_parts.append(member_text)
         text_parts.append(f", the composition is: {'; '.join(composition_parts)}.")
     
-    # Append any remarks for crucial extra context
     if "remarks" in context and context["remarks"]:
         remarks_text = format_remarks(context["remarks"])
         text_parts.append(f" Important remarks include: {remarks_text}")
@@ -101,7 +101,6 @@ def process_entry(data: Dict, parent_context: Dict = None) -> List[Dict]:
     chunks = []
 
     # --- Handler 1: Simple Item Lists (e.g., Annexure A, Financial Concurrence) ---
-    # This creates small, specific chunks for each item in a list.
     list_key = next((key for key in ["items", "exclusions"] if key in data and isinstance(data.get(key), list)), None)
     if list_key:
         base_title = context.get('title', 'a policy')
@@ -111,7 +110,6 @@ def process_entry(data: Dict, parent_context: Dict = None) -> List[Dict]:
         return chunks
 
     # --- Handler 2: Recursive Traversal ---
-    # This is the main recursive step. It traverses deeper into nested lists of dictionaries.
     has_recursed = False
     for key, value in data.items():
         if isinstance(value, list) and value and all(isinstance(item, dict) for item in value):
@@ -120,9 +118,6 @@ def process_entry(data: Dict, parent_context: Dict = None) -> List[Dict]:
             has_recursed = True
     
     # --- Handler 3: Leaf Node Creation ---
-    # If the function has not recursed and the entry contains a core piece of information
-    # (like 'delegation' or 'composition'), it's a "leaf" node. We create a single,
-    # comprehensive chunk for it. This is the core of the new strategy.
     if not has_recursed and ("delegation" in data or "composition" in data or "description" in data):
         text = build_descriptive_text(context)
         chunks.append(create_chunk(context, text))