improved the docs prep json txt

documents_prep.py

@@ -14,6 +14,10 @@ def extract_text_from_json(data, document_id, document_name):
             section_id = section.get('section_id', 'Unknown')
             section_text = section.get('section_text', '')
             
+            # Create hierarchical path for better context
+            section_path = f"{section_id}"
+            section_title = extract_section_title(section_text)
+            
             if section_text.strip():
                 doc = Document(
                     text=section_text,
@@ -22,73 +26,120 @@ def extract_text_from_json(data, document_id, document_name):
                         "document_id": document_id,
                         "document_name": document_name,
                         "section_id": section_id,
-                        "level": "section"
+                        "section_text": section_title,  # Store section title
+                        "section_path": section_path,
+                        "level": "section",
+                        "parent_sections": []  # Empty for top level
                     }
                 )
                 documents.append(doc)
             
+            # Process subsections with inherited context
             if 'subsections' in section:
                 for subsection in section['subsections']:
                     subsection_id = subsection.get('subsection_id', 'Unknown')
                     subsection_text = subsection.get('subsection_text', '')
+                    subsection_title = extract_section_title(subsection_text)
+                    subsection_path = f"{section_path}.{subsection_id}"
                     
                     if subsection_text.strip():
+                        # Include parent context in the text
+                        enhanced_text = f"[Раздел {section_id} {section_title}]\n{subsection_text}"
+                        
                         doc = Document(
-                            text=subsection_text,
+                            text=enhanced_text,
                             metadata={
                                 "type": "text",
                                 "document_id": document_id,
                                 "document_name": document_name,
-                                "section_id": section_id,
-                                "subsection_id": subsection_id,
-                                "level": "subsection"
+                                "section_id": subsection_id,
+                                "section_text": subsection_title,
+                                "section_path": subsection_path,
+                                "level": "subsection",
+                                "parent_sections": [{"id": section_id, "title": section_title}]
                             }
                         )
                         documents.append(doc)
                     
+                    # Process sub_subsections
                     if 'sub_subsections' in subsection:
                         for sub_subsection in subsection['sub_subsections']:
                             sub_subsection_id = sub_subsection.get('sub_subsection_id', 'Unknown')
                             sub_subsection_text = sub_subsection.get('sub_subsection_text', '')
+                            sub_subsection_title = extract_section_title(sub_subsection_text)
+                            sub_subsection_path = f"{subsection_path}.{sub_subsection_id}"
                             
                             if sub_subsection_text.strip():
+                                # Include full hierarchical context
+                                enhanced_text = f"[Раздел {section_id} {section_title}]\n[Подраздел {subsection_id} {subsection_title}]\n{sub_subsection_text}"
+                                
                                 doc = Document(
-                                    text=sub_subsection_text,
+                                    text=enhanced_text,
                                     metadata={
                                         "type": "text",
                                         "document_id": document_id,
                                         "document_name": document_name,
-                                        "section_id": section_id,
-                                        "subsection_id": subsection_id,
-                                        "sub_subsection_id": sub_subsection_id,
-                                        "level": "sub_subsection"
+                                        "section_id": sub_subsection_id,
+                                        "section_text": sub_subsection_title,
+                                        "section_path": sub_subsection_path,
+                                        "level": "sub_subsection",
+                                        "parent_sections": [
+                                            {"id": section_id, "title": section_title},
+                                            {"id": subsection_id, "title": subsection_title}
+                                        ]
                                     }
                                 )
                                 documents.append(doc)
                             
+                            # Process sub_sub_subsections
                             if 'sub_sub_subsections' in sub_subsection:
                                 for sub_sub_subsection in sub_subsection['sub_sub_subsections']:
                                     sub_sub_subsection_id = sub_sub_subsection.get('sub_sub_subsection_id', 'Unknown')
                                     sub_sub_subsection_text = sub_sub_subsection.get('sub_sub_subsection_text', '')
+                                    sub_sub_subsection_title = extract_section_title(sub_sub_subsection_text)
                                     
                                     if sub_sub_subsection_text.strip():
+                                        # Full context chain
+                                        enhanced_text = f"[Раздел {section_id} {section_title}]\n[Подраздел {subsection_id} {subsection_title}]\n[Подподраздел {sub_subsection_id} {sub_subsection_title}]\n{sub_sub_subsection_text}"
+                                        
                                         doc = Document(
-                                            text=sub_sub_subsection_text,
+                                            text=enhanced_text,
                                             metadata={
                                                 "type": "text",
                                                 "document_id": document_id,
                                                 "document_name": document_name,
-                                                "section_id": section_id,
-                                                "subsection_id": subsection_id,
-                                                "sub_subsection_id": sub_subsection_id,
-                                                "sub_sub_subsection_id": sub_sub_subsection_id,
-                                                "level": "sub_sub_subsection"
+                                                "section_id": sub_sub_subsection_id,
+                                                "section_text": sub_sub_subsection_title,
+                                                "section_path": f"{sub_subsection_path}.{sub_sub_subsection_id}",
+                                                "level": "sub_sub_subsection",
+                                                "parent_sections": [
+                                                    {"id": section_id, "title": section_title},
+                                                    {"id": subsection_id, "title": subsection_title},
+                                                    {"id": sub_subsection_id, "title": sub_subsection_title}
+                                                ]
                                             }
                                         )
                                         documents.append(doc)
     
     return documents
 
+def extract_section_title(section_text):
+    if not section_text.strip():
+        return ""
+    
+    lines = section_text.strip().split('\n')
+    first_line = lines[0].strip()
+    
+    if len(first_line) < 200 and not first_line.endswith('.'):
+        return first_line
+    
+    # Otherwise, extract first sentence
+    sentences = first_line.split('.')
+    if len(sentences) > 1:
+        return sentences[0].strip()
+    
+    return first_line[:100] + "..." if len(first_line) > 100 else first_line
+
 def extract_zip_and_process_json(zip_path):
     documents = []
     

utils.py

@@ -51,10 +51,24 @@ def format_context_for_llm(nodes):
         doc_id = metadata.get('document_id', 'Неизвестный документ')
         
         section_info = ""
-        if metadata.get('section_id') and metadata.get('section_text'):
-            section_info = f"пункт {metadata['section_id']} {metadata['section_text']}"
-
         
+        # Handle hierarchical section information
+        if metadata.get('section_path'):
+            section_path = metadata['section_path']
+            section_text = metadata.get('section_text', '')
+            if section_text:
+                section_info = f"пункт {section_path} ({section_text})"
+            else:
+                section_info = f"пункт {section_path}"
+        elif metadata.get('section_id'):
+            section_id = metadata['section_id']
+            section_text = metadata.get('section_text', '')
+            if section_text:
+                section_info = f"пункт {section_id} ({section_text})"
+            else:
+                section_info = f"пункт {section_id}"
+
+        # Handle tables and images as before
         if metadata.get('type') == 'table' and metadata.get('table_number'):
             table_num = metadata['table_number']
             if not str(table_num).startswith('№'):
@@ -139,7 +153,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
             log_message(f"Пример узла {i+1}: {retrieved_nodes[i].text[:200]}...")
         
         log_message("Применяю переранжировку")
-        reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=15)
+        reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=10)
         
         formatted_context = format_context_for_llm(reranked_nodes)
         log_message(f"fорматированный контекст для LLM:\n{formatted_context[:500]}...")