refactor

src/pipeline.py

@@ -1,5 +1,7 @@
 """Main pipeline orchestrator for the Audit QA system."""
+import os
 import time
+import traceback
 from pathlib import Path
 from dataclasses import dataclass
 from typing import Dict, Any, List, Optional
@@ -11,11 +13,21 @@ except ModuleNotFoundError as me:
     from langchain.schema import Document
 
 from .logging import log_error
-from .llm.adapters import LLMRegistry
+
 from .loader import chunks_to_documents
 from .vectorstore import VectorStoreManager
+from .reporting.service import ReportService
 from .retrieval.context import ContextRetriever
-from .config.loader import get_embedding_model_for_collection
+from .llm.adapters import LLMRegistry, get_llm_client
+from .llm.templates import DEFAULT_AUDIT_SYSTEM_PROMPT
+from .config.loader import load_config, get_embedding_model_for_collection
+from .retrieval.filter import get_available_metadata, infer_filters_from_query
+
+try:
+    from langchain_openai import ChatOpenAI
+    LANGCHAIN_OPENAI_AVAILABLE = True
+except ImportError:
+    LANGCHAIN_OPENAI_AVAILABLE = False
 
 
 
@@ -41,12 +53,13 @@ class PipelineManager:
         """
         Initialize the pipeline manager.
         """
+        self.chunks = None
+        self.llm_client = None
         self.config = config or {}
+        self.report_service = None
         self.vectorstore_manager = None
         self.context_retriever = None  # Initialize as None
-        self.llm_client = None
-        self.report_service = None
-        self.chunks = None
+    
         
         # Initialize components
         self._initialize_components()
@@ -118,13 +131,7 @@ class PipelineManager:
         try:
             # Load config if not provided
             if not self.config:
-                try:
-                    from src.config.loader import load_config
-                    self.config = load_config()
-                except ImportError:
-                    # Try alternate import path
-                    from src.config.loader import load_config
-                    self.config = load_config()
+                self.config = load_config()
             
             # Validate config structure
             if not isinstance(self.config, dict):
@@ -159,7 +166,6 @@ class PipelineManager:
                 print("✅ VectorStoreManager initialized successfully")
             except Exception as vs_error:
                 print(f"❌ Error initializing VectorStoreManager: {vs_error}")
-                import traceback
                 traceback.print_exc()
                 self.vectorstore_manager = None
                 raise  # Re-raise to be caught by outer try-except
@@ -175,40 +181,35 @@ class PipelineManager:
             except Exception as e:
                 try:
                     # Try direct instantiation with config
-                    from src.llm.adapters import get_llm_client
                     self.llm_client = get_llm_client("openai", self.config)
                     print("✅ LLM CLIENT: Initialized using direct get_llm_client function with config")
                 except Exception as e2:
                     print(f"❌ LLM CLIENT: Registry methods failed - {e2}")
                     # Try to create a simple LLM client directly
                     try:
-                        from langchain_openai import ChatOpenAI
-                        import os
-                        api_key = os.getenv("OPENAI_API_KEY") or os.getenv("OPENROUTER_API_KEY")
-                        if api_key:
-                            self.llm_client = ChatOpenAI(
-                                model="gpt-3.5-turbo",
-                                api_key=api_key,
-                                temperature=0.1,
-                                max_tokens=1000
-                            )
-                            print("✅ LLM CLIENT: Initialized using direct ChatOpenAI")
+                        if LANGCHAIN_OPENAI_AVAILABLE:
+                            api_key = os.getenv("OPENAI_API_KEY") or os.getenv("OPENROUTER_API_KEY")
+                            if api_key:
+                                self.llm_client = ChatOpenAI(
+                                    model="gpt-3.5-turbo",
+                                    api_key=api_key,
+                                    temperature=0.1,
+                                    max_tokens=1000
+                                )
+                                print("✅ LLM CLIENT: Initialized using direct ChatOpenAI")
+                            else:
+                                print("❌ LLM CLIENT: No API key available")
                         else:
-                            print("❌ LLM CLIENT: No API key available")
+                            print("❌ LLM CLIENT: langchain-openai not available")
                     except Exception as e3:
                         print(f"❌ LLM CLIENT: Direct instantiation also failed - {e3}")
                         self.llm_client = None
             
             # Load system prompt
-            from src.llm.templates import DEFAULT_AUDIT_SYSTEM_PROMPT
             self.system_prompt = DEFAULT_AUDIT_SYSTEM_PROMPT
             
             # Initialize report service
             try:
-                try:
-                    from src.reporting.service import ReportService
-                except ImportError:
-                    from src.reporting.service import ReportService
                 self.report_service = ReportService()
             except Exception as e:
                 print(f"Warning: Could not initialize report service: {e}")
@@ -216,7 +217,6 @@ class PipelineManager:
             
         except Exception as e:
             print(f"❌ Error initializing components: {e}")
-            import traceback
             traceback.print_exc()
             # Don't set vectorstore_manager to None if it was already set
             if not hasattr(self, 'vectorstore_manager') or self.vectorstore_manager is None:
@@ -337,7 +337,6 @@ class PipelineManager:
                     return False
             except Exception as init_error:
                 print(f"❌ Error initializing vector store manager: {init_error}")
-                import traceback
                 traceback.print_exc()
                 return False
         
@@ -352,7 +351,6 @@ class PipelineManager:
         except Exception as e:
             print(f"❌ Error connecting to vector store: {e}")
             log_error(e, {"component": "vectorstore_connection"})
-            import traceback
             traceback.print_exc()
             
             # If it's a dimension mismatch error, try with force_recreate
@@ -541,9 +539,6 @@ Answer:"""
             if auto_infer_filters and not any([reports, sources, subtype]):
                 print(f"🤖 AUTO-INFERRING FILTERS: No explicit filters provided, analyzing query...")
                 try:
-                    # Import get_available_metadata here to avoid circular imports
-                    from src.retrieval.filter import get_available_metadata, infer_filters_from_query
-                    
                     # Get available metadata
                     available_metadata = get_available_metadata(self.vectorstore_manager.get_vectorstore())