03_System_Tools/luuna_orchestrator.py

#!/usr/bin/env python3
"""
Luuna Automated Workspace Orchestrator
Main integration system that coordinates all components of the automated workspace workflow
"""

import os
import sys
import json
import time
import logging
from pathlib import Path
from typing import Dict, List, Optional
import threading
from dataclasses import dataclass

# Import our custom modules
from workspace_monitor_service import WorkspaceMonitorService
from document_processor import DocumentBatchProcessor
from task_orchestrator import TaskOrchestrator
from construction_engine import ConstructionEngine

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('luuna_orchestrator.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

@dataclass
class WorkspacePipelineResult:
    """Results from processing a workspace through the complete pipeline"""
    workspace_path: str
    workspace_name: str
    status: str
    processing_steps: Dict[str, any]
    generated_artifacts: Dict[str, any]
    timestamp: float
    error: Optional[str] = None

class LuunaOrchestrator:
    """Main orchestrator that integrates all Luuna workspace automation components"""
    
    def __init__(self, workspaces_base_path: str = "c:\\Luuna\\WORKSPACES"):
        self.workspaces_base_path = Path(workspaces_base_path)
        self.workspaces_base_path.mkdir(parents=True, exist_ok=True)
        
        # Initialize component systems
        self.monitor_service = WorkspaceMonitorService(str(self.workspaces_base_path))
        self.document_processor = DocumentBatchProcessor()
        self.task_orchestrator = TaskOrchestrator()
        self.construction_engine = ConstructionEngine()
        
        # Track processing results
        self.processing_results: Dict[str, WorkspacePipelineResult] = {}
        self.running = False
        
        logger.info("Luuna Orchestrator initialized")
        
    def start_autonomous_operation(self):
        """Start the complete autonomous workspace processing system"""
        if self.running:
            logger.warning("Orchestrator is already running")
            return
            
        logger.info("Starting Luuna Autonomous Workspace Orchestrator")
        self.running = True
        
        try:
            # Start workspace monitoring in background
            monitor_thread = threading.Thread(target=self._run_monitor_service, daemon=True)
            monitor_thread.start()
            
            # Process any existing workspaces
            self._process_existing_workspaces()
            
            # Main orchestration loop
            self._run_orchestration_loop()
            
        except KeyboardInterrupt:
            logger.info("Received shutdown signal")
        finally:
            self.shutdown()
            
    def _run_monitor_service(self):
        """Run the workspace monitoring service"""
        try:
            self.monitor_service.start_monitoring()
        except Exception as e:
            logger.error(f"Monitor service error: {e}")
            
    def _process_existing_workspaces(self):
        """Process any workspaces that already exist"""
        logger.info("Processing existing workspaces...")
        
        for item in self.workspaces_base_path.iterdir():
            if item.is_dir() and not item.name.startswith('.'):
                self._process_workspace_pipeline(item)
                
    def _run_orchestration_loop(self):
        """Main orchestration loop"""
        logger.info("Entering orchestration loop - waiting for new workspaces...")
        
        while self.running:
            # Check for completed workspace processing
            self._check_completed_processes()
            
            # Handle any queued operations
            self._handle_queued_operations()
            
            time.sleep(5)  # Check every 5 seconds
            
    def _process_workspace_pipeline(self, workspace_path: Path) -> Optional[WorkspacePipelineResult]:
        """Process a workspace through the complete automation pipeline"""
        workspace_name = workspace_path.name
        logger.info(f"Starting pipeline processing for workspace: {workspace_name}")
        
        try:
            # Initialize result tracking
            pipeline_result = WorkspacePipelineResult(
                workspace_path=str(workspace_path),
                workspace_name=workspace_name,
                status='processing',
                processing_steps={},
                generated_artifacts={},
                timestamp=time.time()
            )
            
            # Step 1: Document Processing and Analysis
            logger.info(f"[{workspace_name}] Step 1: Document Processing")
            document_results = self.document_processor.process_directory(workspace_path)
            pipeline_result.processing_steps['document_processing'] = {
                'status': 'completed',
                'files_processed': len(document_results['processed_files']),
                'files_failed': len(document_results['failed_files']),
                'timestamp': time.time()
            }
            
            # Step 2: Task Generation and Distribution
            logger.info(f"[{workspace_name}] Step 2: Task Orchestration")
            orchestration_results = self.task_orchestrator.process_workspace_documents(
                str(workspace_path), document_results
            )
            pipeline_result.processing_steps['task_orchestration'] = {
                'status': 'completed',
                'tasks_generated': orchestration_results['total_tasks_generated'],
                'tasks_assigned': orchestration_results['tasks_assigned'],
                'timestamp': time.time()
            }
            
            # Step 3: Skill and Application Construction
            logger.info(f"[{workspace_name}] Step 3: Construction Engine")
            workspace_requirements = self._extract_workspace_requirements(document_results)
            construction_results = self.construction_engine.construct_from_workspace(
                {
                    'tasks': [task.__dict__ for task in self.task_orchestrator.active_tasks.values() 
                             if task.workspace_path == str(workspace_path)],
                    'requirements': workspace_requirements
                },
                str(workspace_path)
            )
            
            # Save construction artifacts
            output_dir = workspace_path / "luuna_output"
            self.construction_engine.save_construction_artifacts(construction_results, output_dir)
            
            pipeline_result.processing_steps['construction'] = {
                'status': 'completed',
                'skills_built': len(construction_results['skills']),
                'applications_built': len(construction_results['applications']),
                'output_directory': str(output_dir),
                'timestamp': time.time()
            }
            
            # Step 4: Generate Final Report
            logger.info(f"[{workspace_name}] Step 4: Final Reporting")
            final_report = self._generate_final_report(pipeline_result, construction_results)
            report_file = workspace_path / "luuna_workspace_report.json"
            
            with open(report_file, 'w', encoding='utf-8') as f:
                json.dump(final_report, f, indent=2, ensure_ascii=False)
                
            pipeline_result.generated_artifacts['final_report'] = str(report_file)
            pipeline_result.processing_steps['reporting'] = {
                'status': 'completed',
                'report_file': str(report_file),
                'timestamp': time.time()
            }
            
            # Mark as completed
            pipeline_result.status = 'completed'
            pipeline_result.timestamp = time.time()
            
            # Store result
            self.processing_results[str(workspace_path)] = pipeline_result
            
            logger.info(f"[{workspace_name}] Pipeline processing completed successfully")
            return pipeline_result
            
        except Exception as e:
            logger.error(f"[{workspace_name}] Pipeline processing failed: {e}")
            error_result = WorkspacePipelineResult(
                workspace_path=str(workspace_path),
                workspace_name=workspace_name,
                status='failed',
                processing_steps={},
                generated_artifacts={},
                timestamp=time.time(),
                error=str(e)
            )
            self.processing_results[str(workspace_path)] = error_result
            return error_result
            
    def _extract_workspace_requirements(self, document_results: Dict) -> Dict:
        """Extract high-level requirements from document analysis"""
        requirements = {
            'source': 'document_analysis',
            'main_purpose': 'Automated Processing System',
            'interfaces': [],
            'data_sources': [],
            'constraints': []
        }
        
        # Analyze document categories and topics
        categories = []
        topics = []
        
        for file_result in document_results.get('processed_files', []):
            analysis = file_result['result'].get('analysis_results', {})
            categories.append(analysis.get('content_category', 'general'))
            topics.extend(analysis.get('key_topics', []))
            
        # Determine main purpose based on dominant category
        if categories:
            dominant_category = max(set(categories), key=categories.count)
            category_purposes = {
                'crypto_finance': 'Cryptocurrency Analysis Platform',
                'technical_docs': 'Technical Implementation System',
                'business_strategy': 'Business Intelligence Dashboard',
                'research_analysis': 'Research Analysis Tool'
            }
            requirements['main_purpose'] = category_purposes.get(dominant_category, 'Automated Processing System')
            
        # Identify potential interfaces
        if any('web' in str(topic).lower() or 'dashboard' in str(topic).lower() for topic in topics):
            requirements['interfaces'].append('web_ui')
        if any('api' in str(topic).lower() or 'integration' in str(topic).lower() for topic in topics):
            requirements['interfaces'].append('api')
            
        # Extract data sources from entities
        for file_result in document_results.get('processed_files', []):
            analysis = file_result['result'].get('analysis_results', {})
            entities = analysis.get('entities', [])
            for entity in entities:
                if entity['type'] == 'exchanges':
                    requirements['data_sources'].append(entity['value'].lower())
                    
        return requirements
        
    def _generate_final_report(self, pipeline_result: WorkspacePipelineResult, construction_results: Dict) -> Dict:
        """Generate comprehensive final report for workspace processing"""
        return {
            'workspace_info': {
                'name': pipeline_result.workspace_name,
                'path': pipeline_result.workspace_path,
                'processing_timestamp': pipeline_result.timestamp
            },
            'pipeline_summary': {
                'status': pipeline_result.status,
                'total_processing_time': time.time() - pipeline_result.timestamp,
                'steps_completed': len([step for step in pipeline_result.processing_steps.values() 
                                      if step.get('status') == 'completed'])
            },
            'document_analysis': pipeline_result.processing_steps.get('document_processing', {}),
            'task_orchestration': pipeline_result.processing_steps.get('task_orchestration', {}),
            'construction_results': {
                'skills_built': len(construction_results.get('skills', [])),
                'applications_built': len(construction_results.get('applications', [])),
                'skill_categories': construction_results.get('construction_summary', {}).get('skill_categories', []),
                'output_directory': construction_results.get('construction_summary', {}).get('output_directory', '')
            },
            'generated_artifacts': pipeline_result.generated_artifacts,
            'recommendations': self._generate_recommendations(pipeline_result, construction_results)
        }
        
    def _generate_recommendations(self, pipeline_result: WorkspacePipelineResult, construction_results: Dict) -> List[str]:
        """Generate recommendations based on processing results"""
        recommendations = []
        
        # Document processing recommendations
        doc_step = pipeline_result.processing_steps.get('document_processing', {})
        if doc_step.get('files_failed', 0) > 0:
            recommendations.append("Some documents failed processing - check file formats and permissions")
            
        # Task orchestration recommendations
        task_step = pipeline_result.processing_steps.get('task_orchestration', {})
        if task_step.get('tasks_assigned', 0) < task_step.get('tasks_generated', 0):
            recommendations.append("Not all tasks were assigned - consider adding more specialized agents")
            
        # Construction recommendations
        skills_count = len(construction_results.get('skills', []))
        apps_count = len(construction_results.get('applications', []))
        
        if skills_count == 0:
            recommendations.append("No skills were generated - document content may need more specific action items")
        elif apps_count == 0:
            recommendations.append("Consider creating a custom application to integrate the generated skills")
            
        if skills_count > 5:
            recommendations.append(f"Large number of skills ({skills_count}) generated - consider modular organization")
            
        return recommendations
        
    def _check_completed_processes(self):
        """Check for and handle completed workspace processes"""
        # This would integrate with the monitor service to check for new workspaces
        # In a full implementation, this would be event-driven
        pass
        
    def _handle_queued_operations(self):
        """Handle any queued operations or maintenance tasks"""
        # Clean up old log files, rotate logs, etc.
        pass
        
    def get_workspace_status(self, workspace_path: str) -> Optional[Dict]:
        """Get current status of a workspace processing"""
        if workspace_path in self.processing_results:
            result = self.processing_results[workspace_path]
            return {
                'workspace_name': result.workspace_name,
                'status': result.status,
                'steps_completed': len([s for s in result.processing_steps.values() 
                                      if s.get('status') == 'completed']),
                'total_steps': len(result.processing_steps),
                'error': result.error
            }
        return None
        
    def get_all_workspaces_status(self) -> Dict[str, Dict]:
        """Get status of all workspaces"""
        return {
            path: self.get_workspace_status(path) 
            for path in self.processing_results.keys()
        }
        
    def shutdown(self):
        """Gracefully shutdown the orchestrator"""
        logger.info("Shutting down Luuna Orchestrator")
        self.running = False
        
        # Stop monitor service
        if hasattr(self, 'monitor_service'):
            self.monitor_service.stop_monitoring()
            
        # Save final state
        self._save_orchestrator_state()
        
        logger.info("Luuna Orchestrator shutdown complete")
        
    def _save_orchestrator_state(self):
        """Save orchestrator state to persistent storage"""
        state_file = self.workspaces_base_path / "orchestrator_state.json"
        state = {
            'processing_results': {
                path: {
                    'workspace_name': result.workspace_name,
                    'status': result.status,
                    'timestamp': result.timestamp,
                    'error': result.error
                }
                for path, result in self.processing_results.items()
            },
            'shutdown_timestamp': time.time()
        }
        
        try:
            with open(state_file, 'w', encoding='utf-8') as f:
                json.dump(state, f, indent=2, ensure_ascii=False)
            logger.info("Orchestrator state saved")
        except Exception as e:
            logger.error(f"Failed to save orchestrator state: {e}")

def create_sample_workspace(base_path: str = "c:\\Luuna\\WORKSPACES") -> Path:
    """Create a sample workspace for testing"""
    base = Path(base_path)
    workspace_name = f"test_workspace_{int(time.time())}"
    workspace_path = base / workspace_name
    workspace_path.mkdir(parents=True, exist_ok=True)
    
    # Create sample documents
    sample_docs = {
        "market_analysis.md": """# Cryptocurrency Market Analysis Requirements

## Objective
Create a system for analyzing cryptocurrency market trends and generating trading signals.

## Key Requirements
- Real-time price data integration
- Technical indicator calculations (RSI, MACD, Bollinger Bands)
- Trend detection algorithms
- Risk assessment capabilities
- Dashboard for visualization

## Data Sources
- CoinGecko API
- CoinMarketCap API
- Binance exchange data

## Expected Outcomes
- Daily market reports
- Trading signal generation
- Portfolio risk analysis
""",
        "technical_spec.md": """# Technical Specifications

## System Architecture
- Microservices architecture
- RESTful API endpoints
- Containerized deployment
- Database: PostgreSQL
- Cache: Redis

## Core Components
1. Data ingestion service
2. Analysis engine
3. Signal generation module
4. Risk assessment system
5. Web dashboard

## Integration Points
- External API connections
- Database connections
- Message queues (RabbitMQ)
- Monitoring and logging
""",
        "business_plan.md": """# Business Strategy Document

## Market Opportunity
The cryptocurrency analytics market is growing rapidly with increasing demand for automated trading tools.

## Target Audience
- Individual traders
- Investment firms
- Hedge funds
- Crypto enthusiasts

## Revenue Model
- Subscription-based pricing
- Tiered feature access
- Premium analytics reports
- Custom integration services

## Competitive Advantage
- Real-time processing
- Advanced ML algorithms
- Comprehensive risk management
- User-friendly interface
"""
    }
    
    # Write sample documents
    for filename, content in sample_docs.items():
        file_path = workspace_path / filename
        with open(file_path, 'w', encoding='utf-8') as f:
            f.write(content)
            
    logger.info(f"Created sample workspace: {workspace_path}")
    return workspace_path

def main():
    """Main entry point"""
    print("🚀 Luuna Autonomous Workspace Orchestrator")
    print("=" * 50)
    
    # Check if running in test mode
    if len(sys.argv) > 1 and sys.argv[1] == "--test":
        print("🔧 Running in test mode...")
        
        # Create test workspace
        test_workspace = create_sample_workspace()
        print(f"📁 Created test workspace: {test_workspace}")
        
        # Process single workspace
        orchestrator = LuunaOrchestrator()
        result = orchestrator._process_workspace_pipeline(test_workspace)
        
        if result and result.status == 'completed':
            print("✅ Test workspace processed successfully!")
            print(f"📊 Skills generated: {len(result.processing_steps.get('construction', {}).get('skills_built', []))}")
            print(f"📱 Applications generated: {len(result.processing_steps.get('construction', {}).get('applications_built', []))}")
        else:
            print("❌ Test workspace processing failed")
            if result and result.error:
                print(f"Error: {result.error}")
                
    else:
        print("🔄 Starting autonomous operation...")
        print("Press Ctrl+C to stop")
        
        # Start full autonomous system
        orchestrator = LuunaOrchestrator()
        orchestrator.start_autonomous_operation()

if __name__ == "__main__":
    main()