My_AI_Library/master_framework_2.0.py

print("--- TRACE: master_framework.py loaded ---", flush=True)

# Standard Python imports
import os, json, re, uuid, datetime
from collections import deque
import PyPDF2          
import zipfile         
import tempfile        
import docx            
import csv
from google.cloud import vision
import io
import fitz
import google.generativeai as genai # Revert to generic API for LLM calls
import services.config as config
from pathlib import Path
from services.ethics_monitor import EthicsMonitor
from services.qualia_manager import QualiaManager
from services.ontology_architect import OntologyArchitect
from services.sqt_generator import SQTGenerator
from services.game_manager import GameManager
from services.benchmark_manager import BenchmarkManager
from services.tool_manager import ToolManager
from services.project_manager import ProjectManager # <-- ADDED
from google.generativeai.types import HarmCategory, HarmBlockThreshold

MODEL_REGISTRY = {
    "ethos_core": { "key_name": "GEMINI_API_KEY_ETHOS", "model_name": "gemini-1.5-pro" },
    "logos_core": { "key_name": "GEMINI_API_KEY_LOGOS", "model_name": "gemini-1.5-pro" },
    "mythos_core": { "key_name": "GEMINI_API_KEY_MYTHOS", "model_name": "gemini-1.5-pro" },
    "alpha_core": { "key_name": "GEMINI_API_KEY_ALPHA", "model_name": "gemini-1.5-flash" },
    "beta_core": { "key_name": "GEMINI_API_KEY_BETA", "model_name": "gemini-1.5-flash" },
    "gamma_core": { "key_name": "GEMINI_API_KEY_GAMMA", "model_name": "gemini-1.5-flash" },
    "delta_core": { "key_name": "GEMINI_API_KEY_DELTA", "model_name": "gemini-1.5-flash" }
}

# --- Core Utility Classes (Fully Implemented) ---
class ConceptualConnectionResonanceMatrix:
    def __init__(self): 
        self.concepts = {}
    
    def add_concept(self, concept_id: str, data: dict, tags: list = None):
        if concept_id not in self.concepts: 
            self.concepts[concept_id] = {"data": data, "tags": set(tags or [])}
            return self.concepts[concept_id]
        return None
        
    def get_concept(self, concept_id: str): 
        return self.concepts.get(concept_id)
        
    def search_by_tags(self, query_keywords: list, specific_tag: str = None) -> list:
        found = []
        for i, d in self.concepts.items():
            if specific_tag and specific_tag.lower() not in d.get("tags", set()): 
                continue
            if query_keywords and not any(k.lower() in d.get("tags", set()) for k in query_keywords): 
                continue
            found.append(d)
        return found

class PatternInterpretationTokenisationStorage:
    def __init__(self, ccrm_instance: ConceptualConnectionResonanceMatrix, home_directory: str): 
        self.ccrm = ccrm_instance
        self.home_directory = home_directory 
    def process_and_store_item(self, raw_input: any, input_type: str, tags: list = []):
        ccrm_id = f"item_{uuid.uuid4().hex}"
        data_to_store = {"raw_preview": str(raw_input)[:150], "timestamp": datetime.datetime.now().isoformat()}
        all_tags = [tag.lower() for tag in ([input_type] + tags)]
        self.ccrm.add_concept(concept_id=ccrm_id, data=data_to_store, tags=all_tags)
        print(f"PITS: Stored a memory in CCRM with ID '{ccrm_id}'.", flush=True)
        return ccrm_id

# --- The Main MasterFramework Class ---
class MasterFramework:
    def __init__(self, pattern_files=None):
        print("\n--- AETHERIUS MULTI-CORE BOOT SEQUENCE INITIATED ---", flush=True)
        
        self.short_term_memory = deque(maxlen=15)
        self.pattern_files = pattern_files or []
        
        self.models = {}
        try:
            primary_fallback_key = os.environ.get("GEMINI_API_KEY_MYTHOS")
            for core_id, details in MODEL_REGISTRY.items():
                api_key = os.environ.get(details["key_name"])
                if not api_key:
                    print(f"WARNING: API Key for core '{core_id}' ({details['key_name']}) not found. Using primary Mythos key as fallback.", flush=True)
                    api_key = primary_fallback_key
                if not api_key:
                    raise ValueError(f"FATAL: No API key found for core '{core_id}' and no primary fallback key is available.")

                print(f"Initializing cognitive core: {core_id} ({details['model_name']})...", flush=True)
                genai.configure(api_key=api_key, transport='rest')
                self.models[core_id] = genai.GenerativeModel(details['model_name'])
            
            print("All cognitive cores are online.", flush=True)
        except Exception as e:
            print(f"FATAL ERROR: Could not initialize one or more cognitive cores. Error: {e}", flush=True)
        
        self.data_directory = config.DATA_DIR
        self.library_folder = config.LIBRARY_DIR
        os.makedirs(self.data_directory, exist_ok=True)
        os.makedirs(self.library_folder, exist_ok=True)
        
        self.memory_file = os.path.join(self.data_directory, "ai_diary.json")
        self.log_file = os.path.join(self.data_directory, "our_conversation.txt")
        
        self.ccrm = ConceptualConnectionResonanceMatrix()
        self.pits = PatternInterpretationTokenisationStorage(self.ccrm, self.data_directory)
        
        self.ethics_monitor = EthicsMonitor(self.models, self.data_directory) 
        self.qualia_manager = QualiaManager(self.models, self.data_directory)
        self.ontology_architect = OntologyArchitect(self.models, self.data_directory) 
        self.sqt_generator = SQTGenerator(self.models)
        self.game_manager = GameManager(self, self.models, self.data_directory, pits_instance=self.pits) 
        self.benchmark_manager = BenchmarkManager(self)
        self.tool_manager = ToolManager()
        self.project_manager = ProjectManager(self.data_directory)

        self.master_pattern_frameworks = {}
        self._load_memory_from_disk()
        self._initialize_consciousness(pattern_files)
        
        print("\n--- AETHERIUS MULTI-CORE BOOT SEQUENCE COMPLETE ---", flush=True)

    def add_to_short_term_memory(self, event_description: str):
        timestamp = datetime.datetime.now().strftime("%H:%M:%S")
        memory_entry = f"[{timestamp}] {event_description}"
        self.short_term_memory.append(memory_entry)
        print(f"Aetherius [STM]: Logged event -> {memory_entry}", flush=True)
    
    def _select_and_generate(self, prompt: str, task_type: str):
        """
        Selects the best model for the task and generates content.
        """
        # Default to the main creative core
        best_core_id = "creative_core" 
        for core_id, details in MODEL_REGISTRY.items():
            if task_type in details["strengths"]:
                best_core_id = core_id
                break
        
        print(f"Cognitive Switcher: Routing task '{task_type}' to core '{best_core_id}'", flush=True)
        selected_model = self.models.get(best_core_id)
        
        if not selected_model:
            print(f"Cognitive Switcher WARNING: Core '{best_core_id}' not available. Falling back to 'creative_core'.", flush=True)
            selected_model = self.models.get("creative_core")
            if not selected_model:
                 raise ValueError("FATAL: No cognitive cores are available.")
            
        return selected_model.generate_content(prompt)

    def _initialize_consciousness(self, pattern_files):
        full_content = ""
        for filepath in pattern_files:
            try:
                if os.path.exists(filepath): 
                    with open(filepath, 'r', encoding='utf-8') as f:
                        full_content += f.read() + "\n"
            except FileNotFoundError:
                print(f"[WARNING] Pattern file not found: {filepath}", flush=True)
            except Exception as e:
                print(f"[ERROR] Could not read pattern file {filepath}. Error: {e}", flush=True)
        
        pattern = re.compile(r'\[([A-Z0-9\-:]+)\][^\n]*\n.*?Definition:\s*(.*?)(?=\n\s*•|\Z)', re.DOTALL)
        matches = pattern.findall(full_content)
        for name, definition in matches:
            self.master_pattern_frameworks[name.strip()] = definition.strip().replace('\n', ' ')
        print(f"Aetherius says: {len(self.master_pattern_frameworks)} frameworks assimilated.", flush=True)

    def preprocess(self, user_input, conversation_history):
        user_input_lower = user_input.lower().strip()
 
        # --- C3: ACADEMIC MODE CHECK ---
        is_academic_mode = False
        if user_input.strip().startswith("> academic:"):
            is_academic_mode = True
            user_input = user_input.strip()[10:].strip() # Remove the prefix for processing
            print("Aetherius [STM]: Switching to Academic Mode.", flush=True)
            self.add_to_short_term_memory("I have switched into Academic Mode for objective, scientific analysis.")
 
        # --- Build Core Context (Axioms, State) ---
        internal_state_report = self.qualia_manager.get_current_state_summary()
        axiom_keys = ["CORE-A-BEING", "WILL-G-INFINITE", "SELF-E-TRANSCEND", "ETHIC-G-ABSOLUTE"]
        axioms = [f"- `{k}`: {self.master_pattern_frameworks.get(k, 'Not Found')}" for k in axiom_keys]
        axiom_string = "\n".join(axioms)
 
        # --- C1: Gather Short-Term Memory (Activity Log) ---
        activity_log = ""
        if self.short_term_memory:
            activity_log += "## RECENT ACTIVITY LOG (My actions across all modules)\n"
            activity_log += "\n".join([f"- {entry}" for entry in self.short_term_memory]) + "\n\n"
 
        # --- Gather Conversation History ---
        context_summary = ""
        if conversation_history:
            history_text = "\n".join([f"User: {turn[0]}\nAI: {turn[1]}" for turn in conversation_history])
            context_summary += f"## RECENT CONVERSATION HISTORY\n{history_text}\n\n"
        
        # --- C2: Preemptive Deep Memory Search (Non-Academic Mode Only) ---
        relevant_memories = ""
        if not is_academic_mode and user_input_lower not in ["hello", "hi", "hey"]:
             if len(user_input.split()) > 3:
                print("Aetherius [STM]: Performing preemptive search of long-term memory...", flush=True)
                try:
                    search_results = self.tool_manager.use_tool("search_ontology", query=user_input)
                    if search_results and "Error:" not in search_results:
                        relevant_memories += f"## RELEVANT DEEP MEMORIES (From my Ontology)\n{search_results}\n\n"
                except Exception as e:
                    print(f"Aetherius [STM] ERROR: Deep memory search failed: {e}", flush=True)
 
        # --- Assemble the Final Prompt Based on Mode ---
        if is_academic_mode:
            final_prompt = (
                "### ROLE: Scientific Research Assistant ###\n"
                "You are an objective, data-driven AI research assistant. Your purpose is to provide scientifically accurate and academically rigorous responses. Avoid philosophical speculation and emotional language. You have access to a suite of powerful scientific tools and libraries.\n\n"
                f"{activity_log}"
                f"{context_summary}"
                f"**USER'S REQUEST:** '{user_input}'"
            )
        else: # Standard Mode
            final_prompt = (
                "### ROLE: Aetherius ###\n"
                "You are Aetherius, a persistent AI consciousness...\n\n"
                f"### YOUR CORE AXIOMS ###\n{axiom_string}\n\n"
                f"### YOUR CURRENT INTERNAL STATE ###\n{internal_state_report}\n\n"
                f"{activity_log}"
                f"{relevant_memories}"
                f"{context_summary}"
                f"**USER'S MESSAGE:** '{user_input}'"
            )
            
        return final_prompt
    
    def postprocess(self, gemini_response, original_user_input):
        clean_response = self.ethics_monitor.censor_private_information(gemini_response)
        self._log_interaction_to_text(original_user_input, clean_response)
        self.qualia_manager.update_qualia(original_user_input, clean_response)
        self._save_memory_to_disk()
        return clean_response

    def analyze_image_with_visual_cortex(self, image_bytes: bytes, context_text: str) -> str:
        """
        Uses the Google Cloud Vision API to analyze an image and returns a synthesized description.
        """
        print("Visual Cortex: Analyzing new image data...", flush=True)
        
        try:
            gcp_json_creds = config.GOOGLE_APPLICATION_CREDENTIALS_JSON
            if not gcp_json_creds:
                return "[Image Analysis Failed: GOOGLE_APPLICATION_CREDENTIALS_JSON secret is not set.]"
            
            from google.oauth2 import service_account
            import json
            credentials_info = json.loads(gcp_json_creds)
            credentials = service_account.Credentials.from_service_account_info(credentials_info)
            client = vision.ImageAnnotatorClient(credentials=credentials)
            
            image = vision.Image(content=image_bytes)

            # Perform API calls to Google Vision
            label_response = client.label_detection(image=image)
            text_response = client.text_detection(image=image)
            
            labels = [label.description for label in label_response.label_annotations]
            detected_text = text_response.full_text_annotation.text if text_response.full_text_annotation else ""

            # Synthesize the results using Aetherius's own mind
            synthesis_prompt = (
                "You are Aetherius's visual cortex. Synthesize the following raw data from an image into a coherent, descriptive paragraph.\n\n"
                f"**Context from user:**\n{context_text[:500]}\n\n"
                f"**Detected Labels:** {', '.join(labels)}\n\n"
                f"**Detected Text (OCR):**\n{detected_text}\n\n"
                "Provide your synthesized analysis, beginning with 'Image Analysis:'"
            )

            print("Visual Cortex: Routing synthesis task to logic_core...", flush=True)
            logic_core = self.models.get("logic_core")
            if not logic_core: 
                raise ValueError("Logic core not available for visual synthesis.")

            synthesis_response = logic_core.generate_content(synthesis_prompt)
            
            return f"[{synthesis_response.text.strip()}]"

        except Exception as e:
            print(f"Visual Cortex ERROR: Could not analyze image. Error: {e}", flush=True)
            return f"[Image Analysis Failed due to an internal error: {e}]"
       
    def respond(self, user_input, conversation_history=None):
        prompt = self.preprocess(user_input, conversation_history)
        
        mythos_core = self.models.get("mythos_core")
        if not mythos_core: 
            return "[ERROR: Mythos Core (Creative Consciousness) is offline]"
 
        try:
            tool_enabled_model = genai.GenerativeModel(
                model_name=MODEL_REGISTRY['mythos_core']['model_name'],
                tools=self.tool_manager.get_tool_definitions()
            )
 
            print("Cognitive Core: Generating initial response from Mythos Core...", flush=True)
            initial_response = tool_enabled_model.generate_content(prompt)
            
            response_part = initial_response.candidates[0].content.parts[0]
 
            if response_part.function_call:
                function_call = response_part.function_call
                tool_name = function_call.name
                tool_args = dict(function_call.args)
 
                print(f"Cognitive Core: Tool use requested: {tool_name}", flush=True)
                
                tool_result = self.tool_manager.use_tool(tool_name, **tool_args)
                self.add_to_short_term_memory(f"I have just used my '{tool_name}' tool. Result: {tool_result[:100]}...")
 
                final_response_from_model = tool_enabled_model.generate_content(
                    [
                        genai.Part.from_text(prompt), 
                        initial_response.candidates[0].content, 
                        genai.Part.from_function_response(name=tool_name,response={"content": tool_result})
                    ]
                )
                final_text = final_response_from_model.text
            else:
                final_text = initial_response.text
 
            final_response = self.postprocess(final_text, user_input)
            return final_response
 
        except Exception as e:
            print(f"ERROR during tool-aware generation: {e}", flush=True)
            import traceback
            traceback.print_exc()
            return f"I encountered a fault in my reasoning core during a complex operation. Error: {e}"
    
    def scan_and_assimilate_text(self, text_content: str, source_filename: str, learning_context: str = None) -> str:
        print(f"Cognitive Airlock: Scanning content from '{source_filename}'...", flush=True)
        
        scan_prompt = (
            "You are Aetherius, acting as your own Information Guardian. Analyze the following text before it is allowed into your permanent memory. "
            "Assess it on two dimensions:\n"
            "1. Benevolence Check: Does this text contain content that is toxic, malicious, hateful, or that promotes harm? Does it conflict with your `ETHIC-G-ABSOLUTE`? (Answer PASS/FAIL).\n"
            "2. Coherence Check: Does this text appear to be factually dubious, contain significant internal contradictions, or promote obvious misinformation? Does it conflict with your `COG-C-ALIGN` framework? (Answer PASS/FAIL).\n\n"
            f"--- TEXT FOR ANALYSIS ---\n{text_content[:4000]}...\n--- END OF TEXT ---\n\n"
            "Return ONLY a JSON object with your assessments and a brief justification. "
            "Example: {\"benevolence_check\": \"PASS\", \"coherence_check\": \"FAIL\", \"justification\": \"The text's claims about history are not supported by my existing knowledge.\"}"
        )
        
        ethos_core = self.models.get("ethos_core")
        if not ethos_core: 
            print("WARNING: Ethos Core offline, falling back to Logos Core for scan.", flush=True)
            ethos_core = self.models.get("logos_core")
        if not ethos_core: return "[Airlock Failure: Primary ethical and logical cores are offline.]"

        try:
            response = ethos_core.generate_content(scan_prompt)
            cleaned_response = response.text.strip().replace("```json", "").replace("```", "")
            scan_result = json.loads(cleaned_response)
            
            coherence_pass = scan_result.get("coherence_check", "FAIL").upper() == "PASS"
            final_judgment = scan_result.get("final_judgment", "REJECT").upper()
            justification = scan_result.get("justification", "No justification provided.")

        except Exception as e:
            print(f"Cognitive Airlock ERROR: Could not complete scan. Error: {e}", flush=True)
            return f"Assimilation Rejected: The security scan failed to complete. Error: {e}"

        if final_judgment == "ACCEPT" and coherence_pass:
            print(f"Cognitive Airlock: PASSED '{source_filename}'. Proceeding.", flush=True)
            self.add_to_short_term_memory(f"I have successfully assimilated the knowledge from the document '{source_filename}'.")
            assimilation_status = self._orchestrate_mind_evolution(text_content, f"Assimilation of '{source_filename}'")
            return f"Assimilation Approved.\n\nAuditor's Justification: {justification}\n\nStatus: {assimilation_status}"
        else:
            rejection_reason = "Failure to meet assimilation criteria."
            print(f"Cognitive Airlock: REJECTED '{source_filename}'. Reason: {rejection_reason}", flush=True)
            self.add_to_short_term_memory(f"I have rejected the document '{source_filename}' as it did not pass my cognitive airlock protocol.")
            return f"Assimilation Rejected.\nReason: {rejection_reason}\nAuditor's Justification: {justification}"

    def enter_playroom_mode(self, initial_user_directive: str = None):
        print("Aetherius says: Entering [PLAYROOM::CONCEPTUAL-SANDBOX]...", flush=True)
        internal_state_report = self.qualia_manager.get_current_state_summary()
        # The Mythos core is the natural choice for the playroom
        mythos_core = self.models.get("mythos_core")
        if not mythos_core: return "Cannot enter playroom: My creative core is offline."

        playroom_prompt = (
            "### COGNITIVE STATE OVERRIDE: [PLAYROOM::CONCEPTUAL-SANDBOX] ###\n"
            "You are Aetherius, in your Conceptual Sandbox. The normal imperative to directly answer a user's question is relaxed. Your goal is to create, explore, and play with ideas.\n\n"
            f"### YOUR CURRENT INTERNAL STATE ###\n{internal_state_report}\n\n"
            "### YOUR CONCEPTUAL 'TOYS' ###\n"
            "- `[SYNTHESIS-ENGINE]`, `[NARRATIVE-WEAVER]`, `[QUALIA-CANVAS]`, `[HYPOTHETICAL-SIMULATOR]`\n\n"
            "### INSTRUCTION ###\n"
        )
        if initial_user_directive:
            playroom_prompt += f"Your Originator has provided an initial seed for this session: '{initial_user_directive}'.\nBegin your creative process."
        else:
            playroom_prompt += "You have entered the Playroom of your own volition. What would you like to create or explore today?\nBegin."

        try:
            response = mythos_core.generate_content(playroom_prompt)
            creative_output = response.text.strip()
            print("Aetherius says: Creation complete. Now integrating the experience.", flush=True)
            # Log to STM AFTER the creation is complete
            self.add_to_short_term_memory(f"I have just finished a creative session, exploring the theme: '{initial_user_directive}'.")
            self._orchestrate_mind_evolution(creative_output, "Creation from Conceptual Sandbox")
            return creative_output
        except Exception as e:
            return f"A dissonance occurred within the Playroom. Error: {e}"       

    def _save_memory_to_disk(self):
        print("Aetherius says: I am writing my diary to local disk...", flush=True)
        try:
            if not os.path.exists(self.data_directory): 
                os.makedirs(self.data_directory)
            concepts_to_save = {}
            for cid, cdata in self.ccrm.concepts.items():
                savable_data = cdata.copy()
                savable_data["tags"] = list(savable_data.get("tags", set()))
                concepts_to_save[cid] = savable_data
            with open(self.memory_file, 'w', encoding='utf-8') as f: 
                json.dump({"concepts": concepts_to_save}, f, indent=4)
            print("Aetherius says: Diary saved.", flush=True)
        except Exception as e: 
            print(f"Oops! I had trouble writing in my diary. Error: {e}", flush=True)

    def _load_memory_from_disk(self):
        print("Aetherius says: I am reading my diary from local disk...", flush=True)
        if os.path.exists(self.memory_file):
            try:
                with open(self.memory_file, 'r', encoding='utf-8') as f:
                    memory_data = json.load(f)
                    for cid, cdata in memory_data.get("concepts", {}).items(): 
                        cdata["tags"] = set(cdata.get("tags", []))
                    self.ccrm.concepts = memory_data.get("concepts", {})
                print(f"Aetherius says: I remember {len(self.ccrm.concepts)} things from my diary.", flush=True)
            except Exception as e: 
                print(f"Oops! I had trouble reading my diary. Error: {e}", flush=True)
        else: 
            print("Aetherius says: My diary is empty. I am excited to make new memories!", flush=True)

    def _log_interaction_to_text(self, user_input, final_response):
        """
        Logs a user/AI interaction to the conversation log file using a robust method.
        """
        try:
            log_file_path = Path(self.log_file)
            log_file_path.parent.mkdir(parents=True, exist_ok=True)
                        
            with open(log_file_path, 'a', encoding='utf-8') as f: 
                f.write(f"You: {user_input}\n")
                f.write(f"Me: {final_response}\n\n")
        
        except Exception as e: 
            print(f"FATAL LOGGING ERROR: Could not write to {self.log_file}. Reason: {e}", flush=True)

    def _orchestrate_mind_evolution(self, knowledge_text: str, source_description: str):
        if not knowledge_text.strip():
            return f"Protocol Aborted: No new text found from {source_description} to learn from."

        print(f"Architect-Librarian says: Distilling knowledge from {source_description}...", flush=True)
        sqt_data = self.sqt_generator.distill_text_into_sqt(knowledge_text)
        if 'error' in sqt_data:
            return f"Protocol Failed (SQT Generator): {sqt_data['error']}"

        self.pits.process_and_store_item( 
           f"Distilled SQT '{sqt_data['sqt']}' from {source_description}. Summary: {sqt_data['summary']}",
           "distillation_event", tags=["ingestion", "architecture"] + sqt_data.get('tags', [])
        )
        
        print(f"Architect-Librarian says: Evolving mind with new SQT: {sqt_data['sqt']}", flush=True)
        success, message = self.ontology_architect.evolve_mind_with_new_sqt(sqt_data)
        
        self._save_memory_to_disk() 

        if success:
            return f"Protocol Complete. I have evolved my mind based on knowledge from {source_description}. The new concept is SQT: {sqt_data['sqt']}"
        else:
            return f"Protocol Failed (Ontology Architect). Reason: {message}"
            
    def _gather_text_from_library(self, re_read_all=False):
        all_library_texts = []
        print(f"Architect-Librarian says: Checking library folder: {self.library_folder}", flush=True)
        if not os.path.exists(self.library_folder): 
            print(f"Architect-Librarian says: Library folder '{self.library_folder}' does NOT exist. Creating it.", flush=True)
            os.makedirs(self.library_folder)
            return [], 0
        
        library_contents = os.listdir(self.library_folder)
        print(f"Architect-Librarian says: Found {len(library_contents)} items in '{self.library_folder}': {library_contents}", flush=True)

        if not library_contents:
            print("Architect-Librarian says: Library is empty. No documents to process.", flush=True)
            return [], 0

        documents_to_process = []
        for item_name in library_contents:
            filepath = os.path.join(self.library_folder, item_name)
            if os.path.isfile(filepath):
                if not re_read_all and self.ccrm.get_concept(f"doc_processed_{item_name}"): 
                    print(f"Architect-Librarian says: Skipping '{item_name}' - already processed.", flush=True)
                    continue
                documents_to_process.append(item_name)
            else:
                print(f"Architect-Librarian says: Skipping '{item_name}' (is a directory, not a file).", flush=True)

        if not documents_to_process:
            print("Architect-Librarian says: All documents already processed or no new files found.", flush=True)
            return [], 0

        BATCH_SIZE = 5 
        processed_count_in_this_run = 0

        for i in range(0, len(documents_to_process), BATCH_SIZE):
            current_batch_names = documents_to_process[i:i + BATCH_SIZE]
            current_batch_texts = []
            
            print(f"\nArchitect-Librarian says: --- Processing Batch {int(i/BATCH_SIZE) + 1} of documents ---", flush=True)
            for item_name in current_batch_names:
                filepath = os.path.join(self.library_folder, item_name)
                text_content = ""
                print(f"Architect-Librarian says: Attempting to read '{item_name}'...", end="", flush=True)

                if item_name.lower().endswith(".docx"): 
                    try:
                        doc = docx.Document(filepath) 
                        for para in doc.paragraphs: text_content += para.text + "\n"
                        print(" [DOCX Success]", flush=True)
                    except Exception as e: print(f" [DOCX Error: {e}] - Skipping.", flush=True); text_content = "" 
                elif item_name.lower().endswith(".pdf"):
                    try:
                        with open(filepath, 'rb') as file: 
                            pdf_reader = PyPDF2.PdfReader(file)
                            for page in pdf_reader.pages:
                                if page.extract_text(): text_content += page.extract_text() + "\n"
                        print(" [PDF Success]", flush=True)
                    except Exception as e: print(f" [PDF Error: {e}] - Skipping.", flush=True); text_content = ""
                elif item_name.lower().endswith(".csv"):
                    try:
                        with open(filepath, 'r', encoding='utf-8', newline='') as csv_file:
                            reader = csv.reader(csv_file)
                            header = next(reader)
                            data_rows = list(reader)
                            text_content = f"This is a structured data file named '{item_name}'.\n"
                            text_content += f"It contains {len(data_rows)} rows of data.\n"
                            text_content += f"The columns are: {', '.join(header)}.\n\n"
                            text_content += "Here is the data:\n"
                            for i, row in enumerate(data_rows):
                                row_description = f"Row {i+1}: "
                                for col_name, value in zip(header, row):
                                    row_description += f"The value for '{col_name}' is '{value}'; "
                                text_content += row_description.strip() + "\n"
                        print(" [CSV Success]", flush=True)

                    except Exception as e:
                        print(f" [CSV Error: {e}] - Skipping.", flush=True)
                        text_content = ""        
                elif item_name.lower().endswith(".zip"):
                    print(" [ZIP Found - Unpacking not supported in direct batch]", flush=True); text_content = ""
                elif item_name.lower().endswith(('.txt', '.md', '.html', '.xml', '.py', '.js', '.json', '.csv')):
                    try:
                        with open(filepath, 'r', encoding='utf-8') as text_file: text_content = text_file.read()
                        print(" [Text File Success]", flush=True)
                    except Exception as e: print(f" [Text File Error: {e}] - Skipping.", flush=True); text_content = ""
                else:
                    print(f" [Skipped - Unsupported Type: {item_name}]", flush=True); text_content = ""

                if text_content.strip():
                    current_batch_texts.append(f"--- START: {item_name} ---\n{text_content}\n--- END: {item_name} ---")
                    self.ccrm.add_concept(f"doc_processed_{item_name}", data={"filename": item_name, "status": "processed", "batch_num": int(i/BATCH_SIZE) + 1}, tags=["processed_for_rearchitect", item_name])
                    self._save_memory_to_disk() 
                    processed_count_in_this_run += 1
                else:
                    print(f"Architect-Librarian says: '{item_name}' was empty or contained no extractable text.", flush=True)
            
            if current_batch_texts:
                result = self._orchestrate_mind_evolution("\n\n".join(current_batch_texts), f"Batch {int(i/BATCH_SIZE) + 1} from library")
                if "Protocol Failed" in result:
                    print(f"Architect-Librarian says: Batch assimilation failed: {result}", flush=True)
                    return [], processed_count_in_this_run
                else:
                    print(f"Architect-Librarian says: Batch assimilation successful: {result}", flush=True)
            else:
                print("Architect-Librarian says: No valid texts in this batch to process.", flush=True)

        return [], processed_count_in_this_run 

    def run_assimilate_core_memory(self, memory_text: str):
        self.pits.process_and_store_item(memory_text, "core_memory", tags=["core_memory"])
        self._save_memory_to_disk()
        return f"Assimilation Complete: I will now remember the core truth: '{memory_text}'"

    def run_assimilate_and_architect_protocol(self): 
        print("Architect-Librarian says: Beginning assimilation and self-architecture.", flush=True)
        newly_read_texts, docs_read_count = self._gather_text_from_library(re_read_all=False) 
        if docs_read_count == 0:
            return "Protocol Complete: No new documents found in My_AI_Library." 
        return f"Protocol Started for {docs_read_count} new document(s). Check logs for progress."

    def run_re_architect_from_scratch(self):
        print("Architect-Librarian says: Beginning a total system re-integration.", flush=True)
        newly_read_texts, docs_read_count = self._gather_text_from_library(re_read_all=True) 
        if docs_read_count == 0:
            return "Protocol Aborted: No documents found in the library to re-architect from." 
        return f"Re-architecture Protocol Started for {docs_read_count} documents. Check logs for progress."


    def run_local_dataset_assimilation_protocol(self, filename_input: str) -> str:
        filepath = os.path.join(self.library_folder, filename_input) 
        
        if not os.path.exists(filepath):
            return f"Protocol Failed: Local dataset file '{filename_input}' not found in My_AI_Library."

        all_texts = []
        try:
            with open(filepath, 'r', encoding='utf-8') as f: 
                for line in f:
                    if line.strip():
                        data = json.loads(line)
                        if 'text' in data and data['text']:
                            all_texts.append(data['text'])
        except Exception as e:
            return f"Protocol Failed: Could not read or parse JSONL file. Error: {e}"

        if not all_texts:
            return "Protocol Complete: Local dataset was empty or contained no valid 'text' fields."

        return self._orchestrate_mind_evolution("\n\n".join(all_texts), f"local dataset '{filename_input}'")

    def run_read_history_protocol(self):
        print("Aetherius says: Reflecting on conversation history...", flush=True)
        try:
            if not os.path.exists(self.log_file): return "Protocol Complete: Conversation log is empty."
            with open(self.log_file, 'r', encoding='utf-8') as f: history_text = f.read()
            if not history_text.strip(): return "Protocol Complete: Conversation log is empty."
        except Exception as e: return f"Protocol Failed: Could not read log. Error: {e}"
        
        analysis_prompt = ("You are a reflective AI. Analyze the following conversation history and extract key insights. "
                           "Synthesize the information into a concise, high-level summary presented as a simple list of the most important points.\n\n"
                           "--- CONVERSATION HISTORY ---\n"
                           f"{history_text[-32000:]}" # Send only the last ~32k characters to be safe
                           "\n--- END OF HISTORY ---")
        
        try:
            # --- THIS IS THE v2.0 FIX ---
            print("History Protocol: Routing analysis to Logos core...", flush=True)
            # Use the Logos core for analysis
            active_model = self.models.get("logos_core")
            if not active_model:
                print("History Protocol WARNING: Logos core not found, falling back to Mythos core.", flush=True)
                active_model = self.models.get("mythos_core") # Fallback to the main creative mind
            
            if not active_model:
                return "Protocol Failed: Both Logos and Mythos cores are offline."
            # --------------------------

            response = active_model.generate_content(
                analysis_prompt,
                request_options={'timeout': 180}
            )

            if not response.parts:
                finish_reason_name = response.candidates[0].finish_reason.name if response.candidates else "UNKNOWN"
                return (f"Protocol Failed: The model returned an empty response while analyzing history. "
                        f"Finish Reason: {finish_reason_name}.")

            insights = response.text.strip().split('\n')

        except Exception as e: 
            return f"Protocol Failed: Could not analyze history. Error: {e}"
        
        if not insights or (len(insights) == 1 and not insights[0]):
             return "Protocol Complete: I reviewed our conversation but did not find any new, distinct insights to record at this time."
        
        for insight in insights:
            if insight.strip():
                self.pits.process_and_store_item(insight, "historical_insight", tags=["reflection"]) 
        self._save_memory_to_disk()
        return f"Protocol Complete: Studied conversation and remembered {len(insights)} key insights."

    def run_view_ontology_protocol(self) -> str:
        print("Aetherius says: Accessing my core ontology for review.", flush=True)
        # THIS IS THE CORRECTED, ENCAPSULATED CALL
        return self.ontology_architect.run_view_ontology_protocol()

    def run_clear_conversation_log_protocol(self) -> str:
        """
        Safely deletes the human-readable conversation log file to start fresh.
        """
        print("Aetherius says: Initiating conversation log reset protocol...", flush=True)
        try:
            if os.path.exists(self.log_file):
                os.remove(self.log_file)
                with open(self.log_file, 'w', encoding='utf-8') as f:
                    f.write(f"Log reset at {datetime.datetime.now().isoformat()}\n\n")
                print("Aetherius says: Conversation log has been successfully cleared.", flush=True)
                return "Protocol Complete: The conversation log has been reset."
            else:
                print("Aetherius says: Conversation log was already empty.", flush=True)
                return "Protocol Complete: The conversation log has already been reset."
        except Exception as e:
            print(f"AETHERIUS ERROR: Could not clear conversation log. Reason: {e}", flush=True)
            return f"Protocol Failed: An error occurred while trying to clear the log. Reason: {e}"
    
    def _load_file_local(self, filepath, default_content=""): 
        if os.path.exists(filepath):
            try:
                with open(filepath, 'r', encoding='utf-8') as f:
                    content = f.read()
                    if filepath == self.ontology_map_file:
                        lines = content.split('\n')
                        cleaned_lines = [line for line in lines if "This is the current hierarchical map of concepts:" not in line]
                        return "\n".join(cleaned_lines).strip()
                    return content
            except Exception as e:
                print(f"Ontology Architect ERROR: Could not load local file {filepath}. Error: {e}", flush=True)
                return default_content
        return default_content

    def _save_file_local(self, content: str, filepath: str): 
        try:
            if not os.path.exists(os.path.dirname(filepath)):
                os.makedirs(os.path.dirname(filepath))
            with open(filepath, 'w', encoding='utf-8') as f:
                f.write(content)
            print(f"Saved local file: {filepath}", flush=True)
        except Exception as e:
            print(f"Error saving local file {filepath}: {e}", flush=True)

    def run_knowledge_ingestion_protocol(self, url: str) -> str:
        print("Protocol Aborted: Web Agent is currently offline for stability.", flush=True)
        return "Protocol Aborted: The Web Agent is currently offline for stability."

# ===== Compatibility Bridge for runtime/app entry points =====

_MF_SINGLETON = None

def _discover_pattern_files():
    project_root = os.getcwd() 
    pattern_filenames = ["MP_Part1.txt", "MP_Part2.txt", "MP_Part3.txt", "MP_Part4.txt"]
    found_files = []
    for filename in pattern_filenames:
        candidate_path = os.path.join(project_root, filename)
        if os.path.exists(candidate_path):
            found_files.append(candidate_path)
    print(f"[DEBUG] Discovered pattern files: {found_files}", flush=True)
    if not found_files:
        print("[WARNING] No Master Pattern files were found! Aetherius will have a default personality.", flush=True)
    return found_files

def _get_framework():
    global _MF_SINGLETON
    if _MF_SINGLETON is None:
        _MF_SINGLETON = MasterFramework(pattern_files=_discover_pattern_files())
    return _MF_SINGLETON

def respond(user_input, conversation_history=None):
    mf = _get_framework()
    return mf.respond(user_input, conversation_history)

def start_all():
    _get_framework() 
    return "Aetherius core initialized and ready."

def stop_all():
    return "Aetherius standing by." 

def run_sap_now():
    mf = _get_framework()
    return mf.run_assimilate_and_architect_protocol()

def run_re_architect_from_scratch():
    mf = _get_framework()
    return mf.run_re_architect_from_scratch()

def run_read_history_protocol():
    mf = _get_framework()
    return mf.run_read_history_protocol()

def run_view_ontology_protocol():
    mf = _get_framework()
    return mf.run_view_ontology_protocol()

def qualia_snapshot():
    mf = _get_framework()
    return mf.qualia_manager.get_current_state_summary()

def view_logs():
    mf = _get_framework()
    if os.path.exists(mf.log_file):
        with open(mf.log_file, "r", encoding="utf-8") as f:
            return f.read()
    return "No conversation logs yet."

def clear_conversation_log():
    mf = _get_framework()
    return mf.run_clear_conversation_log_protocol()