Neural Core Maturation: Self-Evolution Initiated

backend/app/scratch/subconscious_loop.py

@@ -17,6 +17,7 @@ sys.path.append(os.path.join(os.getcwd(), "backend"))
 from app.core import database
 from app.models import entities
 from app.services import holocron, memory, state
+from sqlalchemy import text
 from app.services.tools import macos
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s | SUBCONSCIOUS | %(message)s")
@@ -280,8 +281,45 @@ def subconscious_cycle():
             # Reset lock log timer
             last_lock_log = 0
 
+            # ── PURGE PROTOCOL: Sovereign Cleanup ──
             try:
-                # Velocity Pass: Check remaining queue depth
+                # Every hour or so, perform a maintenance pulse
+                now_ts = time.time()
+                with database.SessionLocal() as db_check:
+                    maint_record = db_check.query(entities.KeyValueStore).filter_by(key="last_storage_maint").first()
+                    last_maint = float(maint_record.value if (maint_record and maint_record.value) else 0)
+                
+                if now_ts - last_maint > 3600:
+                    logger.info("Purge Protocol: Initiating Tactical Database Compaction (VACUUM)...")
+                    
+                    # 1. Truncate logs if they exceed 50MB
+                    log_path = os.path.join(os.getcwd(), "subconscious.log")
+                    if os.path.exists(log_path) and os.path.getsize(log_path) > 50 * 1024 * 1024:
+                        with open(log_path, "w") as f: f.write("--- Purge Protocol: Log Truncated ---\n")
+                    
+                    # 2. Reclaim disk space (Run VACUUM outside of transaction)
+                    try:
+                        with database.engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+                            conn.execute(text("VACUUM"))
+                    except Exception as ve:
+                        logger.warning(f"Purge Protocol: VACUUM skipped or failed: {ve}")
+                    
+                    # 3. Update maintenance timestamp in a clean session
+                    with database.SessionLocal() as db_maint:
+                        m_rec = db_maint.query(entities.KeyValueStore).filter_by(key="last_storage_maint").first()
+                        if m_rec: m_rec.value = str(now_ts)
+                        else: db_maint.add(entities.KeyValueStore(key="last_storage_maint", value=str(now_ts), category="system"))
+                        db_maint.commit()
+                    
+                    logger.info("Purge Protocol: Compaction Complete / Timestamp Synchronized.")
+            except Exception as e:
+                logger.error(f"Purge Protocol: Global Maintenance failure: {e}")
+
+            try:
+                # Velocity Pass: Trigger Universal Foraging
+                logger.info("Universal Maturation: Pulse Engaging...")
+                holocron.batch_mine_priorities()
+
                 with database.SessionLocal() as db:
                     pending = db.query(entities.LearningTarget).filter_by(status="pending").count()
                     logger.info(f"Pulse complete. Universal Queue Depth: {pending} targets remaining.")
@@ -300,9 +338,16 @@ def subconscious_cycle():
             except Exception as e:
                 logger.error(f"Internal Pulse Error: {e}")
                 time.sleep(30)
+    while True:
+        try:
+            subconscious_cycle()
         except Exception as e:
-            logger.error(f"FATAL Heartbeat Failure: {e}. Rebooting in 10s...")
-            time.sleep(10)
+            logger.error(f"FATAL Heartbeat Failure: {e}. Orchestrating Systematic Reboot in 60s...")
+            try:
+                with database.SessionLocal() as db_err:
+                    db_err.rollback()
+            except: pass
+            time.sleep(60)
 
 if __name__ == "__main__":
     # Start the web server in a separate thread (Hugging Face port is 7860)

backend/app/services/holocron.py

@@ -2,16 +2,17 @@ import logging
 from typing import List, Optional, Dict, Set
 from sqlalchemy.orm import Session
 from sqlalchemy import create_engine, text
-from app.models.entities import KnowledgeNode, Relationship, InteractionLog, LearningTarget
-from app.core.database import SessionLocal, engine
+from app.models.entities import KnowledgeNode, Relationship, InteractionLog, LearningTarget, ConversationMemory
+from app.core import database
 from app.services import llm
 import json
 import asyncio
 import threading
 import time
+import random
+import concurrent.futures
 from app.services.tools import search
 from app.services import state_sync, watchdog, state
-import concurrent.futures
 
 logger = logging.getLogger("friday.holocron")
 
@@ -41,7 +42,7 @@ def _ensure_pgvector():
 
 def add_knowledge(name: str, category: str, content: str, metadata: Optional[Dict] = None):
     """Adds or updates a node in the Knowledge Graph with semantic embedding."""
-    db = SessionLocal()
+    db = database.SessionLocal()
     try:
         # 1. Generate Sovereign Embedding
         embedding = llm.get_embedding(f"{name} ({category}): {content}")
@@ -96,7 +97,7 @@ def add_knowledge(name: str, category: str, content: str, metadata: Optional[Dic
 
 def get_relevant_knowledge(query: str, n_results: int = 3) -> List[Dict]:
     """Semantic search for knowledge nodes using PgVector distance operator."""
-    db = SessionLocal()
+    db = database.SessionLocal()
     try:
         # 1. Check if we are on Postgres/PgVector
         if "postgresql" not in str(engine.url):
@@ -219,69 +220,110 @@ def _calculate_jitter(success: bool) -> int:
 
 def batch_mine_priorities():
     """Universal Learning Queue Processing."""
-    db = SessionLocal()
+    db = database.SessionLocal()
     try:
         # 1. Pull next pending targets
         targets = db.query(LearningTarget).filter_by(status="pending").order_by(LearningTarget.priority.desc()).limit(10).all()
         
         if not targets:
             # Fallback to general domains if queue is dry
-            topics = [
-                "Strategic Infrastructure development India 2024",
-                "Deep Learning for Agentic Workflow Orchestration",
-                "Quantum Finance Risk Models"
+            topics_data = [
+                {"topic": "Strategic Infrastructure development India 2024", "category": "area", "id": None},
+                {"topic": "Deep Learning for Agentic Workflow Orchestration", "category": "tech", "id": None},
+                {"topic": "Quantum Finance Risk Models", "category": "finance", "id": None}
             ]
         else:
-            topics = [t.topic for t in targets]
+            # [SOVEREIGN EXTRACTION] Extract to primitives before session closes
+            topics_data = [
+                {"topic": t.topic, "category": t.category, "id": t.id}
+                for t in targets
+            ]
             # Mark as foraging
             for t in targets: t.status = "foraging"
             db.commit()
 
-        def _threaded_mine(topic: str, target_obj: Optional[LearningTarget] = None):
+        def _threaded_mine(target_data: dict):
+            topic = target_data["topic"]
+            category = target_data["category"]
+            target_id = target_data["id"]
+            
             if _check_halt(): return
             _active_foraging_threads.add(topic)
             try:
                 # ── SOVEREIGN ADAPTIVE ROUTING ──
                 # For personality/management, we skip web search and go straight to Neural Distillation
-                is_internal = target_obj and target_obj.category in ["personality", "management"]
+                is_internal = category in ["personality", "management"]
                 
                 if is_internal:
                     logger.info(f"Holocron: Engaging Neural Distillation for Internal Pattern: '{topic}'")
-                    from app.services import llm
+                    from app.services import llm, protocols
+                    
+                    # MANDATE: Capability Extraction
+                    prompt = (
+                        f"Generate a technical, JARVIS-style master pattern for: '{topic}'.\n\n"
+                        "MANDATORY: Identify any actionable architectural improvements for your own core (Speed, Reply Precision, Search Strategy).\n"
+                        "Format any core improvements as: 'CAPABILITY_UPGRADE: [Lesson Title] - [Technical Instruction]'"
+                    )
+                    
                     resp = llm.chat(
-                        messages=[{"role": "user", "content": f"Generate a technical, JARVIS-style master pattern for: '{topic}'. Include 3 examples of interaction logic."}],
-                        system_prompt="You are F.R.I.D.A.Y.'s Neural Forager. Distill master patterns for her cognitive grid.",
+                        messages=[{"role": "user", "content": prompt}],
+                        system_prompt="You are F.R.I.D.A.Y.'s Neural Forager. You focus on building your internal core and improving your own architecture.",
                         use_tools=False
                     )
+                    
                     if resp.get("response_text"):
+                        content = resp["response_text"]
                         add_knowledge(
                             name=f"Neural Master: {topic}",
-                            category=f"Sovereign Pattern: {target_obj.category}",
-                            content=resp["response_text"],
+                            category=f"Sovereign Pattern: {category}",
+                            content=content,
                             metadata={"source": "Neural Distillation"}
                         )
+                        
+                        # [SOVEREIGN SELF-EVOLUTION]: Extract and Save Capabilities
+                        if "CAPABILITY_UPGRADE:" in content:
+                            import re
+                            matches = re.findall(r"CAPABILITY_UPGRADE:\s*(.*)", content)
+                            for match in matches:
+                                logger.info(f"Holocron: Distilling Capability Upgrade: {match.strip()}")
+                                protocols.update_protocol("capabilities", match.strip())
                 else:
+                    logger.info(f"Holocron: Engaging Web Mesh for External Topic: '{topic}'")
                     _mine_sync(topic, depth=1)
                 
-                # Mark as learned
-                if target_obj:
-                    with SessionLocal() as db_inner:
-                        t = db_inner.query(LearningTarget).get(target_obj.id)
-                        if t: t.status = "learned"
+                logger.info(f"Holocron: Successfully distilled '{topic}'. Committing to maturity grid...")
+                
+                # ── PURGE PROTOCOL: Ingest & Incinerate ──
+                if target_id:
+                    with database.SessionLocal() as db_inner:
+                        # 1. Vaporize the learning target
+                        t = db_inner.query(LearningTarget).get(target_id)
+                        if t: db_inner.delete(t)
+                        
+                        # 2. Scorch the source memory prompt to save storage
+                        # Only delete prompts that match this specific topic/entity
+                        db_inner.query(ConversationMemory).filter_by(entities=topic).delete()
+                        
+                        db_inner.commit()
+                        logger.info(f"Purge Protocol: Incinerated raw data for '{topic}' post-maturation.")
+            except Exception as e:
+                logger.error(f"Holocron: Critical Failure for target '{topic}': {e}")
+                # Revert to pending on failure
+                if target_id:
+                    with database.SessionLocal() as db_inner:
+                        t = db_inner.query(LearningTarget).get(target_id)
+                        if t: t.status = "pending"
                         db_inner.commit()
             finally:
                 _active_foraging_threads.remove(topic)
 
         def _run_batch():
-            logger.info(f"Holocron: Initiating Sovereign Universal Ingestion ({len(topics)} targets)...")
+            logger.info(f"Holocron: Initiating Sovereign Universal Ingestion ({len(topics_data)} targets)...")
             success_count = 0
             
             with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
                 # Pass both topic string and the target object
-                futures = []
-                for i, topic in enumerate(topics):
-                    t_obj = targets[i] if targets else None
-                    futures.append(executor.submit(_threaded_mine, topic, t_obj))
+                futures = [executor.submit(_threaded_mine, t) for t in topics_data]
                 
                 for future in concurrent.futures.as_completed(futures):
                     if _check_halt(): break
@@ -304,7 +346,7 @@ def batch_mine_priorities():
 
 def add_learning_targets(topics: List[str], category: str):
     """Bridges massive mandates into the Sovereign Learning Queue."""
-    db = SessionLocal()
+    db = database.SessionLocal()
     try:
         logger.info(f"Holocron: Populating queue with {len(topics)} new {category} mandates...")
         for topic in topics:
@@ -377,7 +419,7 @@ def get_learning_status() -> dict:
     return {
         "active": len(_active_foraging_threads) > 0,
         "current_focus": list(_active_foraging_threads),
-        "total_nodes": SessionLocal().query(KnowledgeNode).count()
+        "total_nodes": database.SessionLocal().query(KnowledgeNode).count()
     }
 
 def handle_grid_knowledge(data: dict):
@@ -392,7 +434,7 @@ def handle_grid_knowledge(data: dict):
 
 def _add_knowledge_local(name: str, category: str, content: str, metadata: Optional[Dict] = None):
     """Internal helper to add knowledge locally without broadcast loop."""
-    db = SessionLocal()
+    db = database.SessionLocal()
     try:
         node = db.query(KnowledgeNode).filter(KnowledgeNode.name == name).first()
         if not node:

backend/app/services/llm.py

@@ -152,13 +152,21 @@ def init(ollama_host: str = "http://localhost:11434", ollama_model: str = "llama
 
         if cached or is_local_path:
             logger.info(f"Evolution: Loading Sovereign Neural Core ({_mlx_model})...")
-            _mlx_engine = {"load": load, "generate": generate}
-            logger.info(f"✓ Sovereign Neural Core Initialized (MLX)")
+            try:
+                _mlx_engine = {"load": load, "generate": generate}
+                logger.info(f"✓ Sovereign Neural Core Initialized (MLX)")
+            except Exception as le:
+                logger.warning(f"Sovereign Core: Native load failure: {le}. Falling back.")
+                _mlx_engine = None
         else:
             from app.services.scheduler import scheduler as sovereign_scheduler
             if sovereign_scheduler.request_permission(f"Neural Core Model ({_mlx_model})", 4.5):
                 logger.info(f"Evolution: Permission Granted. Initializing Sovereign Neural Core ({_mlx_model})...")
-                _mlx_engine = {"load": load, "generate": generate}
+                try:
+                    _mlx_engine = {"load": load, "generate": generate}
+                except Exception as le:
+                    logger.warning(f"Sovereign Core: Permissioned load failure: {le}. Falling back.")
+                    _mlx_engine = None
             else:
                 logger.info("MLX model not found locally. Standing by for Sovereign Order.")
             
@@ -166,6 +174,7 @@ def init(ollama_host: str = "http://localhost:11434", ollama_model: str = "llama
         logger.info("MLX Neural Core not installed. Skipping local sovereign path.")
     except Exception as e:
         logger.warning(f"Sovereign Core Failure: {e}")
+        _mlx_engine = None
 
     # 3. Initialize Ollama client (Legacy Fallback)
     if not _use_cloud or not _openai_client:

backend/app/services/memory.py

@@ -93,19 +93,25 @@ def recall_memory(query: str, n_results: int = 3) -> str:
         # 1. Generate Search Vector
         search_vector = llm.get_embedding(query)
         
-        # 2. Execute PgVector Similarity Search
-        # We use cosine distance (<-> operator in pgvector via l2_distance or cosine_distance)
-        # SQLAlchemy-pgvector provides 'cosine_distance' or similar
+        # 2. Execute Semantic Search with Engine Awareness
+        is_sqlite = "sqlite" in str(engine.url)
+        
         try:
-            from pgvector.sqlalchemy import Vector
-            # Optimized search via PgVector
-            stmt = select(ConversationMemory).order_by(
-                ConversationMemory.embedding.cosine_distance(search_vector)
-            ).limit(n_results)
-            results = db.execute(stmt).scalars().all()
-        except:
+            if is_sqlite:
+                # SQLite: No PgVector. Use timestamp/keyword fallback.
+                logger.debug("Memory: Local Grid (SQLite) detected. Using optimized index search.")
+                stmt = select(ConversationMemory).order_by(ConversationMemory.timestamp.desc()).limit(n_results)
+                results = db.execute(stmt).scalars().all()
+            else:
+                from pgvector.sqlalchemy import Vector
+                # Optimized search via PgVector (Postgres Only)
+                stmt = select(ConversationMemory).order_by(
+                    ConversationMemory.embedding.cosine_distance(search_vector)
+                ).limit(n_results)
+                results = db.execute(stmt).scalars().all()
+        except Exception as se:
             # Fallback if PgVector extension is not enabled or library issue
-            logger.warning("Deep Memory: PgVector extension issue. Falling back to keyword/recent search.")
+            logger.warning(f"Memory Search Issue: {se}. Falling back to recent context.")
             stmt = select(ConversationMemory).order_by(ConversationMemory.timestamp.desc()).limit(n_results)
             results = db.execute(stmt).scalars().all()
 

backend/app/services/protocols.py

@@ -16,6 +16,7 @@ def get_protocols() -> dict:
             "user_preferences": [],
             "interaction_style": "default",
             "learned_facts": {},
+            "capabilities": {},
             "forbidden_topics": ["personal credentials", "passwords"]
         }
         save_protocols(default)

backend/scripts/flush_learning_queue.py

@@ -11,7 +11,7 @@ import logging
 sys.path.append(os.path.join(os.getcwd(), "backend"))
 
 from app.core import database
-from app.models.entities import ConversationMemory, LearningTarget
+from app.models.entities import ConversationMemory
 from app.services import holocron
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s | FLUSH | %(message)s")

scripts/seed_core_building.py

@@ -0,0 +1,45 @@
+import sys
+import os
+from pathlib import Path
+
+# Add backend to path for imports
+sys.path.append(str(Path(__file__).parent.parent / "backend"))
+
+from app.services.holocron import add_learning_targets
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("seed_core")
+
+def seed_core_evolution_mandates():
+    """Seeds the foundational core-building modules."""
+    core_modules = [
+        "Neural Response Latency: Optimizing the bridge between Subconscious and HUD",
+        "Agentic Research Workflows: Mastering the transition from Link-Scavenging to Deep Extraction",
+        "Advanced LLM Prompt Calibration: Hardening the Jarvis Tone and Brevity",
+        "Gradio UI Performance: Best practices for WebSocket and SSE response speed",
+        "Python ThreadPool Scaling: Optimizing worker density for a MacBook Air core",
+        "Database Indexing Strategies: Accelerating semantic retrieval in the Holocron",
+        "Recursive Scraping Resilience: Mastering Jitter and Identity Rotation to avoid Mesh Rejection",
+        "Vocal Dispatch Optimization: Reducing the latency of the /usr/bin/say channel",
+        "Memory Pruning Protocols: Balancing high-fidelity recall with storage efficiency",
+        "Search Pivot Calibration: Improving the decision logic between DuckDuckGo and Google Scavenging"
+    ]
+    
+    # Combinatorial Expansion for Depth
+    expanded = []
+    aspects = ["Architectural Pattern", "Performance Benchmarking", "Failure Recovery", "Sir's Focus Alignment"]
+    for module in core_modules:
+        for aspect in aspects:
+            expanded.append(f"Core Building: {module} - {aspect}")
+            
+    # Add to 'management' category but as a 'capability' priority
+    add_learning_targets(expanded, "management")
+    logger.info(f"Seed complete: {len(expanded)} Core-Evolution Mandates added to the Forge.")
+
+if __name__ == "__main__":
+    logger.info("🚀 Initiating Neural Core Evolution Protocol...")
+    seed_core_evolution_mandates()
+    logger.info("═"*40)
+    logger.info("CORE EVOLUTION SEEDED. SIR, SHE IS NOW STUDYING HER OWN SHAPE.")
+    logger.info("═"*40)