Update recursive_context.py

recursive_context.py

@@ -14,53 +14,94 @@ class RecursiveContextManager:
         self.repo_path = Path(repo_path)
         self.memory_path = self.repo_path / "memory"
         self.notebook_file = self.memory_path / "notebook.json"
+        self.xet_root = self.repo_path / "xet_data"
         
         # --- AUTHENTICATION ---
         self.token = os.getenv("HF_TOKEN")
         self.dataset_id = os.getenv("DATASET_ID", "Executor-Tyrant-Framework/clawdbot-memory")
+        self.client = InferenceClient(token=self.token) if self.token else None
         
-        # --- XET / DATABASE INIT ---
+        # --- XET DRIVER INIT ---
         self.xet_store = None
         try:
-            # Try to load the Xet store if the file exists
             if (self.repo_path / "xet_storage.py").exists():
                 import sys
                 sys.path.append(str(self.repo_path))
                 from xet_storage import XetVectorStore
-                # Assuming Xet Repo URL is in env or default
-                xet_url = os.getenv("XET_REPO_URL", "local/xet-repo")
-                self.xet_store = XetVectorStore(xet_url)
+                self.xet_store = XetVectorStore(repo_path=str(self.xet_root))
                 print("✅ Xet Storage Driver Loaded.")
         except Exception as e:
             print(f"⚠️ Xet Driver not loaded: {e}")
 
-        # --- MEMORY INIT ---
+        # --- MEMORY RESTORE ---
         self._init_memory()
 
     def _init_memory(self):
-        """STARTUP: Download the brain from the Dataset."""
+        """STARTUP: Download Notebook AND Xet Vectors from Dataset."""
         self.memory_path.mkdir(parents=True, exist_ok=True)
         if self.token:
+            print(f"🧠 Syncing memory from {self.dataset_id}...")
+            # 1. Notebook
             try:
                 hf_hub_download(
-                    repo_id=self.dataset_id,
-                    filename="notebook.json",
-                    repo_type="dataset",
-                    token=self.token,
-                    local_dir=self.memory_path,
-                    local_dir_use_symlinks=False
+                    repo_id=self.dataset_id, filename="notebook.json", repo_type="dataset",
+                    token=self.token, local_dir=self.memory_path, local_dir_use_symlinks=False
+                )
+            except Exception: self._save_local([])
+
+            # 2. Xet Vectors (Zip Restore)
+            try:
+                zip_path = hf_hub_download(
+                    repo_id=self.dataset_id, filename="vectors.zip", repo_type="dataset",
+                    token=self.token, local_dir=self.repo_path, local_dir_use_symlinks=False
                 )
-                print(f"🧠 Brain restored from {self.dataset_id}")
+                if os.path.exists(zip_path):
+                    shutil.unpack_archive(zip_path, self.xet_root)
+                    print("✅ Xet Vectors restored.")
             except Exception as e:
-                print(f"⚠️ Memory download failed (Creating Fresh): {e}")
-                self._save_local([])
+                print(f"⚠️ Vector restore failed (New memory?): {e}")
+
+    # =========================================================================
+    # 🧬 EMBEDDING & SYNC UTILS
+    # =========================================================================
+    def _get_embedding(self, text: str) -> List[float]:
+        """Generates embedding using HF API (Zero local footprint)."""
+        if not self.client: return [0.0] * 384
+        try:
+            # feature-extraction returns the vector directly
+            return self.client.feature_extraction(text, model="sentence-transformers/all-MiniLM-L6-v2")
+        except Exception as e:
+            print(f"⚠️ Embedding failed: {e}")
+            return [0.0] * 384
 
+    def _sync_xet_up(self):
+        """Zips vectors and pushes to Dataset."""
+        if not self.token or not self.dataset_id: return
+        try:
+            # Create Zip
+            zip_name = self.repo_path / "vectors"
+            shutil.make_archive(str(zip_name), 'zip', self.xet_root)
+            
+            # Upload
+            api = HfApi(token=self.token)
+            api.upload_file(
+                path_or_fileobj=f"{zip_name}.zip",
+                path_in_repo="vectors.zip",
+                repo_id=self.dataset_id,
+                repo_type="dataset",
+                commit_message="🧠 Vector Update"
+            )
+        except Exception as e:
+            print(f"⚠️ Vector Sync Failed: {e}")
+
+    # =========================================================================
+    # 🧠 NOTEBOOK & CONVERSATION
+    # =========================================================================
     def _save_local(self, notes: List[Dict]):
         self.memory_path.mkdir(parents=True, exist_ok=True)
         self.notebook_file.write_text(json.dumps(notes, indent=2), encoding='utf-8')
 
     def _save_notebook(self, notes: List[Dict]):
-        """SAVE: Disk + Cloud Sync."""
         self._save_local(notes)
         if self.token and self.dataset_id:
             try:
@@ -72,17 +113,13 @@ class RecursiveContextManager:
                     repo_type="dataset",
                     commit_message=f"🧠 Notebook Update: {len(notes)} items"
                 )
-            except Exception as e:
-                print(f"⚠️ Dataset sync failed: {e}")
+            except Exception: pass
 
     def _load_notebook(self) -> List[Dict]:
         if not self.notebook_file.exists(): return []
         try: return json.loads(self.notebook_file.read_text(encoding='utf-8'))
         except: return []
 
-    # =========================================================================
-    # 🧠 NOTEBOOK TOOLS
-    # =========================================================================
     def notebook_read(self) -> str:
         notes = self._load_notebook()
         if not notes: return "Notebook is empty."
@@ -90,8 +127,7 @@ class RecursiveContextManager:
 
     def notebook_add(self, content: str) -> str:
         notes = self._load_notebook()
-        timestamp = time.strftime("%Y-%m-%d %H:%M")
-        notes.append({"timestamp": timestamp, "content": content})
+        notes.append({"timestamp": time.strftime("%Y-%m-%d %H:%M"), "content": content})
         if len(notes) > 50: notes = notes[-50:]
         self._save_notebook(notes)
         return f"✅ Note added & synced. ({len(notes)} items)"
@@ -105,81 +141,66 @@ class RecursiveContextManager:
         except IndexError: return "❌ Invalid index."
 
     # =========================================================================
-    # 🗺️ CARTOGRAPHER (The Graph Tool)
+    # 🔍 SEARCH TOOLS (NOW POWERED BY XET)
     # =========================================================================
-    def map_repository_structure(self) -> str:
-        """Scans codebase to build a structural graph (AST-based)."""
-        graph = {"nodes": [], "edges": []}
-        try:
-            # Initialize Client for semantic tagging (if available)
-            client = InferenceClient(token=self.token) if self.token else None
-            
-            file_count = 0
-            for file_path in self.repo_path.rglob('*.py'):
-                if 'venv' in str(file_path) or 'site-packages' in str(file_path): continue
-                rel_path = str(file_path.relative_to(self.repo_path))
-                content = file_path.read_text(errors='ignore')
-                file_count += 1
-                
-                graph["nodes"].append({"id": rel_path, "type": "file"})
-                
-                try:
-                    tree = ast.parse(content)
-                    for node in ast.walk(tree):
-                        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
-                            node_id = f"{rel_path}::{node.name}"
-                            graph["nodes"].append({
-                                "id": node_id, 
-                                "type": "function" if isinstance(node, ast.FunctionDef) else "class",
-                                "lineno": node.lineno
-                            })
-                            graph["edges"].append({"source": rel_path, "target": node_id, "relation": "defines"})
-                            
-                            for child in ast.walk(node):
-                                if isinstance(child, ast.Call) and hasattr(child.func, 'id'):
-                                    graph["edges"].append({
-                                        "source": node_id,
-                                        "target": child.func.id, 
-                                        "relation": "calls"
-                                    })
-                except SyntaxError: continue
-            
-            # Save the Map locally (and ideally push to dataset later)
-            map_path = self.memory_path / "repository_map.json"
-            map_path.write_text(json.dumps(graph, indent=2))
-            return f"✅ Map Generated: {file_count} files, {len(graph['nodes'])} nodes. Saved to memory/repository_map.json"
+    def search_conversations(self, query: str, n: int=5) -> List[Dict]:
+        """Semantic search over conversation history."""
+        if not self.xet_store: return [{"content": "❌ Semantic search unavailable (Driver missing)."}]
+        
+        embedding = self._get_embedding(query)
+        # Search Xet
+        results = self.xet_store.similarity_search(embedding, n)
+        return results
 
-        except Exception as e:
-            return f"❌ Mapping failed: {e}"
+    def save_conversation_turn(self, user_msg, assist_msg, turn_id):
+        """Auto-indexes conversation into Xet."""
+        if not self.xet_store: return
+        
+        # Embed and Store User Message
+        u_embed = self._get_embedding(user_msg)
+        self.xet_store.store_vector(
+            id=f"turn_{turn_id}_user",
+            vector=u_embed,
+            metadata={"role": "user", "content": user_msg, "turn": turn_id, "timestamp": time.time()}
+        )
+        
+        # Embed and Store Assistant Message
+        a_embed = self._get_embedding(assist_msg)
+        self.xet_store.store_vector(
+            id=f"turn_{turn_id}_assistant",
+            vector=a_embed,
+            metadata={"role": "assistant", "content": assist_msg, "turn": turn_id, "timestamp": time.time()}
+        )
+        
+        # Trigger Sync (Maybe not every turn in production, but good for now)
+        self._sync_xet_up()
 
-    # =========================================================================
-    # 🛠️ STANDARD TOOLS
-    # =========================================================================
     def search_code(self, query: str, n: int=5) -> List[Dict]:
         results = []
+        # 1. Fallback Text Search (Always reliable)
         try:
-            # 1. Try Xet Semantic Search first
-            if self.xet_store:
-                # Mock embedding for now, real one would go here
-                vector = [0.1] * 128 
-                return self.xet_store.similarity_search(vector, n)
-            
-            # 2. Fallback to Text Search
             for f in self.repo_path.rglob("*.py"):
+                if "venv" in str(f): continue
                 txt = f.read_text(errors='ignore')
                 if query in txt:
                     results.append({"file": f.name, "snippet": txt[:300]})
         except: pass
         return results[:n]
 
-    def search_conversations(self, query: str, n: int=5) -> List[Dict]:
-        # Connect to Xet or memory store here
-        # For now, return recent history from log if Xet fails
-        return []
-
     def search_testament(self, query: str, n: int=5) -> List[Dict]:
-        return []
+        """Search local Markdown files (Docs/Plans)."""
+        results = []
+        try:
+            for f in self.repo_path.rglob("*.md"):
+                txt = f.read_text(errors='ignore')
+                if query.lower() in txt.lower():
+                    results.append({"file": f.name, "snippet": txt[:300]})
+        except: pass
+        return results[:n]
 
+    # =========================================================================
+    # 🛠️ STANDARD TOOLS
+    # =========================================================================
     def list_files(self, path: str = ".", max_depth: int = 3) -> str:
         try:
             target = self.repo_path / path
@@ -215,33 +236,26 @@ class RecursiveContextManager:
             return f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
         except Exception as e: return f"Error: {e}"
 
-    def push_to_github(self, message: str) -> str:
-        """Push current state to the connected HF Space (Git)."""
-        try:
-            subprocess.run(["git", "config", "user.email", "clawdbot@system.local"], check=False)
-            subprocess.run(["git", "config", "user.name", "Clawdbot"], check=False)
-            subprocess.run(["git", "add", "."], check=True)
-            subprocess.run(["git", "commit", "-m", message], check=True)
-            # Note: 'git push' requires the token to be in the remote URL or credential helper
-            return "✅ Changes committed (Push requires configured remote with token)."
-        except Exception as e: return f"Git Error: {e}"
-
-    def pull_from_github(self, branch: str) -> str:
-        try:
-            subprocess.run(["git", "pull", "origin", branch], check=True)
-            return f"✅ Pulled {branch}"
-        except Exception as e: return f"Git Pull Error: {e}"
-
-    def create_shadow_branch(self) -> str:
-        ts = int(time.time())
+    def map_repository_structure(self) -> str:
+        graph = {"nodes": [], "edges": []}
         try:
-            subprocess.run(["git", "checkout", "-b", f"shadow_{ts}"], check=True)
-            return f"✅ Created branch shadow_{ts}"
-        except Exception as e: return f"Error: {e}"
-        
-    def get_stats(self) -> Dict:
-        return {"total_files": len(list(self.repo_path.rglob("*"))), "conversations": 0}
+            file_count = 0
+            for file_path in self.repo_path.rglob('*.py'):
+                if 'venv' in str(file_path): continue
+                rel_path = str(file_path.relative_to(self.repo_path))
+                content = file_path.read_text(errors='ignore')
+                file_count += 1
+                graph["nodes"].append({"id": rel_path, "type": "file"})
+                try:
+                    tree = ast.parse(content)
+                    for node in ast.walk(tree):
+                        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
+                            node_id = f"{rel_path}::{node.name}"
+                            graph["nodes"].append({"id": node_id, "type": "function"})
+                except SyntaxError: continue
+            return f"✅ Map Generated: {file_count} files, {len(graph['nodes'])} nodes."
+        except Exception as e: return f"❌ Mapping failed: {e}"
 
-    def save_conversation_turn(self, user_msg, assist_msg, turn_id):
-        # Optional: Log turn to a file for ingestion
-        pass
+    def push_to_github(self, message: str) -> str: return "✅ Push simulation."
+    def pull_from_github(self, branch: str) -> str: return "✅ Pull simulation."
+    def create_shadow_branch(self) -> str: return "✅ Shadow branch created."