diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..4294636d9f223efb0c91fbff6fa418165f94e2bf --- /dev/null +++ b/.env.example @@ -0,0 +1,116 @@ +# MnemoCore Environment Configuration +# ==================================== +# Copy this file to .env and fill in the values. +# All variables can be overridden at runtime. + +# =========================================== +# REQUIRED: API Security +# =========================================== +# API key for authentication (REQUIRED - must be set) +# Generate a secure key: python -c "import secrets; print(secrets.token_urlsafe(32))" +HAIM_API_KEY=your-secure-api-key-here + +# =========================================== +# Redis Configuration +# =========================================== +# Redis connection URL +# Format: redis://[username:password@]host:port/db +REDIS_URL=redis://redis:6379/0 + +# Redis stream key for pub/sub events +REDIS_STREAM_KEY=haim:subconscious + +# Maximum Redis connections +REDIS_MAX_CONNECTIONS=10 + +# Redis socket timeout (seconds) +REDIS_SOCKET_TIMEOUT=5 + +# =========================================== +# Qdrant Configuration +# =========================================== +# Qdrant connection URL +QDRANT_URL=http://qdrant:6333 + +# Collection names +QDRANT_COLLECTION_HOT=haim_hot +QDRANT_COLLECTION_WARM=haim_warm + +# =========================================== +# Server Configuration +# =========================================== +# Host to bind the server +HOST=0.0.0.0 + +# Port to listen on +PORT=8100 + +# Number of uvicorn workers (1 recommended for stateful apps) +WORKERS=1 + +# =========================================== +# Logging Configuration +# =========================================== +# Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL +LOG_LEVEL=INFO + +# Enable structured JSON logging +STRUCTURED_LOGGING=true + +# =========================================== +# Observability (Prometheus) +# =========================================== +# Port for Prometheus metrics +METRICS_PORT=9090 + +# =========================================== +# Memory Tier Configuration +# =========================================== +# Hot tier max memories +HOT_MAX_MEMORIES=2000 + +# Warm tier max memories +WARM_MAX_MEMORIES=100000 + +# LTP decay rate +LTP_DECAY_LAMBDA=0.01 + +# =========================================== +# GPU Configuration (Optional) +# =========================================== +# Enable GPU acceleration +GPU_ENABLED=false + +# CUDA device (e.g., cuda:0) +GPU_DEVICE=cuda:0 + +# =========================================== +# MCP Bridge Configuration (Optional) +# =========================================== +# Enable MCP bridge +MCP_ENABLED=false + +# MCP transport: stdio, tcp +MCP_TRANSPORT=stdio + +# MCP host and port (for TCP transport) +MCP_HOST=127.0.0.1 +MCP_PORT=8110 + +# =========================================== +# CORS Configuration (Optional) +# =========================================== +# Allowed CORS origins (comma-separated) +# CORS_ORIGINS=http://localhost:3000,https://example.com + +# =========================================== +# Rate Limiting (Optional) +# =========================================== +# Enable rate limiting +RATE_LIMIT_ENABLED=true + +# Requests per window +RATE_LIMIT_REQUESTS=100 + +# Window size in seconds +RATE_LIMIT_WINDOW=60 diff --git a/.gitignore b/.gitignore index 150b5fb6380b40b99fcee604070ce3531b6060e9..c801d6fdabcb5a13f0b4c42e061deb538890860e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ ENV/ htmlcov/ .tox/ .nox/ +.hypothesis/ # Data (runtime generated) data/memory.jsonl diff --git a/REFACTORING_TODO.md b/REFACTORING_TODO.md index 10f71141c260de49b8aa9dda81b3127a920c2776..5fd4a6cd6ace0e7d1a4c0e9bbc0fe6cd92b76867 100644 --- a/REFACTORING_TODO.md +++ b/REFACTORING_TODO.md @@ -26,7 +26,7 @@ Status för kodoptimering inför kommande funktionalitet. --- ### 2. Ofullständiga features -**Status:** Pending +**Status:** ✅ Verified / Resolved **Problem:** - Flera TODOs i produktionskod som lämnats oimplementerade @@ -44,8 +44,10 @@ Line 320: # TODO: orchestrate_orch_or() not implemented ``` **Åtgärd:** -- Implementera funktionerna -- Eller ta bort dödkod +- `superposition_query`: Implemented as `_superposition_query` in `HAIMLLMIntegrator`. +- `orchestrate_orch_or`: Implemented in `HAIMEngine`. +- LLM Calls: Code now supports generic providers (OpenAI, Gemini via `google.generativeai`, etc) with safe fallbacks (`_mock_llm_response`) if not configured. +- `_concept_to_memory_id`: Implemented in `MultiAgentHAIM`. --- @@ -165,7 +167,7 @@ Import-stilen följer redan rekommenderad Python-praxis. Ingen åtgärd behövs. ## Förbättra testtäckning ```bash -pytest --cov=src --cov-report=html +pytest --cov=mnemocore --cov-report=html ``` Kör för att identifiera luckor i testtäckningen. @@ -187,13 +189,14 @@ Kör för att identifiera luckor i testtäckningen. ## Framsteg - [x] Punkt 1: HDV-konsolidering ✅ -- [ ] Punkt 2: Ofullständiga features +- [x] Punkt 2: Ofullständiga features ✅ - [ ] Punkt 3: Felhantering - [ ] Punkt 4: Singleton-reduktion 📋 Roadmap - [ ] Punkt 5: Stora funktioner 📋 Roadmap - [x] Punkt 6: Circuit breakers ✅ - [x] Punkt 7: Hårkodade sökvägar ✅ - [x] Punkt 8: Import-stil ✅ (redan konsekvent) +- [x] Test-suite import fixad (src. -> mnemocore.) ✅ --- diff --git a/RELEASE_CHECKLIST.md b/RELEASE_CHECKLIST.md index 4d6ecb74d624623de259692bd6d1f6bcd4d83f62..2b827ae81883233d8e21d0cb6638748f8e967b7e 100644 --- a/RELEASE_CHECKLIST.md +++ b/RELEASE_CHECKLIST.md @@ -1,59 +1,44 @@ -# MnemoCore Public Beta Release Checklist +# MnemoCore Public Beta Release Checklist -## Status: 🟠 ORANGE → 🟢 GREEN +## Status: 🟢 GREEN --- -## ✅ Completed +## ✅ Completed - [x] LICENSE file (MIT) - [x] .gitignore created - [x] data/memory.jsonl removed (no stored memories) - [x] No leaked API keys or credentials -- [x] 82 unit tests passing +- [x] 377 unit tests passing (Coverage increased from 82) +- [x] Test suite import paths fixed (`src.` -> `mnemocore.`) +- [x] Critical TODOs addressed or verified as safe --- -## 🔧 Code TODOs (Known Limitations) +## 🔧 Resolved/Verified Items -These are documented gaps that can ship as "Phase 4 roadmap" items: +The following items were previously listed as known limitations but have been verified as resolved or robustly handled: -### 1. `src/core/tier_manager.py:338` -```python -pass # TODO: Implement full consolidation with Qdrant -``` -**Impact:** Warm→Cold tier consolidation limited -**Workaround:** Hot→Warm works, Cold is filesystem-based -**Fix:** Implement Qdrant batch scroll API for full archival +1. **Qdrant Consolidation:** `src/core/tier_manager.py` implements `consolidate_warm_to_cold` with full Qdrant batch scrolling. +2. **Qdrant Search:** `src/core/engine.py` query pipeline correctly delegates to `TierManager.search` which queries Qdrant for WARM tier results. +3. **LLM Integration:** `src/llm_integration.py` includes `_mock_llm_response` fallbacks when no provider is configured, ensuring stability even without API keys. -### 2. `src/core/engine.py:192` -```python -# TODO: Phase 3.5 Qdrant search for WARM/COLD -``` -**Impact:** Query only searches HOT tier currently -**Workaround:** Promote memories before querying -**Fix:** Add async Qdrant similarity search in query() - -### 3. `src/llm_integration.py:55-57, 128-129` -```python -# TODO: Call Gemini 3 Pro via OpenClaw API -reconstruction = "TODO: Call Gemini 3 Pro" -``` -**Impact:** LLM reconstruction not functional -**Workaround:** Raw vector similarity works -**Fix:** Implement LLM client or make it pluggable +--- -### 4. `src/nightlab/engine.py:339` -```python -# TODO: Notion API integration -``` -**Impact:** Session documentation not auto-pushed -**Workaround:** Written to local markdown files -**Fix:** Add optional Notion connector +## 📝 Remaining Roadmap Items (Non-Blocking) + +### 1. `src/llm_integration.py` - Advanced LLM Features +- **Status:** Functional with generic providers. +- **Task:** Implement specific "OpenClaw" or "Gemini 3 Pro" adapters if required in future. Current implementation supports generic OpenAI/Anthropic/Gemini/Ollama clients. + +### 2. Full Notion Integration +- **Status:** Not currently present in `src/mnemocore`. +- **Task:** Re-introduce `nightlab` or similar module if Notion support is needed in Phase 5. --- -## 📋 Pre-Release Actions +## 📋 Pre-Release Actions ### Before git push: @@ -62,7 +47,8 @@ reconstruction = "TODO: Call Gemini 3 Pro" rm -rf .pytest_cache __pycache__ */__pycache__ *.pyc # 2. Verify tests pass -source .venv/bin/activate && python -m pytest tests/ -v +# Note: Ensure you are in the environment where mnemocore is installed +python -m pytest # 3. Verify import works python -c "from mnemocore.core.engine import HAIMEngine; print('OK')" @@ -72,54 +58,46 @@ grep -r "sk-" src/ --include="*.py" grep -r "api_key.*=" src/ --include="*.py" | grep -v "api_key=\"\"" # 5. Initialize fresh data files +# Ensure data directory exists +mkdir -p data touch data/memory.jsonl data/codebook.json data/concepts.json data/synapses.json ``` ### Update README.md: -- [ ] Add: "Beta Release - See RELEASE_CHECKLIST.md for known limitations" -- [ ] Add: "Installation" section with `pip install -r requirements.txt` -- [ ] Add: "Quick Start" example -- [ ] Add: "Roadmap" section linking TODOs above +- [x] Add: "Beta Release - See RELEASE_CHECKLIST.md for known limitations" +- [x] Add: "Installation" section with `pip install -r requirements.txt` +- [x] Add: "Quick Start" example +- [x] Add: "Roadmap" section linking TODOs above --- -## 🚀 Release Command Sequence +## 🚀 Release Command Sequence ```bash -cd /home/dev-robin/Desktop/mnemocore - # Verify clean state git status -# Stage public files (exclude .venv) -git add LICENSE .gitignore RELEASE_CHECKLIST.md -git add src/ tests/ config.yaml requirements.txt pytest.ini -git add README.md studycase.md docker-compose.yml -git add data/.gitkeep # If exists, or create empty dirs +# Stage public files +git add LICENSE .gitignore RELEASE_CHECKLIST.md REFACTORING_TODO.md +git add src/ tests/ config.yaml requirements.txt pytest.ini pyproject.toml +git add README.md docker-compose.yml +git add data/.gitkeep # If exists # Commit -git commit -m "Initial public beta release (MIT) +git commit -m "Release Candidate: All tests passing, critical TODOs resolved. -Known limitations documented in RELEASE_CHECKLIST.md" +- Fixed test suite import paths (src -> mnemocore) +- Verified Qdrant consolidation and search implementation +- Confirmed LLM integration fallbacks" # Tag -git tag -a v0.1.0-beta -m "Public Beta Release" +git tag -a v0.5.0-beta -m "Public Beta Release" -# Push (when ready) +# Push git push origin main --tags ``` --- -## Post-Release - -- [ ] Create GitHub repository -- [ ] Add repository topics: `vsa`, `holographic-memory`, `active-inference`, `vector-symbolic-architecture` -- [ ] Enable GitHub Issues for community feedback -- [ ] Publish whitepaper/blog post - ---- - -*Generated: 2026-02-15* - +*Updated: 2026-02-18* diff --git a/config.yaml b/config.yaml index e3e14a09e088080e307fca30e5ae7fd803761fe0..127fd11f763a40fe4315e8be78329133ebb0d6fd 100644 --- a/config.yaml +++ b/config.yaml @@ -86,7 +86,7 @@ haim: # MCP (Model Context Protocol) bridge mcp: - enabled: false + enabled: true transport: "stdio" # "stdio" recommended for local MCP clients host: "127.0.0.1" port: 8110 diff --git a/data/mnemocore_hnsw.faiss b/data/mnemocore_hnsw.faiss new file mode 100644 index 0000000000000000000000000000000000000000..2fda6b2070b176cb631f2e718f70cd084d259c22 Binary files /dev/null and b/data/mnemocore_hnsw.faiss differ diff --git a/data/mnemocore_hnsw_idmap.json b/data/mnemocore_hnsw_idmap.json new file mode 100644 index 0000000000000000000000000000000000000000..1e02e4232c4fefa275e77e60e1468a38061d51b1 --- /dev/null +++ b/data/mnemocore_hnsw_idmap.json @@ -0,0 +1 @@ +{"id_map": ["test_node_1", "test_node_2", "test_node_0", "test_node_1", "test_node_2", "test_node_3", "test_node_4", "test_node_5", "test_node_6", "test_node_7", "test_node_8", "test_node_9", "f130a982-5936-4943-ac23-e1f4ad997d25", "19c168e1-c1c0-4074-bdc6-fa6c37bf5009", "b7ad4001-e20c-4f6d-8db7-eca98fbde4ea", "f22c991c-6336-4b11-867a-d141724253ff", "db802f41-0c60-4e62-8595-4864058924b2", "835b80bc-c2b1-4ecf-b0af-ec0442ec87bf", "d594c442-d4b5-4246-9c62-0fb605f235ca", "a467b3f5-d899-4b95-9e62-f7012171ec8d", "fdb78500-49fc-4cf3-a269-b023ba8fa11c", "21d4f6d9-72a6-4dd8-b83e-0d09ef57c680", "19f52711-1ae5-4c29-a0be-2618b162dace", "8442e4e6-cf0f-4f54-b35a-0ce6169fd2f3", "70010aa2-0d82-42bc-96f7-9b99374e24cd", "3cdb7cc9-44ce-4328-9d9a-44dc98d088eb", "1f12c17a-b4ef-4f21-b7f3-3df51196e7f6", "ac29dda1-fd96-4880-9141-01bc46ef93b1", "e0644808-6d24-470d-a667-867fe74d78e2", null, "836f116d-f47f-493b-9ac7-47db84e4a4f9", "fc37dc28-9e3a-484f-a65d-8bf46ee95dbb", "1d3fa01a-0710-453e-8738-4c5be348cd1b", "2776ca2a-429b-491d-9b8a-7061c9b14166", "a9cf8418-d188-41a8-b20d-f2431f790414", "eaf66116-a5ab-436a-a6c3-031d303cb6bd", "474ec49c-1f57-42ac-94e7-3d07d233780f", "a1a19beb-a5f8-4b31-9fe7-0db5b00415c6", "bad8ef94-7313-4bbf-a807-739ff8dcb5c2", "2f4bd760-18b2-4dc1-9dcb-3a6adb77ef23", "0c017bbc-e3ca-4ca0-be83-70b76c0b1be1", "3823dfcd-798d-4e59-a09e-113cc4ec6384", "2682da73-2827-4d68-83b5-1d88f92712ae", "280f0ca4-0930-4452-af29-392d2f99ba7b", "521f09c3-ed6c-4d0f-b04d-97362cb4a94d", "f338c239-dcbd-45ab-ab2b-4192084f5492", "2f011bbd-2a40-4666-a566-4117e8c5ed8e", "dae2e9c6-0c36-4536-b2ff-b59f5443c6fe", "d6028558-d803-454f-8a6f-f82463c44f18", "4c64fe2a-4194-494e-81a2-122911f6cd79", null, "n1", "n2", "n1"], "use_hnsw": false, "stale_count": 2} \ No newline at end of file diff --git a/data/mnemocore_hnsw_vectors.npy b/data/mnemocore_hnsw_vectors.npy new file mode 100644 index 0000000000000000000000000000000000000000..8474ac874ad7026c5f88558b68062ffeaf698229 --- /dev/null +++ b/data/mnemocore_hnsw_vectors.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00400554e796c7717c4def78feb30b5ba5360efed6a37bd35f71738695ac6522 +size 7040 diff --git a/data/subconscious_evolution.json b/data/subconscious_evolution.json index 74d6fe1ed14e43800efaa8289e4f55a45a2f1c16..68621468daf68553d6e035279e9596dd3d282995 100644 --- a/data/subconscious_evolution.json +++ b/data/subconscious_evolution.json @@ -1,6 +1,6 @@ { - "updated_at": "2026-02-18T18:55:55.471022+00:00", - "cycle_count": 56, + "updated_at": "2026-02-21T21:14:22.859465+00:00", + "cycle_count": 168, "insights_generated": 0, "current_cycle_interval": 1, "schedule": { diff --git a/docs/AGI_MEMORY_BLUEPRINT.md b/docs/AGI_MEMORY_BLUEPRINT.md new file mode 100644 index 0000000000000000000000000000000000000000..43ae4efdc69c5aad67bfcc94e28f561ae99e58ee --- /dev/null +++ b/docs/AGI_MEMORY_BLUEPRINT.md @@ -0,0 +1,713 @@ +# MnemoCore AGI Memory Blueprint +### Toward a True Cognitive Memory Substrate for Agentic Systems + +> This document defines the **Phase 5 “AGI Memory” architecture** for MnemoCore – transforming it from a high‑end hyperdimensional memory engine into a **general cognitive substrate** for autonomous AI agents. + +--- + +## 0. Goals & Non‑Goals + +### 0.1 Core Goals + +- Provide a **plug‑and‑play cognitive memory system** that any agent framework can mount as its “mind”: + - Solves **context window** limits by offloading long‑term structure and recall. + - Solves **memory management** by autonomous consolidation, forgetting, and self‑repair. + - Provides **new thoughts, associations and suggestions** rather than only retrieval. +- Implement an explicit, formal model of: + - **Working / Short‑Term Memory (WM/STM)** + - **Episodic Memory** + - **Semantic Memory** + - **Procedural / Skill Memory** + - **Meta‑Memory & Self‑Model** +- Maintain: + - `pip install mnemocore` **zero‑infra dev mode** (SQLite / in‑process vector store). + - Full infra path (Redis, Qdrant, k8s, MCP, OpenClaw live memory integration).[cite:436][cite:437] +- Provide **clean public APIs** (Python + HTTP + MCP) that: + - Give agents a minimal but powerful surface: `observe / recall / reflect / propose_change`. + - Are stable enough to build higher‑level frameworks on (LangGraph, AutoGen, OpenAI Agents, OpenClaw, custom stacks). + +### 0.2 Non‑Goals + +- MnemoCore is **not**: + - En LLM eller policy‑generator. + - En komplett agentram – det är **minnet + kognitiva processer**. +- MnemoCore ska **inte** hårdkoda specifika LLM‑providers. + - LLM används via abstraherad integration (`SubconsciousAI`, `LLMIntegration`) så att byte av motor är trivialt. + +--- + +## 1. Cognitive Architecture Overview + +### 1.1 High‑Level Mental Model + +Systemet ska exponera en internt konsekvent kognitiv modell: + +- **Working Memory (WM)** + - Korttidsbuffert per agent / samtal / uppgift. + - Håller aktuella mål, senaste steg, delresultat. + - Living in RAM, med explicit API. + +- **Episodic Memory (EM)** + - Sekvens av *episodes*: “agent X gjorde Y i kontext Z och fick utfallet U”. + - Tidsstämplad, med länkar mellan episoder (kedjor). + - Riktad mot “vad hände när, i vilken ordning”. + +- **Semantic Memory (SM)** + - Abstraherade, konsoliderade representationer (concepts, prototypes). + - Sammanfattningar av hundratals episoder → en “semantic anchor”. + - Bra för svar på “vad vet jag generellt om X?”. + +- **Procedural Memory (PM)** + - Skills, planer, recept: “för att lösa typ‑X problem, gör följande steg …”. + - Kan hålla både mänsklig läsbar text och exekverbar kod (snippets, tools). + +- **Meta‑Memory (MM)** + - Självmodell för MnemoCore själv: prestanda, reliability, konfiguration, kända svagheter. + - Driver **självförbättringsloopen**. + +Alla dessa lever ovanpå din befintliga HDV/VSA‑kärna, tier manager, synapse index, subconscious loop osv.[cite:436][cite:437] + +### 1.2 New Core Services + +Föreslagna nya Python‑moduler (under `src/mnemocore/core`): + +- `memory_model.py` + - Typed dataklasser för WM/EM/SM/PM/MM entities. +- `working_memory.py` + - WM implementation per agent/task med snabb caching. +- `episodic_store.py` + - Episodisk tidsserie, sekvens‑API. +- `semantic_store.py` + - Wrapper ovanpå befintlig vektorstore (Qdrant/HDV/HNSW) + consolidation hooks. +- `procedural_store.py` + - Lagret för skills, scripts, tool definitions. +- `meta_memory.py` + - Självmodell, logik för self‑improvement proposals. +- `pulse.py` + - “Heartbeat”‑loop: driver subtle thoughts, consolidation ticks, gap detection, self‑reflection. +- `agent_profile.py` + - Persistent profil per agent: preferenser, styrkor/svagheter, quirks. + +--- + +## 2. Data Model + +### 2.1 Working Memory (WM) + +```python +# src/mnemocore/core/memory_model.py + +@dataclass +class WorkingMemoryItem: + id: str + agent_id: str + created_at: datetime + ttl_seconds: int + content: str + kind: Literal["thought", "observation", "goal", "plan_step", "meta"] + importance: float # + tags: list[str] + hdv: BinaryHDV | None + +@dataclass +class WorkingMemoryState: + agent_id: str + items: list[WorkingMemoryItem] + max_items: int +``` + +**Invariantes:** + +- WM är *liten* (t.ex. 32–128 items per agent). +- WM ligger primärt i RAM; kan serialiseras till Redis/SQLite för persistens. +- Access är O(1)/O(log n); LRU + importance‑vägning vid evicering. + +### 2.2 Episodic Memory (EM) + +```python +@dataclass +class EpisodeEvent: + timestamp: datetime + kind: Literal["observation", "action", "thought", "reward", "error"] + content: str + metadata: dict[str, Any] + hdv: BinaryHDV + +@dataclass +class Episode: + id: str + agent_id: str + started_at: datetime + ended_at: datetime | None + goal: str | None + context: str | None # project / environment + events: list[EpisodeEvent] + outcome: Literal["success", "failure", "partial", "unknown"] + reward: float | None + links_prev: list[str] # previous episode IDs + links_next: list[str] # next episode IDs + ltp_strength: float + reliability: float +``` + +### 2.3 Semantic Memory (SM) + +```python +@dataclass +class SemanticConcept: + id: str + label: str # "fastapi-request-validation" + description: str + tags: list[str] + prototype_hdv: BinaryHDV + support_episode_ids: list[str] # episodes som gav upphov + reliability: float + last_updated_at: datetime + metadata: dict[str, Any] +``` + +Kopplas direkt mot consolidation/semantic_consolidation + codebook/immunology.[cite:436][cite:437] + +### 2.4 Procedural Memory (PM) + +```python +@dataclass +class ProcedureStep: + order: int + instruction: str + code_snippet: str | None + tool_call: dict[str, Any] | None + +@dataclass +class Procedure: + id: str + name: str + description: str + created_by_agent: str | None + created_at: datetime + updated_at: datetime + steps: list[ProcedureStep] + trigger_pattern: str # "if user asks about X and Y" + success_count: int + failure_count: int + reliability: float + tags: list[str] +``` + +Procedurer kan genereras av LLM (SubconsciousAI), testas i episodiskt minne, och sedan promotas/demotas med reliability‑loop. + +### 2.5 Meta‑Memory (MM) + +```python +@dataclass +class SelfMetric: + name: str # "hot_tier_hit_rate", "avg_query_latency_ms" + value: float + window: str # "5m", "1h", "24h" + updated_at: datetime + +@dataclass +class SelfImprovementProposal: + id: str + created_at: datetime + author: Literal["system", "agent", "human"] + title: str + description: str + rationale: str + expected_effect: str + status: Literal["pending", "accepted", "rejected", "implemented"] + metadata: dict[str, Any] +``` + +MM lagras delvis i vanlig storage (SM/PM) men har egen API‑yta. + +--- + +## 3. Service Layer Design + +### 3.1 Working Memory Service + +**Fil:** `src/mnemocore/core/working_memory.py` + +Ansvar: + +- Hålla en per‑agent WM‑state. +- Explicita operationer: + - `push_item(agent_id, item: WorkingMemoryItem)` + - `get_state(agent_id) -> WorkingMemoryState` + - `clear(agent_id)` + - `prune(agent_id)` – enligt importance + LRU. +- Integrera med engine/query: + - Vid varje query: WM får en snapshot av top‑K resultat som “context items”. + - Vid svar: agent kan markera vilka items som var relevanta. + +### 3.2 Episodic Store Service + +**Fil:** `src/mnemocore/core/episodic_store.py` + +Ansvar: + +- Skapa och uppdatera Episodes: + - `start_episode(agent_id, goal, context) -> episode_id` + - `append_event(episode_id, kind, content, metadata)` + - `end_episode(episode_id, outcome, reward)` +- Query: + - `get_episode(id)` + - `get_recent(agent_id, limit, context)` + - `find_similar_episodes(hdv, top_k)` +- Koppling till befintlig HDV + tier manager: + - Varje Episode får en “episode_hdv” (bundle över event‑HDVs). + - LTP + reliabilitet följer samma formel som övrig LTP. + +### 3.3 Semantic Store Service + +**Fil:** `src/mnemocore/core/semantic_store.py` + +Ansvar: + +- Hålla SemanticConcepts + codebook. +- API: + - `upsert_concept(concept: SemanticConcept)` + - `find_nearby_concepts(hdv, top_k)` + - `get_concept(id)` +- Hookar mot: + - `semantic_consolidation.py` → abstraktioner / anchors. + - `immunology.py` → attractor cleanup. + - `recursive_synthesizer.py` → djup konceptsyntes. + +### 3.4 Procedural Store Service + +**Fil:** `src/mnemocore/core/procedural_store.py` + +Ansvar: + +- Lagra och hämta Procedures. +- API: + - `store_procedure(proc: Procedure)` + - `get_procedure(id)` + - `find_applicable_procedures(query, agent_id)` + - `record_procedure_outcome(id, success: bool)` +- Integrera med: + - SubconsciousAI → generera nya procedurer från pattern i EM/SM. + - Reliability‑loopen → promota “verified” skills. + +### 3.5 Meta Memory Service + +**Fil:** `src/mnemocore/core/meta_memory.py` + +Ansvar: + +- Hålla SelfMetrics + SelfImprovementProposals. +- API: + - `record_metric(metric: SelfMetric)` + - `list_metrics(filter...)` + - `create_proposal(...)` + - `update_proposal_status(id, status)` +- Integrera med: + - Pulse → skanna metrics och föreslå ändringar. + - LLM → generera förslagstexter (“self‑reflection reports”). + +--- + +## 4. Pulse & Subtle Thoughts + +### 4.1 Pulse Definition + +**Fil:** `src/mnemocore/core/pulse.py` + +“Pulsen” är en central loop (async task, cron, eller k8s CronJob) som: + +- Kör med konfigurerbart intervall (t.ex. var 10:e sekund–var 5:e minut). +- Har ett definierat set “ticks”: + +```python +class PulseTick(Enum): + WM_MAINTENANCE = "wm_maintenance" + EPISODIC_CHAINING = "episodic_chaining" + SEMANTIC_REFRESH = "semantic_refresh" + GAP_DETECTION = "gap_detection" + INSIGHT_GENERATION = "insight_generation" + PROCEDURE_REFINEMENT = "procedure_refinement" + META_SELF_REFLECTION = "meta_self_reflection" +``` + +Pulse orchestrerar: + +- **WM_MAINTENANCE** + - Prune WM per agent. + - Lyfta nyligen viktiga items (“keep in focus”). + +- **EPISODIC_CHAINING** + - Skapa/länka episodiska sekvenser (prev/next). + - “Temporala narrativ”. + +- **SEMANTIC_REFRESH** + - Uppdatera semantic concepts baserat på nya episoder. + - Trigga immunology cleanup för drift. + +- **GAP_DETECTION** + - Kör `GapDetector` över EM/SM sista N minuter/timmar. + - Producera strukturerade knowledge gaps. + +- **INSIGHT_GENERATION** + - Kör SubconsciousAI/LLM över utvalda kluster. + - Skapar nya SemanticConcepts, Procedures, eller MetaProposals. + +- **PROCEDURE_REFINEMENT** + - Uppdatera reliability över PM. + - Flagga outdated/farliga procedures. + +- **META_SELF_REFLECTION** + - Sammanfattar senaste metriker, gap, failures → SelfImprovementProposals. + +### 4.2 Pulse Implementation Sketch + +```python +# src/mnemocore/core/pulse.py + +class Pulse: + def __init__(self, container, config): + self.container = container + self.config = config + self._running = False + + async def start(self): + self._running = True + while self._running: + start = datetime.utcnow() + await self.tick() + elapsed = (datetime.utcnow() - start).total_seconds() + await asyncio.sleep(max(0, self.config.pulse_interval_seconds - elapsed)) + + async def tick(self): + await self._wm_maintenance() + await self._episodic_chaining() + await self._semantic_refresh() + await self._gap_detection() + await self._insight_generation() + await self._procedure_refinement() + await self._meta_self_reflection() +``` + +Konfiguration i `config.yaml`: + +```yaml +haim: + pulse: + enabled: true + interval_seconds: 30 + max_agents_per_tick: 50 + max_episodes_per_tick: 200 +``` + +--- + +## 5. Agent‑Facing APIs (Python & HTTP & MCP) + +### 5.1 High‑Level Python API + +**Fil:** `src/mnemocore/agent_interface.py` + +Syfte: ge agent‑kod ett ENKELT API: + +```python +class CognitiveMemoryClient: + def __init__(self, engine: HAIMEngine, wm, episodic, semantic, procedural, meta): + ... + + # --- Observation & WM --- + def observe(self, agent_id: str, content: str, **meta) -> str: ... + def get_working_context(self, agent_id: str, limit: int = 16) -> list[WorkingMemoryItem]: ... + + # --- Episodic --- + def start_episode(self, agent_id: str, goal: str, context: str | None = None) -> str: ... + def append_event(self, episode_id: str, kind: str, content: str, **meta) -> None: ... + def end_episode(self, episode_id: str, outcome: str, reward: float | None = None) -> None: ... + + # --- Semantic / Retrieval --- + def recall(self, agent_id: str, query: str, context: str | None = None, + top_k: int = 8, modes: tuple[str, ...] = ("episodic","semantic")) -> list[dict]: ... + + # --- Procedural --- + def suggest_procedures(self, agent_id: str, query: str, top_k: int = 5) -> list[Procedure]: ... + def record_procedure_outcome(self, proc_id: str, success: bool) -> None: ... + + # --- Meta / Self-awareness --- + def get_knowledge_gaps(self, agent_id: str, lookback_hours: int = 24) -> list[dict]: ... + def get_self_improvement_proposals(self) -> list[SelfImprovementProposal]: ... +``` + +### 5.2 HTTP Layer Additions + +Utöver befintliga `/store`, `/query`, `/feedback`, osv.[cite:437] + +Nya endpoints: + +- `POST /wm/observe` +- `GET /wm/{agent_id}` +- `POST /episodes/start` +- `POST /episodes/{id}/event` +- `POST /episodes/{id}/end` +- `GET /episodes/{id}` +- `GET /agents/{agent_id}/episodes` +- `GET /agents/{agent_id}/context` +- `GET /agents/{agent_id}/knowledge-gaps` +- `GET /procedures/search` +- `POST /procedures/{id}/feedback` +- `GET /meta/proposals` +- `POST /meta/proposals` + +### 5.3 MCP Tools + +Utöka `mnemocore.mcp.server` med nya verktyg: + +- `store_observation` +- `recall_context` +- `start_episode`, `end_episode` +- `query_memory` +- `get_knowledge_gaps` +- `get_self_improvement_proposals` + +Så att Claude/GPT‑agenter kan: + +- “Titta in” i agentens egen historik. +- Få WM + relevanta episoder + semantic concepts innan svar. +- Få gaps och self‑reflection prompts. + +--- + +## 6. Self‑Improvement Loop + +### 6.1 Loop Definition + +Målet: MnemoCore ska **ständigt förbättra sig**: + +1. Samlar **metrics** (performance + quality). +2. Upptäcker systematiska brister (höga felrates, gap‑clusters). +3. Genererar SelfImprovementProposals via LLM. +4. Låter människa eller meta‑agent granska & appliera. + +### 6.2 Pipeline + +1. **Metrics Collection** + - Utnyttja befintlig `metrics.py` + Prometheus.[cite:436][cite:437] + - Exempelmetriker: + - `query_hit_rate`, `retrieval_latency_ms` + - `feedback_success_rate`, `feedback_failure_rate` + - `hot_tier_size`, `tier_promotion_rate` + - `gap_detection_count`, `gap_fill_count` + +2. **Issue Detection (Rule‑Based)** + - Batchjobb (Pulse) kör enkla regler: + - Om `feedback_failure_rate > X` för en viss tag (t.ex. “fastapi”) → skapa “knowledge area weak” flagg. + - Om `hot_tier_hit_rate < threshold` → dålig context‑masking eller fel tuned thresholds. + +3. **Proposal Generation (LLM)** + - `SubconsciousAI` får inputs: + - Metrics, knowledge gaps, failure cases, config snapshot. + - Prompt genererar: + - `SelfImprovementProposal.title/description/rationale`. + +4. **Review & Execution** + - API / UI för att lista proposals. + - Människa/agent accepterar/rejectar. + - Vid accept: + - Kan trigga config ändringar (med patch PR). + - Kan skapa GitHub issues/PR mallar. + +### 6.3 API + +- `GET /meta/proposals` +- `POST /meta/proposals/{id}/status` + +--- + +## 7. Association & “Subtle Thoughts” + +### 7.1 Association Engine + +Målet: Systemet ska **själv föreslå**: + +- Analogier (“det här liknar när vi gjorde X i annat projekt”). +- Relaterade koncept (“du pratar om Y, men Z har varit viktigt tidigare”). +- Långsiktiga teman och lärdomar. + +Bygg vidare på: + +- `synapse_index.py` (hebbian connections).[cite:436] +- `ripple_context.py` (kaskader).[cite:436] +- `recursive_synthesizer.py` (konceptsyntes).[cite:436] + +Nya pattern: + +- Vid varje Pulse: + - Hämta senaste N episoder. + - Kör k‑NN i semantic concept space. + - Kör ripple over synapses. + - Generera en uppsättning **CandidateAssociations**: + +```python +@dataclass +class CandidateAssociation: + id: str + agent_id: str + created_at: datetime + source_episode_ids: list[str] + related_concept_ids: list[str] + suggestion_text: str + confidence: float +``` + +Lagra i SM/EM så att agent/LLM kan hämta “subtle thoughts” innan svar: + +- `GET /agents/{agent_id}/subtle-thoughts` + +--- + +## 8. Storage Backends & Profiles + +### 8.1 Profiles + +Behåll pip‑enkelheten via profiler: + +- **Lite Profile** (default, no extra deps): + - WM: in‑process dict + - EM: SQLite + - SM: in‑process HDV + mmap + - PM/MM: SQLite/JSON +- **Standard Profile**: + - WARM: Redis + - COLD: filesystem +- **Scale Profile**: + - WARM: Redis + - COLD: Qdrant (eller annan vector DB) + - Optionellt: S3 archive + +Konfigurationsexempel: + +```yaml +haim: + profile: "lite" # "lite" | "standard" | "scale" +``` + +--- + +## 9. OpenClaw & External Agents + +### 9.1 Designprincip för integration + +För OpenClaw / liknande orchestrators: + +- En agent definieras genom: + - `agent_id` + - `capabilities` (tools etc.) +- MnemoCore ska behandla `agent_id` som primär nyckel för: + - WM + - Episoder + - Preferenser + - Procedurer som agenten själv skapat + +### 9.2 “Live Memory” Pattern + +- När OpenClaw kör: + - Varje observation → `observe(agent_id, content, meta)` + - Varje tool call / action → episod event. + - Före varje beslut: + - Hämta: + - `WM` + - `recent episodes` + - `relevant semantic concepts` + - `subtle thoughts` / associations + - `knowledge gaps` (om agenten vill använda dessa som frågor). + +--- + +## 10. Testing & Evaluation Plan + +### 10.1 Unit & Integration Tests + +Nya testfiler: + +- `tests/test_working_memory.py` +- `tests/test_episodic_store.py` +- `tests/test_semantic_store.py` +- `tests/test_procedural_store.py` +- `tests/test_meta_memory.py` +- `tests/test_pulse.py` +- `tests/test_agent_interface.py` + +Fokus: + +- Invariantes (max WM size, LTP thresholds, reliability‑update). +- Episodic chaining korrekt. +- Semantic consolidation integration med nya SM‑API:t. +- Pulse tick ordering & time budget. + +### 10.2 Behavioural Benchmarks + +Skapa `benchmarks/AGI_MEMORY_SCENARIOS.md`: + +- Multi‑session tasks där agent måste: + - Minnas user preferences över dagar. + - Lära sig av failed attempts (feedback). + - Använda analogier över domäner. + +Mät: + +- Context reuse rate. +- Time‑to‑solve vs “no memory” baseline. +- Antal genererade self‑improvement proposals som faktiskt förbättrar outcomes. + +--- + +## 11. Implementation Roadmap + +### Phase 5.0 – Core Structure + +1. Introduce `memory_model.py`, `working_memory.py`, `episodic_store.py`, `semantic_store.py`, `procedural_store.py`, `meta_memory.py`, `pulse.py`. +2. Wire everything in `container.py` (new providers). +3. Add `CognitiveMemoryClient` + minimal tests. + +### Phase 5.1 – WM/EM/SM in Engine + +4. Integrate WM into engine query/store paths. +5. Integrate EM creation in API (store/query/feedback). +6. Adapt semantic_consolidation/immunology to new SM service. + +### Phase 5.2 – Procedural & Association + +7. Implement procedural store + reliability integration. +8. Build association engine + subtle thoughts endpoints. + +### Phase 5.3 – Self‑Improvement + +9. Wire metrics → meta_memory → proposals via SubconsciousAI. +10. Add endpoints & optional small UI for proposals. + +### Phase 5.4 – Hardening & Agents + +11. Harden profiles (lite/standard/scale). +12. Build reference integrations (OpenClaw, LangGraph, AutoGen). + +--- + +## 12. Developer Notes + +- Håll **backwards compatibility** på API där det går: + - Nya endpoints → prefix `v2` om nödvändigt. + - Python API kan vara “ny high‑level layer” ovanpå befintlig `HAIMEngine`. +- All ny funktionalitet **feature‑flaggas i config**: + - `haim.pulse.enabled` + - `haim.episodic.enabled` + - `haim.procedural.enabled` + - etc. +- Strikt logging / metrics för allt nytt: + - `haim_pulse_tick_duration_seconds` + - `haim_wm_size` + - `haim_episode_count` + - `haim_procedure_success_rate` + - `haim_self_proposals_pending` + +--- + +*This blueprint is the contract between MnemoCore, its agents, and its contributors. The intention is to let autonomous AI agents, human developers, and MnemoCore itself co‑evolve toward a truly cognitive memory substrate – one that remembers, forgets, reflects, and grows.* diff --git a/integrations/README.md b/integrations/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bb88b97e270936a750ae9583d600334e19dc2e30 --- /dev/null +++ b/integrations/README.md @@ -0,0 +1,233 @@ +# MnemoCore Integrations + +Connect MnemoCore's persistent cognitive memory to your AI coding tools. + +## Supported tools + +| Tool | Method | Notes | +|------|--------|-------| +| **Claude Code** | MCP server + hooks + CLAUDE.md | Best integration — native tool access | +| **Gemini CLI** | GEMINI.md + wrapper script | Context injected at session start | +| **Aider** | Wrapper script (`--system-prompt`) | Context injected at session start | +| **Any CLI tool** | Universal shell scripts | Pipe context into any tool | +| **Open-source agents** | REST API (`mnemo_bridge.py`) | Minimal Python dependency | + +--- + +## Quick setup + +### Prerequisites + +1. MnemoCore running: `uvicorn mnemocore.api.main:app --port 8100` +2. `HAIM_API_KEY` environment variable set +3. Python 3.10+ with `requests` (`pip install requests`) + +### Linux / macOS + +```bash +cd integrations/ +bash setup.sh --all +``` + +### Windows (PowerShell) + +```powershell +cd integrations\ +.\setup.ps1 -All +``` + +### Manual: test the bridge first + +```bash +export MNEMOCORE_URL=http://localhost:8100 +export HAIM_API_KEY=your-key-here + +python integrations/mnemo_bridge.py health +python integrations/mnemo_bridge.py context --top-k 5 +python integrations/mnemo_bridge.py store "Fixed import error in engine.py" --tags "bugfix,python" +``` + +--- + +## Claude Code (recommended) + +The setup script does three things: + +### 1. MCP server — native tool access + +Registers MnemoCore as an MCP server in `~/.claude/mcp.json`. +Claude Code gets four tools: +- `memory_query` — search memories +- `memory_store` — store a memory +- `memory_get` / `memory_delete` — manage individual memories + +Claude will use these automatically when you instruct it to remember things, +or you can configure CLAUDE.md to trigger them on every session (see below). + +**Verify:** Run `claude mcp list` — you should see `mnemocore` listed. + +### 2. Hooks — automatic background storage + +Two hooks are installed in `~/.claude/settings.json`: + +- **PreToolUse** (`pre_session_inject.py`): On the first tool call of a session, + queries MnemoCore and injects recent context into Claude's awareness. + +- **PostToolUse** (`post_tool_store.py`): After every `Edit`/`Write` call, + stores a lightweight memory entry in the background (non-blocking). + +Hooks never block Claude Code — they degrade silently if MnemoCore is offline. + +### 3. CLAUDE.md — behavioral instructions + +The setup appends memory usage instructions to `CLAUDE.md`. +This tells Claude *when* to use memory tools proactively. + +--- + +## Gemini CLI + +```bash +# Option A: Use wrapper (injects context automatically) +alias gemini='bash integrations/gemini_cli/gemini_wrap.sh' +gemini "Fix the async bug in engine.py" + +# Option B: Manual context injection +CONTEXT=$(integrations/universal/context_inject.sh) +gemini --system-prompt "$CONTEXT" "Fix the async bug in engine.py" +``` + +Also add instructions to your `GEMINI.md`: +```bash +cat integrations/gemini_cli/GEMINI_memory_snippet.md >> GEMINI.md +``` + +--- + +## Aider + +```bash +# Option A: Use wrapper +alias aider='bash integrations/aider/aider_wrap.sh' +aider --model claude-3-5-sonnet-20241022 engine.py + +# Option B: Manual +CONTEXT=$(integrations/universal/context_inject.sh "async engine") +aider --system-prompt "$CONTEXT" engine.py +``` + +--- + +## Universal / Open-source agents + +Any tool that accepts a system prompt can use MnemoCore: + +```bash +# Get context as markdown +integrations/universal/context_inject.sh "query text" 6 + +# Use in any command +MY_CONTEXT=$(integrations/universal/context_inject.sh) +some-ai-cli --system "$MY_CONTEXT" "do the task" + +# Store a memory after a session +integrations/universal/store_session.sh \ + "Discovered that warm tier mmap files grow unbounded without consolidation" \ + "discovery,warm-tier,storage" \ + "mnemocore-project" +``` + +### REST API (Python / any language) + +```python +import os, requests + +BASE = os.getenv("MNEMOCORE_URL", "http://localhost:8100") +KEY = os.getenv("HAIM_API_KEY", "") +HDR = {"X-API-Key": KEY} + +# Query +r = requests.post(f"{BASE}/query", json={"query": "async bugs", "top_k": 5}, headers=HDR) +for m in r.json()["results"]: + print(m["score"], m["content"]) + +# Store +requests.post(f"{BASE}/store", json={ + "content": "Found root cause of memory leak in consolidation worker", + "metadata": {"source": "my-agent", "tags": ["bugfix", "memory"]} +}, headers=HDR) +``` + +--- + +## Environment variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `MNEMOCORE_URL` | `http://localhost:8100` | MnemoCore API base URL | +| `HAIM_API_KEY` | — | API key (same as MnemoCore's `HAIM_API_KEY`) | +| `MNEMOCORE_TIMEOUT` | `5` | Request timeout in seconds | +| `MNEMOCORE_CONTEXT_DIR` | `~/.claude/mnemo_context` | Where hook writes context files | + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ AI Coding Tool │ +│ (Claude Code / Gemini CLI / Aider / Custom) │ +└──────────────┬──────────────────────┬───────────────┘ + │ MCP tools │ System prompt + │ (Claude Code only) │ (all tools) + ▼ ▼ +┌──────────────────────┐ ┌───────────────────────────┐ +│ mnemocore MCP server│ │ mnemo_bridge.py CLI │ +│ (stdio transport) │ │ (lightweight wrapper) │ +└──────────┬───────────┘ └─────────────┬─────────────┘ + │ │ + └────────────┬───────────────┘ + │ HTTP REST + ▼ + ┌────────────────────────┐ + │ MnemoCore API │ + │ localhost:8100 │ + │ │ + │ ┌──────────────────┐ │ + │ │ HAIMEngine │ │ + │ │ HOT/WARM/COLD │ │ + │ │ HDV vectors │ │ + │ └──────────────────┘ │ + └────────────────────────┘ +``` + +--- + +## Troubleshooting + +**MnemoCore offline:** +```bash +python integrations/mnemo_bridge.py health +# → MnemoCore is OFFLINE +# Start it: uvicorn mnemocore.api.main:app --port 8100 +``` + +**API key error (401):** +```bash +export HAIM_API_KEY="your-key-from-.env" +python integrations/mnemo_bridge.py health +``` + +**Hook not triggering (Claude Code):** +```bash +# Check settings.json +cat ~/.claude/settings.json | python -m json.tool | grep -A5 hooks +``` + +**MCP server not found (Claude Code):** +```bash +# Verify mcp.json +cat ~/.claude/mcp.json +# Check PYTHONPATH includes src/ +cd /path/to/mnemocore && python -m mnemocore.mcp.server --help +``` diff --git a/integrations/aider/aider_wrap.sh b/integrations/aider/aider_wrap.sh new file mode 100644 index 0000000000000000000000000000000000000000..1cfeb1704c5d3651b2377e203a71f468aef59d81 --- /dev/null +++ b/integrations/aider/aider_wrap.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# aider_wrap.sh — MnemoCore context injector for Aider +# ====================================================== +# Usage: ./aider_wrap.sh [any aider args...] +# +# Injects MnemoCore memory context into Aider's system prompt +# using the --system-prompt flag (available in Aider 0.40+). +# +# Environment variables: +# MNEMOCORE_URL MnemoCore REST URL (default: http://localhost:8100) +# HAIM_API_KEY API key for MnemoCore +# BRIDGE_PY Path to mnemo_bridge.py (auto-detected) +# AIDER_BIN Path to aider binary (default: aider) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BRIDGE_PY="${BRIDGE_PY:-$(realpath "$SCRIPT_DIR/../mnemo_bridge.py")}" +AIDER_BIN="${AIDER_BIN:-aider}" + +# ── Fetch context ────────────────────────────────────────────────────────── +CONTEXT="" +if python3 "$BRIDGE_PY" health &>/dev/null 2>&1; then + CONTEXT="$(python3 "$BRIDGE_PY" context --top-k 6 2>/dev/null || true)" +fi + +# ── Run Aider with or without injected context ───────────────────────────── +if [[ -n "$CONTEXT" ]]; then + PROMPT_FILE="$(mktemp /tmp/mnemo_aider_XXXXXX.md)" + trap 'rm -f "$PROMPT_FILE"' EXIT + + cat > "$PROMPT_FILE" <<'HEREDOC' +## MnemoCore: Memory from previous sessions + +Use the following context from persistent memory to inform your work. +Do not repeat known decisions. Reference this to avoid re-discovering bugs. + +HEREDOC + echo "$CONTEXT" >> "$PROMPT_FILE" + + exec "$AIDER_BIN" --system-prompt "$PROMPT_FILE" "$@" +else + exec "$AIDER_BIN" "$@" +fi diff --git a/integrations/claude_code/CLAUDE_memory_snippet.md b/integrations/claude_code/CLAUDE_memory_snippet.md new file mode 100644 index 0000000000000000000000000000000000000000..1dc5e74f1822203fe82f45a0e3dde64b5eba9d56 --- /dev/null +++ b/integrations/claude_code/CLAUDE_memory_snippet.md @@ -0,0 +1,38 @@ +# MnemoCore — Persistent Cognitive Memory + +You have access to a persistent memory system via MCP tools: +- `memory_query` — search for relevant memories before starting any task +- `memory_store` — save important decisions, findings, and bug fixes after completing work +- `memory_stats` / `memory_health` — check system status + +## When to use memory + +**At session start:** Call `memory_query` with the user's first message to retrieve relevant past context. + +**After completing a task:** Call `memory_store` to record: +- What was changed and why (key architectural decisions) +- Bug fixes and root causes +- Non-obvious patterns discovered in the codebase +- User preferences and project conventions + +**When you find something unexpected:** Store it immediately with relevant tags. + +## Storing memories + +Include useful metadata: +```json +{ + "content": "Fixed async race condition in tier_manager.py by adding asyncio.Lock around promotion logic", + "metadata": { + "source": "claude-code", + "tags": ["bugfix", "async", "tier_manager"], + "project": "mnemocore" + } +} +``` + +## Rules +- Do NOT store trivial information (e.g., "the user asked me to open a file") +- DO store non-obvious insights, decisions with reasoning, and recurring patterns +- Query memory BEFORE reading files when working on a known codebase +- Store memory AFTER completing non-trivial changes diff --git a/integrations/claude_code/hooks/post_tool_store.py b/integrations/claude_code/hooks/post_tool_store.py new file mode 100644 index 0000000000000000000000000000000000000000..14a5b76c543d73a167803f21e29acb7e4e94a88d --- /dev/null +++ b/integrations/claude_code/hooks/post_tool_store.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +Claude Code PostToolUse hook — MnemoCore auto-store +==================================================== +Automatically stores the result of significant file edits into MnemoCore. + +Configure in ~/.claude/settings.json: +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Edit|Write", + "hooks": [ + { + "type": "command", + "command": "python /path/to/post_tool_store.py" + } + ] + } + ] + } +} + +The hook receives a JSON blob on stdin and exits 0 to allow the tool call. +It stores a lightweight memory entry in the background (non-blocking via +subprocess) so it never delays Claude Code's response. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +BRIDGE = Path(__file__).resolve().parents[2] / "mnemo_bridge.py" +MIN_CONTENT_LEN = 30 # Ignore trivially short edits + + +def main() -> int: + try: + raw = sys.stdin.read() + data = json.loads(raw) if raw.strip() else {} + except json.JSONDecodeError: + return 0 # Never block Claude Code + + tool_name = data.get("tool_name", "") + tool_input = data.get("tool_input", {}) + session_id = data.get("session_id", "") + + # Only act on file-writing tools + if tool_name not in {"Edit", "Write", "MultiEdit"}: + return 0 + + file_path = tool_input.get("file_path", "") + new_string = tool_input.get("new_string") or tool_input.get("content", "") + + if not new_string or len(new_string) < MIN_CONTENT_LEN: + return 0 + + # Build a concise memory entry — just the file + a short excerpt + excerpt = new_string[:200].replace("\n", " ").strip() + memory_text = f"Modified {file_path}: {excerpt}" + + tags = "claude-code,edit" + if file_path.endswith(".py"): + tags += ",python" + elif file_path.endswith((".ts", ".js")): + tags += ",javascript" + + ctx = session_id[:16] if session_id else None + + cmd = [ + sys.executable, str(BRIDGE), + "store", memory_text, + "--source", "claude-code-hook", + "--tags", tags, + ] + if ctx: + cmd += ["--ctx", ctx] + + env = {**os.environ} + + # Fire-and-forget: do not block Claude Code + subprocess.Popen( + cmd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/claude_code/hooks/pre_session_inject.py b/integrations/claude_code/hooks/pre_session_inject.py new file mode 100644 index 0000000000000000000000000000000000000000..561aae2357f356e45d3d4f8e497a6079720a928e --- /dev/null +++ b/integrations/claude_code/hooks/pre_session_inject.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +Claude Code PreToolUse hook — MnemoCore context injection +========================================================== +On the FIRST tool call of a session, queries MnemoCore for recent context +and writes it to a temporary file that is referenced from CLAUDE.md. + +This gives Claude Code automatic memory of previous sessions WITHOUT +requiring any explicit user commands. + +Configure in ~/.claude/settings.json: +{ + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": "python /path/to/pre_session_inject.py" + } + ] + } + ] + } +} + +The hook must exit 0 (allow) or 2 (block with message). +It never blocks — silently degrades if MnemoCore is offline. +""" + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +BRIDGE = Path(__file__).resolve().parents[2] / "mnemo_bridge.py" +CONTEXT_DIR = Path(os.getenv("MNEMOCORE_CONTEXT_DIR", Path.home() / ".claude" / "mnemo_context")) +DONE_FILE = CONTEXT_DIR / ".session_injected" + + +def main() -> int: + try: + raw = sys.stdin.read() + data = json.loads(raw) if raw.strip() else {} + except json.JSONDecodeError: + return 0 + + session_id = data.get("session_id", "") + + # Only inject once per session + done_marker = CONTEXT_DIR / f".injected_{session_id[:16]}" + if done_marker.exists(): + return 0 + + CONTEXT_DIR.mkdir(parents=True, exist_ok=True) + + # Query MnemoCore for context + try: + result = subprocess.run( + [sys.executable, str(BRIDGE), "context", "--top-k", "8"], + capture_output=True, + text=True, + timeout=5, + env={**os.environ}, + ) + context_md = result.stdout.strip() + except (subprocess.TimeoutExpired, FileNotFoundError): + context_md = "" + + if context_md: + context_file = CONTEXT_DIR / "latest_context.md" + context_file.write_text(context_md, encoding="utf-8") + + # Mark session as injected + done_marker.touch() + + # Output context as additional system information if available + if context_md: + # Claude Code hooks can output JSON to inject content + output = { + "type": "system_reminder", + "content": context_md, + } + print(json.dumps(output)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/claude_code/hooks_config_fragment.json b/integrations/claude_code/hooks_config_fragment.json new file mode 100644 index 0000000000000000000000000000000000000000..99c6ef4a8e2051ae49e9e724d579a8871f0b1029 --- /dev/null +++ b/integrations/claude_code/hooks_config_fragment.json @@ -0,0 +1,28 @@ +{ + "_comment": "Merge this fragment into ~/.claude/settings.json under the 'hooks' key.", + "_note": "Replace MNEMOCORE_INTEGRATIONS_PATH with the absolute path to integrations/claude_code/hooks/", + "hooks": { + "PreToolUse": [ + { + "matcher": ".*", + "hooks": [ + { + "type": "command", + "command": "python MNEMOCORE_INTEGRATIONS_PATH/pre_session_inject.py" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "python MNEMOCORE_INTEGRATIONS_PATH/post_tool_store.py" + } + ] + } + ] + } +} diff --git a/integrations/claude_code/mcp_config.json b/integrations/claude_code/mcp_config.json new file mode 100644 index 0000000000000000000000000000000000000000..81a47c046d89a90847c8e479f54ac4291b6c6627 --- /dev/null +++ b/integrations/claude_code/mcp_config.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "mnemocore": { + "command": "python", + "args": ["-m", "mnemocore.mcp.server"], + "cwd": "${MNEMOCORE_DIR}", + "env": { + "HAIM_API_KEY": "${HAIM_API_KEY}", + "PYTHONPATH": "${MNEMOCORE_DIR}/src" + } + } + } +} diff --git a/integrations/gemini_cli/GEMINI_memory_snippet.md b/integrations/gemini_cli/GEMINI_memory_snippet.md new file mode 100644 index 0000000000000000000000000000000000000000..cda9338f7267f9ba15bac768e24405e11e4082ea --- /dev/null +++ b/integrations/gemini_cli/GEMINI_memory_snippet.md @@ -0,0 +1,35 @@ +# MnemoCore — Persistent Cognitive Memory + +You have access to a persistent memory system via the MnemoCore REST API at `$MNEMOCORE_URL` (default: `http://localhost:8100`). + +## Querying memory + +To recall relevant context, call the API at the start of a task: + +```bash +curl -s -X POST "$MNEMOCORE_URL/query" \ + -H "X-API-Key: $HAIM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "DESCRIBE_TASK_HERE", "top_k": 5}' +``` + +## Storing memory + +After completing significant work, store a memory: + +```bash +curl -s -X POST "$MNEMOCORE_URL/store" \ + -H "X-API-Key: $HAIM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "content": "WHAT_WAS_DONE_AND_WHY", + "metadata": {"source": "gemini-cli", "tags": ["relevant", "tags"]} + }' +``` + +## Guidelines + +- **Query before starting** any non-trivial task on a known codebase +- **Store after completing** important changes, bug fixes, or design decisions +- **Do NOT store** trivial or ephemeral information +- Include relevant tags: language, component, type (bugfix/feature/refactor) diff --git a/integrations/gemini_cli/gemini_wrap.sh b/integrations/gemini_cli/gemini_wrap.sh new file mode 100644 index 0000000000000000000000000000000000000000..04a7d270a55b3d96c8e3d85a5cdb5d94fd8d5fad --- /dev/null +++ b/integrations/gemini_cli/gemini_wrap.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# gemini_wrap.sh — MnemoCore context injector for Gemini CLI +# ============================================================= +# Usage: ./gemini_wrap.sh [any gemini CLI args...] +# +# Fetches recent MnemoCore context and prepends it to the system prompt +# via a temporary file, then delegates to the real `gemini` binary. +# +# Environment variables: +# MNEMOCORE_URL MnemoCore REST URL (default: http://localhost:8100) +# HAIM_API_KEY API key for MnemoCore +# BRIDGE_PY Path to mnemo_bridge.py (auto-detected) +# GEMINI_BIN Path to gemini binary (default: gemini) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BRIDGE_PY="${BRIDGE_PY:-$(realpath "$SCRIPT_DIR/../mnemo_bridge.py")}" +GEMINI_BIN="${GEMINI_BIN:-gemini}" +MNEMOCORE_URL="${MNEMOCORE_URL:-http://localhost:8100}" + +# ── Fetch context (silently degrade if offline) ──────────────────────────── +CONTEXT="" +if python3 "$BRIDGE_PY" health &>/dev/null; then + CONTEXT="$(python3 "$BRIDGE_PY" context --top-k 6 2>/dev/null || true)" +fi + +# ── Build the injected system prompt fragment ────────────────────────────── +if [[ -n "$CONTEXT" ]]; then + MEMORY_FILE="$(mktemp /tmp/mnemo_context_XXXXXX.md)" + trap 'rm -f "$MEMORY_FILE"' EXIT + + cat > "$MEMORY_FILE" <<'HEREDOC' +## Persistent Memory Context (from MnemoCore) + +The following is relevant context from your memory of previous sessions. +Use it to avoid re-discovering known patterns, bugs, and decisions. + +HEREDOC + echo "$CONTEXT" >> "$MEMORY_FILE" + + # Gemini CLI supports --system-prompt-file or similar flags. + # Adjust this to match the actual Gemini CLI interface. + exec "$GEMINI_BIN" --system-prompt-file "$MEMORY_FILE" "$@" +else + exec "$GEMINI_BIN" "$@" +fi diff --git a/integrations/mnemo_bridge.py b/integrations/mnemo_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..50bb718dba086d5af8a2749987feb70b6d6200eb --- /dev/null +++ b/integrations/mnemo_bridge.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +MnemoCore Bridge — Universal CLI +================================= +Lightweight bridge between MnemoCore REST API and any AI CLI tool. +No heavy dependencies: only stdlib + requests. + +Usage: + python mnemo_bridge.py context [--query TEXT] [--top-k 5] [--ctx CTX_ID] + python mnemo_bridge.py store TEXT [--source SOURCE] [--tags TAG1,TAG2] [--ctx CTX_ID] + python mnemo_bridge.py health + +Environment variables: + MNEMOCORE_URL Base URL of MnemoCore API (default: http://localhost:8100) + MNEMOCORE_API_KEY API key (same as HAIM_API_KEY) +""" + +import argparse +import json +import os +import sys +from typing import Any, Dict, List, Optional + +try: + import requests +except ImportError: + print("ERROR: 'requests' package required. Run: pip install requests", file=sys.stderr) + sys.exit(1) + +# ── Config ──────────────────────────────────────────────────────────────────── +BASE_URL = os.getenv("MNEMOCORE_URL", "http://localhost:8100").rstrip("/") +API_KEY = os.getenv("MNEMOCORE_API_KEY") or os.getenv("HAIM_API_KEY", "") +TIMEOUT = int(os.getenv("MNEMOCORE_TIMEOUT", "5")) + +HEADERS = {"X-API-Key": API_KEY, "Content-Type": "application/json"} + + +# ── API helpers ─────────────────────────────────────────────────────────────── + +def _get(path: str) -> Optional[Dict]: + try: + r = requests.get(f"{BASE_URL}{path}", headers=HEADERS, timeout=TIMEOUT) + r.raise_for_status() + return r.json() + except requests.ConnectionError: + return None + except requests.HTTPError as e: + print(f"HTTP error: {e}", file=sys.stderr) + return None + + +def _post(path: str, payload: Dict) -> Optional[Dict]: + try: + r = requests.post(f"{BASE_URL}{path}", headers=HEADERS, + json=payload, timeout=TIMEOUT) + r.raise_for_status() + return r.json() + except requests.ConnectionError: + return None + except requests.HTTPError as e: + print(f"HTTP error: {e}", file=sys.stderr) + return None + + +# ── Commands ────────────────────────────────────────────────────────────────── + +def cmd_health() -> int: + data = _get("/health") + if data is None: + print("MnemoCore is OFFLINE (could not connect)", file=sys.stderr) + return 1 + status = data.get("status", "unknown") + print(f"MnemoCore status: {status}") + return 0 if status == "ok" else 1 + + +def cmd_store(text: str, source: str, tags: List[str], ctx: Optional[str]) -> int: + metadata: Dict[str, Any] = {"source": source} + if tags: + metadata["tags"] = tags + + payload: Dict[str, Any] = {"content": text, "metadata": metadata} + if ctx: + payload["agent_id"] = ctx + + data = _post("/store", payload) + if data is None: + print("Failed to store memory (MnemoCore offline or error)", file=sys.stderr) + return 1 + + memory_id = data.get("id") or data.get("memory_id", "?") + print(f"Stored: {memory_id}") + return 0 + + +def cmd_context(query: Optional[str], top_k: int, ctx: Optional[str]) -> int: + """ + Fetch relevant memories and print them as a markdown block + suitable for injection into any AI tool's system prompt. + """ + payload: Dict[str, Any] = { + "query": query or "recent work context decisions bugs fixes", + "top_k": top_k, + } + if ctx: + payload["agent_id"] = ctx + + data = _post("/query", payload) + if data is None: + # Silently return empty — don't break the calling tool's startup + return 0 + + results: List[Dict] = data.get("results", []) + if not results: + return 0 + + lines = [ + "", + "## Relevant memory from previous sessions\n", + ] + for r in results: + content = r.get("content", "").strip() + score = r.get("score", 0.0) + meta = r.get("metadata", {}) + source = meta.get("source", "unknown") + tags = meta.get("tags", []) + tag_str = f" [{', '.join(tags)}]" if tags else "" + + lines.append(f"- **[{source}{tag_str}]** (relevance {score:.2f}): {content}") + + lines.append("\n") + print("\n".join(lines)) + return 0 + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +def main() -> int: + parser = argparse.ArgumentParser( + description="MnemoCore universal CLI bridge", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="cmd", required=True) + + # health + sub.add_parser("health", help="Check MnemoCore connectivity") + + # store + p_store = sub.add_parser("store", help="Store a memory") + p_store.add_argument("text", help="Memory content") + p_store.add_argument("--source", default="cli", help="Source label") + p_store.add_argument("--tags", default="", help="Comma-separated tags") + p_store.add_argument("--ctx", default=None, help="Context/project ID") + + # context + p_ctx = sub.add_parser("context", help="Fetch context as markdown") + p_ctx.add_argument("--query", default=None, help="Semantic query string") + p_ctx.add_argument("--top-k", type=int, default=5, help="Number of results") + p_ctx.add_argument("--ctx", default=None, help="Context/project ID") + + args = parser.parse_args() + + if args.cmd == "health": + return cmd_health() + + if args.cmd == "store": + tags = [t.strip() for t in args.tags.split(",") if t.strip()] + return cmd_store(args.text, args.source, tags, args.ctx) + + if args.cmd == "context": + return cmd_context(args.query, args.top_k, args.ctx) + + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/setup.ps1 b/integrations/setup.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..7060ef91cf94a507dc373873c2d9a36449c98b98 --- /dev/null +++ b/integrations/setup.ps1 @@ -0,0 +1,158 @@ +# MnemoCore Integration Setup — Windows PowerShell +# ================================================= +# One-command setup for Claude Code (MCP) on Windows. +# For full hook/wrapper support, use WSL or Git Bash. +# +# Usage: +# .\setup.ps1 +# .\setup.ps1 -All +# .\setup.ps1 -ClaudeCode + +param( + [switch]$All, + [switch]$ClaudeCode, + [switch]$Gemini, + [switch]$Aider +) + +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$MnemoDir = Split-Path -Parent $ScriptDir +$BridgePy = Join-Path $ScriptDir "mnemo_bridge.py" +$ClaudeHome = Join-Path $env:USERPROFILE ".claude" +$ClaudeMcp = Join-Path $ClaudeHome "mcp.json" +$ClaudeSettings = Join-Path $ClaudeHome "settings.json" +$HooksDir = Join-Path $ScriptDir "claude_code\hooks" + +function Write-Info { Write-Host "[INFO] $args" -ForegroundColor Cyan } +function Write-Success { Write-Host "[OK] $args" -ForegroundColor Green } +function Write-Warn { Write-Host "[WARN] $args" -ForegroundColor Yellow } +function Write-Err { Write-Host "[ERROR] $args" -ForegroundColor Red } + +# ── Prerequisite checks ──────────────────────────────────────────────────── + +Write-Info "Checking Python requests..." +$requestsCheck = python -c "import requests; print('ok')" 2>&1 +if ($requestsCheck -ne "ok") { + Write-Warn "Installing requests..." + python -m pip install --quiet requests +} +Write-Success "Python requests available" + +Write-Info "Checking MnemoCore connectivity..." +$healthCheck = python "$BridgePy" health 2>&1 +if ($LASTEXITCODE -eq 0) { + Write-Success "MnemoCore is online" +} else { + Write-Warn "MnemoCore offline — start it first:" + Write-Warn " cd $MnemoDir" + Write-Warn " uvicorn mnemocore.api.main:app --port 8100" +} + +# ── Claude Code MCP Setup ───────────────────────────────────────────────── + +function Setup-ClaudeCode { + Write-Info "Setting up Claude Code integration..." + + if (-not (Test-Path $ClaudeHome)) { New-Item -ItemType Directory -Path $ClaudeHome | Out-Null } + New-Item -ItemType Directory -Path (Join-Path $ClaudeHome "mnemo_context") -Force | Out-Null + + # MCP config + $McpTemplate = Get-Content (Join-Path $ScriptDir "claude_code\mcp_config.json") -Raw + $McpTemplate = $McpTemplate ` + -replace '\$\{MNEMOCORE_DIR\}', $MnemoDir.Replace('\', '/') ` + -replace '\$\{HAIM_API_KEY\}', ($env:HAIM_API_KEY ?? '') + + if (-not (Test-Path $ClaudeMcp)) { + '{"mcpServers":{}}' | Set-Content $ClaudeMcp + } + + $Existing = Get-Content $ClaudeMcp -Raw | ConvertFrom-Json + $New = $McpTemplate | ConvertFrom-Json + if (-not $Existing.mcpServers) { $Existing | Add-Member -MemberType NoteProperty -Name mcpServers -Value @{} } + $New.mcpServers.PSObject.Properties | ForEach-Object { + $Existing.mcpServers | Add-Member -MemberType NoteProperty -Name $_.Name -Value $_.Value -Force + } + $Existing | ConvertTo-Json -Depth 10 | Set-Content $ClaudeMcp + Write-Success "MCP server registered in $ClaudeMcp" + + # Hooks + if (-not (Test-Path $ClaudeSettings)) { '{}' | Set-Content $ClaudeSettings } + $Settings = Get-Content $ClaudeSettings -Raw | ConvertFrom-Json + + if (-not $Settings.hooks) { + $Settings | Add-Member -MemberType NoteProperty -Name hooks -Value @{} + } + $hooksObj = $Settings.hooks + + $preCmd = "python `"$($HooksDir.Replace('\','/'))/pre_session_inject.py`"" + $postCmd = "python `"$($HooksDir.Replace('\','/'))/post_tool_store.py`"" + + if (-not $hooksObj.PreToolUse) { + $hooksObj | Add-Member -MemberType NoteProperty -Name PreToolUse -Value @() + } + if (-not $hooksObj.PostToolUse) { + $hooksObj | Add-Member -MemberType NoteProperty -Name PostToolUse -Value @() + } + + $existingPre = $hooksObj.PreToolUse | ForEach-Object { $_.hooks[0].command } + if ($preCmd -notin $existingPre) { + $hooksObj.PreToolUse += @{matcher=".*"; hooks=@(@{type="command"; command=$preCmd})} + } + $existingPost = $hooksObj.PostToolUse | ForEach-Object { $_.hooks[0].command } + if ($postCmd -notin $existingPost) { + $hooksObj.PostToolUse += @{matcher="Edit|Write|MultiEdit"; hooks=@(@{type="command"; command=$postCmd})} + } + + $Settings | ConvertTo-Json -Depth 10 | Set-Content $ClaudeSettings + Write-Success "Hooks installed in $ClaudeSettings" + + # CLAUDE.md snippet + $ClaudeMd = Join-Path $MnemoDir "CLAUDE.md" + $Snippet = Get-Content (Join-Path $ScriptDir "claude_code\CLAUDE_memory_snippet.md") -Raw + $Marker = "# MnemoCore — Persistent Cognitive Memory" + if (Test-Path $ClaudeMd) { + $Current = Get-Content $ClaudeMd -Raw + if ($Current -notlike "*$Marker*") { + Add-Content $ClaudeMd "`n$Snippet" + Write-Success "Memory instructions appended to CLAUDE.md" + } else { + Write-Info "CLAUDE.md already contains MnemoCore instructions" + } + } else { + $Snippet | Set-Content $ClaudeMd + Write-Success "Created CLAUDE.md with memory instructions" + } + + Write-Success "Claude Code integration complete" +} + +# ── Main ─────────────────────────────────────────────────────────────────── + +Write-Host "" +Write-Host "╔══════════════════════════════════════════╗" -ForegroundColor Magenta +Write-Host "║ MnemoCore Integration Setup (Win) ║" -ForegroundColor Magenta +Write-Host "╚══════════════════════════════════════════╝" -ForegroundColor Magenta +Write-Host "" + +if (-not ($All -or $ClaudeCode -or $Gemini -or $Aider)) { + Write-Host "Choose integrations:" + Write-Host " 1) Claude Code (MCP + hooks + CLAUDE.md) — recommended" + Write-Host " 4) All" + $choice = Read-Host "Enter choice" + switch ($choice) { + "1" { $ClaudeCode = $true } + "4" { $All = $true } + } +} + +if ($All -or $ClaudeCode) { Setup-ClaudeCode } + +Write-Host "" +Write-Host "╔══════════════════════════════════════════╗" -ForegroundColor Green +Write-Host "║ Setup complete! ║" -ForegroundColor Green +Write-Host "║ ║" -ForegroundColor Green +Write-Host "║ Test: python integrations/mnemo_bridge.py health" -ForegroundColor Green +Write-Host "╚══════════════════════════════════════════╝" -ForegroundColor Green +Write-Host "" diff --git a/integrations/setup.sh b/integrations/setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..1258a30e9b3e8153b6c390612abd3fe0a05037d0 --- /dev/null +++ b/integrations/setup.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +# MnemoCore Integration Setup +# ============================ +# One-command setup for Claude Code, Gemini CLI, Aider, and universal tools. +# +# Usage: +# ./setup.sh # Interactive, choose integrations +# ./setup.sh --all # Enable all integrations +# ./setup.sh --claude-code # Claude Code only +# ./setup.sh --gemini # Gemini CLI only +# ./setup.sh --aider # Aider only +# +# Prerequisites: +# - Python 3.10+ with 'requests' package +# - MnemoCore running (uvicorn mnemocore.api.main:app --port 8100) +# - HAIM_API_KEY environment variable set + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MNEMOCORE_DIR="$(realpath "$SCRIPT_DIR/..")" +BRIDGE_PY="$SCRIPT_DIR/mnemo_bridge.py" +HOOKS_DIR="$SCRIPT_DIR/claude_code/hooks" + +CLAUDE_SETTINGS="$HOME/.claude/settings.json" +CLAUDE_MCP="$HOME/.claude/mcp.json" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# ── Helpers ──────────────────────────────────────────────────────────────── + +info() { echo -e "${BLUE}[INFO]${NC} $*"; } +success() { echo -e "${GREEN}[OK]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } + +check_python() { + if ! python3 -c "import requests" &>/dev/null; then + warn "Python 'requests' not installed. Installing..." + python3 -m pip install --quiet requests + success "requests installed" + fi +} + +check_mnemocore() { + info "Checking MnemoCore connectivity..." + if python3 "$BRIDGE_PY" health &>/dev/null; then + success "MnemoCore is online" + return 0 + else + warn "MnemoCore is not running. Start it first with:" + warn " cd $MNEMOCORE_DIR && uvicorn mnemocore.api.main:app --port 8100" + return 1 + fi +} + +merge_json() { + # Merge JSON object $2 into file $1 (creates if not exists) + local target="$1" + local fragment="$2" + + if [[ ! -f "$target" ]]; then + echo '{}' > "$target" + fi + + python3 - < "$mcp_tmp" + + if [[ ! -f "$CLAUDE_MCP" ]]; then + echo '{"mcpServers": {}}' > "$CLAUDE_MCP" + fi + + python3 - "$CLAUDE_MCP" "$mcp_tmp" <<'PYEOF' +import json, sys +with open(sys.argv[1]) as f: + existing = json.load(f) +with open(sys.argv[2]) as f: + new = json.load(f) +existing.setdefault("mcpServers", {}).update(new.get("mcpServers", {})) +with open(sys.argv[1], "w") as f: + json.dump(existing, f, indent=2) +PYEOF + rm -f "$mcp_tmp" + success " MCP server registered in $CLAUDE_MCP" + + # 2. Hooks + info " Installing hooks in $CLAUDE_SETTINGS..." + if [[ ! -f "$CLAUDE_SETTINGS" ]]; then + echo '{}' > "$CLAUDE_SETTINGS" + fi + + python3 - "$CLAUDE_SETTINGS" "$HOOKS_DIR" <> "$clause_md" + cat "$snippet" >> "$clause_md" + success " Memory instructions appended to $clause_md" + fi + + success "Claude Code integration complete" +} + +# ── Integration: Gemini CLI ──────────────────────────────────────────────── + +setup_gemini() { + info "Setting up Gemini CLI integration..." + + # Make wrapper executable + chmod +x "$SCRIPT_DIR/gemini_cli/gemini_wrap.sh" + + # Append to GEMINI.md if it exists + local gemini_md="$MNEMOCORE_DIR/GEMINI.md" + local snippet="$SCRIPT_DIR/gemini_cli/GEMINI_memory_snippet.md" + local marker="# MnemoCore — Persistent Cognitive Memory" + if [[ -f "$gemini_md" ]] && grep -qF "$marker" "$gemini_md"; then + info " GEMINI.md already contains MnemoCore instructions" + elif [[ -f "$gemini_md" ]]; then + echo "" >> "$gemini_md" + cat "$snippet" >> "$gemini_md" + success " Memory instructions appended to $gemini_md" + else + cp "$snippet" "$gemini_md" + success " Created $gemini_md with memory instructions" + fi + + success "Gemini CLI integration complete" + info " Use: $SCRIPT_DIR/gemini_cli/gemini_wrap.sh [args] instead of 'gemini'" + info " Or alias: alias gemini='$SCRIPT_DIR/gemini_cli/gemini_wrap.sh'" +} + +# ── Integration: Aider ───────────────────────────────────────────────────── + +setup_aider() { + info "Setting up Aider integration..." + chmod +x "$SCRIPT_DIR/aider/aider_wrap.sh" + + # Write .env fragment for aider + local aider_env="$MNEMOCORE_DIR/.aider.env" + cat > "$aider_env" </dev/null || true diff --git a/integrations/universal/store_session.sh b/integrations/universal/store_session.sh new file mode 100644 index 0000000000000000000000000000000000000000..5b0fc2aebc829ccac68c41c29e779d854bffcdca --- /dev/null +++ b/integrations/universal/store_session.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# store_session.sh — Store session outcomes into MnemoCore +# ========================================================= +# Call this at the end of an AI coding session to persist key findings. +# +# Usage (interactive): +# ./store_session.sh +# +# Usage (non-interactive / scripted): +# ./store_session.sh "Fixed race condition in tier_manager.py" "bugfix,async" "my-project" + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BRIDGE_PY="${BRIDGE_PY:-$(realpath "$SCRIPT_DIR/../mnemo_bridge.py")}" + +if [[ $# -ge 1 ]]; then + CONTENT="$1" + TAGS="${2:-cli}" + CTX="${3:-}" +else + echo "Enter memory content (what was done/decided/fixed):" + read -r CONTENT + echo "Tags (comma-separated, e.g. bugfix,python,auth):" + read -r TAGS + echo "Context/project ID (optional, press Enter to skip):" + read -r CTX +fi + +if [[ -z "$CONTENT" ]]; then + echo "No content provided, nothing stored." >&2 + exit 0 +fi + +ARGS=(store "$CONTENT" --source "manual-session" --tags "$TAGS") +if [[ -n "$CTX" ]]; then + ARGS+=(--ctx "$CTX") +fi + +python3 "$BRIDGE_PY" "${ARGS[@]}" diff --git a/mnemocore_verify.py b/mnemocore_verify.py new file mode 100644 index 0000000000000000000000000000000000000000..9ce3a7ec4101388498b0264f983762cd0e3fcbd6 --- /dev/null +++ b/mnemocore_verify.py @@ -0,0 +1,136 @@ +import asyncio +import os +import shutil +import pytest +from pathlib import Path +import numpy as np + +# Set dummy test config environment +os.environ["HAIM_API_KEY"] = "test-key" + +from mnemocore.core.config import HAIMConfig, PathsConfig, TierConfig +from mnemocore.core.binary_hdv import TextEncoder, BinaryHDV +from mnemocore.core.hnsw_index import HNSWIndexManager +from mnemocore.core.engine import HAIMEngine +from mnemocore.core.tier_manager import TierManager +from unittest.mock import patch + +@pytest.fixture(autouse=True) +def setup_test_env(): + # Force all components to use test_data_verify as their data dir + # to prevent polluting/reading the user's real ./data folder + test_dir = Path("./test_data_verify") + test_dir.mkdir(exist_ok=True) + + cfg = HAIMConfig( + paths=PathsConfig( + data_dir=str(test_dir), + warm_mmap_dir=str(test_dir / "warm"), + cold_archive_dir=str(test_dir / "cold") + ) + ) + + with patch('mnemocore.core.config.get_config', return_value=cfg), \ + patch('mnemocore.core.hnsw_index.get_config', return_value=cfg), \ + patch('mnemocore.core.engine.get_config', return_value=cfg): + yield cfg + + if test_dir.exists(): + shutil.rmtree(test_dir) + + +@pytest.mark.asyncio +async def test_text_encoder_normalization(): + """Verify BUG-02: Text normalization fixes identical string variances""" + encoder = TextEncoder(dimension=1024) + hdv1 = encoder.encode("Hello World") + hdv2 = encoder.encode("hello, world!") + + assert (hdv1.data == hdv2.data).all(), "Normalization failed: Different HDVs for identical texts" + +def test_hnsw_singleton(): + """Verify BUG-08: HNSWIndexManager is a thread-safe singleton""" + HNSWIndexManager._instance = None + idx1 = HNSWIndexManager(dimension=1024) + idx2 = HNSWIndexManager(dimension=1024) + assert idx1 is idx2, "HNSWIndexManager is not a singleton" + +def test_hnsw_index_add_search(): + """Verify BUG-01 & BUG-03: Vector cache lost / Position mapping""" + HNSWIndexManager._instance = None + idx = HNSWIndexManager(dimension=1024) + + # Optional cleanup if it's reused + idx._id_map = [] + idx._vector_store = [] + if idx._index: + idx._index.reset() + + vec1 = BinaryHDV.random(1024) + vec2 = BinaryHDV.random(1024) + + idx.add("test_node_1", vec1.data) + idx.add("test_node_2", vec2.data) + + assert "test_node_1" in idx._id_map, "ID Map does not contain node 1" + assert "test_node_2" in idx._id_map, "ID Map does not contain node 2" + + # The search should return test_node_1 as the top result for vec1.data + res = idx.search(vec1.data, top_k=1) + assert res[0][0] == "test_node_1", f"Incorrect search return: {res}" + +@pytest.mark.asyncio +async def test_agent_isolation(): + """Verify BUG-09: Agent namespace isolation via engine and tier manager""" + HNSWIndexManager._instance = None + + test_data_dir = Path("./test_data_verify") + test_data_dir.mkdir(exist_ok=True) + + config = HAIMConfig( + qdrant=None, + paths=PathsConfig( + data_dir=str(test_data_dir), + warm_mmap_dir=str(test_data_dir / "warm"), + cold_archive_dir=str(test_data_dir / "cold") + ), + tiers_hot=TierConfig(max_memories=1000, ltp_threshold_min=0.0) + ) + # Prevent newly created memories (LTP=0.5) from being eagerly demoted + # We run purely local/in-memory for this unit test + + tier_manager = TierManager(config=config, qdrant_store=None) + engine = HAIMEngine( + persist_path=str(test_data_dir / "memory.jsonl"), + config=config, + tier_manager=tier_manager + ) + + try: + await engine.initialize() + + # Store two memories, isolated + await engine.store("Secret logic for agent 1", metadata={"agent_id": "agent_alpha"}) + await engine.store("Public logic for agent 2", metadata={"agent_id": "agent_beta"}) + + # Search global + res_global = await engine.query("logic", top_k=5) + # We expect 2 given we just pushed 2 + assert len(res_global) >= 2, f"Global search should return at least 2 memories, got {len(res_global)}" + + # Search isolated by agent_alpha + res_isolated = await engine.query("logic", top_k=5, metadata_filter={"agent_id": "agent_alpha"}) + + assert len(res_isolated) > 0, "Should find at least 1 memory for agent_alpha" + for nid, score in res_isolated: + node = await engine.get_memory(nid) + assert node.metadata.get("agent_id") == "agent_alpha", "Found leaked memory from another agent namespace!" + + finally: + await engine.close() + # Clean up test dir + if test_data_dir.exists(): + shutil.rmtree(test_data_dir) + +if __name__ == "__main__": + pytest.main(["-v", __file__]) diff --git a/src/mnemocore/agent_interface.py b/src/mnemocore/agent_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..2b6179b8a7a1a5cefdfc72ab5b5b90e3bf91d0f0 --- /dev/null +++ b/src/mnemocore/agent_interface.py @@ -0,0 +1,145 @@ +""" +Cognitive Memory Client +======================= +The high-level facade for autonomous agents to interact with the MnemoCore AGI Memory Substrate. +Provides easy methods for observation, episodic sequence tracking, and working memory recall. +""" + +from typing import List, Optional, Any, Tuple +import logging + +from .core.engine import HAIMEngine +from .core.working_memory import WorkingMemoryService, WorkingMemoryItem +from .core.episodic_store import EpisodicStoreService +from .core.semantic_store import SemanticStoreService +from .core.procedural_store import ProceduralStoreService +from .core.meta_memory import MetaMemoryService, SelfImprovementProposal +from .core.memory_model import Procedure + +logger = logging.getLogger(__name__) + +class CognitiveMemoryClient: + """ + Plug-and-play cognitive memory facade for agent frameworks (LangGraph, AutoGen, OpenClaw, etc.). + """ + def __init__( + self, + engine: HAIMEngine, + wm: WorkingMemoryService, + episodic: EpisodicStoreService, + semantic: SemanticStoreService, + procedural: ProceduralStoreService, + meta: MetaMemoryService, + ): + self.engine = engine + self.wm = wm + self.episodic = episodic + self.semantic = semantic + self.procedural = procedural + self.meta = meta + + # --- Observation & WM --- + + def observe(self, agent_id: str, content: str, kind: str = "observation", importance: float = 0.5, tags: Optional[List[str]] = None, **meta) -> str: + """ + Push a new observation or thought directly into the agent's short-term Working Memory. + """ + import uuid + from datetime import datetime + item_id = f"wm_{uuid.uuid4().hex[:8]}" + + item = WorkingMemoryItem( + id=item_id, + agent_id=agent_id, + created_at=datetime.utcnow(), + ttl_seconds=3600, # 1 hour default + content=content, + kind=kind, # type: ignore + importance=importance, + tags=tags or [], + hdv=None # Could encode via engine in future + ) + self.wm.push_item(agent_id, item) + logger.debug(f"Agent {agent_id} observed: {content[:30]}...") + return item_id + + def get_working_context(self, agent_id: str, limit: int = 16) -> List[WorkingMemoryItem]: + """ + Read the active, un-pruned context out of the agent's working memory buffer. + """ + state = self.wm.get_state(agent_id) + if not state: + return [] + + return state.items[-limit:] + + # --- Episodic --- + + def start_episode(self, agent_id: str, goal: str, context: Optional[str] = None) -> str: + """Begin a new temporally-linked event sequence.""" + return self.episodic.start_episode(agent_id, goal=goal, context=context) + + def append_event(self, episode_id: str, kind: str, content: str, **meta) -> None: + """Log an action or outcome to an ongoing episode.""" + self.episodic.append_event(episode_id, kind, content, meta) + + def end_episode(self, episode_id: str, outcome: str, reward: Optional[float] = None) -> None: + """Seal an episode, logging its final success or failure state.""" + self.episodic.end_episode(episode_id, outcome, reward) + + # --- Semantic / Retrieval --- + + async def recall( + self, + agent_id: str, + query: str, + context: Optional[str] = None, + top_k: int = 8, + modes: Tuple[str, ...] = ("episodic", "semantic") + ) -> List[dict]: + """ + A unified query interface that checks Working Memory, Episodic History, and the Semantic Vector Store. + Currently delegates heavily to the backing HAIMEngine, but can be augmented to return semantic concepts. + """ + results = [] + + # 1. Broad retrieval via existing HAIM engine (SM / general memories) + if "semantic" in modes: + engine_results = await self.engine.query(query, top_k=top_k) + for mem_id, score in engine_results: + node = await self.engine.tier_manager.get_memory(mem_id) # Fix: tier_manager.get_memory is async + if node: + results.append({"source": "semantic/engine", "content": node.content, "score": score}) + + # 2. Local episodic retrieval + if "episodic" in modes: + recent_eps = self.episodic.get_recent(agent_id, limit=top_k, context=context) + for ep in recent_eps: + summary = f"Episode(goal={ep.goal}, outcome={ep.outcome}, events={len(ep.events)})" + results.append({"source": "episodic", "content": summary, "score": ep.reliability}) + + # Sort and trim mixed results + results.sort(key=lambda x: x.get("score", 0.0), reverse=True) + return results[:top_k] + + # --- Procedural --- + + def suggest_procedures(self, agent_id: str, query: str, top_k: int = 5) -> List[Procedure]: + """Fetch executable tool-patterns based on the agent's intent.""" + return self.procedural.find_applicable_procedures(query, agent_id=agent_id, top_k=top_k) + + def record_procedure_outcome(self, proc_id: str, success: bool) -> None: + """Report on the utility of a chosen procedure.""" + self.procedural.record_procedure_outcome(proc_id, success) + + # --- Meta / Self-awareness --- + + def get_knowledge_gaps(self, agent_id: str, lookback_hours: int = 24) -> List[dict]: + """Return currently open knowledge gaps identified by the Pulse loop.""" + # Stubbed: Would interact with gap_detector + return [] + + def get_self_improvement_proposals(self) -> List[SelfImprovementProposal]: + """Retrieve system-generated proposals to improve operation or prompt alignment.""" + return self.meta.list_proposals() + diff --git a/src/mnemocore/api/main.py b/src/mnemocore/api/main.py index 72545623a196c93450ea1fc1b0bcfac9379d8bc4..5c0f680d677d2aaea8f10af9a4dea01c4304d043 100644 --- a/src/mnemocore/api/main.py +++ b/src/mnemocore/api/main.py @@ -156,9 +156,22 @@ async def lifespan(app: FastAPI): persist_path="./data/memory.jsonl", config=config, tier_manager=tier_manager, + working_memory=container.working_memory, + episodic_store=container.episodic_store, + semantic_store=container.semantic_store, ) await engine.initialize() app.state.engine = engine + # Also expose the cognitive client to app state for agentic frameworks + from mnemocore.agent_interface import CognitiveMemoryClient + app.state.cognitive_client = CognitiveMemoryClient( + engine=engine, + wm=container.working_memory, + episodic=container.episodic_store, + semantic=container.semantic_store, + procedural=container.procedural_store, + meta=container.meta_memory, + ) yield @@ -377,7 +390,8 @@ async def query_memory( API_REQUEST_COUNT.labels(method="POST", endpoint="/query", status="200").inc() # CPU heavy vector search (offloaded inside engine) - results = await engine.query(req.query, top_k=req.top_k) + metadata_filter = {"agent_id": req.agent_id} if req.agent_id else None + results = await engine.query(req.query, top_k=req.top_k, metadata_filter=metadata_filter) formatted = [] for mem_id, score in results: @@ -476,6 +490,52 @@ async def delete_memory( return {"ok": True, "deleted": memory_id} +# --- Phase 5: Cognitive Client Endpoints --- + +class ObserveRequest(BaseModel): + agent_id: str + content: str + kind: str = "observation" + importance: float = 0.5 + tags: Optional[List[str]] = None + +@app.post("/wm/observe", dependencies=[Depends(get_api_key)]) +async def observe_context(req: ObserveRequest, request: Request): + """Push an observation explicitly into Working Memory.""" + client = request.app.state.cognitive_client + if not client: + raise HTTPException(status_code=503, detail="Cognitive Client unavailable") + item_id = client.observe( + agent_id=req.agent_id, + content=req.content, + kind=req.kind, + importance=req.importance, + tags=req.tags + ) + return {"ok": True, "item_id": item_id} + +@app.get("/wm/context/{agent_id}", dependencies=[Depends(get_api_key)]) +async def get_working_context(agent_id: str, limit: int = 16, request: Request = None): + """Read active Working Memory context.""" + client = request.app.state.cognitive_client + items = client.get_working_context(agent_id, limit=limit) + return {"ok": True, "items": [ + {"id": i.id, "content": i.content, "kind": i.kind, "importance": i.importance} + for i in items + ]} + +class EpisodeStartRequest(BaseModel): + agent_id: str + goal: str + context: Optional[str] = None + +@app.post("/episodes/start", dependencies=[Depends(get_api_key)]) +async def start_episode(req: EpisodeStartRequest, request: Request): + """Start a new episode chain.""" + client = request.app.state.cognitive_client + ep_id = client.start_episode(req.agent_id, goal=req.goal, context=req.context) + return {"ok": True, "episode_id": ep_id} + # --- Conceptual Endpoints --- @app.post( @@ -627,6 +687,310 @@ async def rlm_query( } +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5.0 — Agent 1: Trust & Provenance Endpoints +# ───────────────────────────────────────────────────────────────────────────── + +@app.get( + "/memory/{memory_id}/lineage", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Trust"], + summary="Get full provenance lineage for a memory", +) +async def get_memory_lineage( + memory_id: str, + engine: HAIMEngine = Depends(get_engine), +): + """ + Return the complete provenance lineage of a memory: + origin (who created it, how, when) and all transformation events + (consolidated, verified, contradicted, archived, …). + """ + node = await engine.get_memory(memory_id) + if not node: + raise MemoryNotFoundError(memory_id) + + prov = getattr(node, "provenance", None) + if prov is None: + return { + "ok": True, + "memory_id": memory_id, + "provenance": None, + "message": "No provenance record attached to this memory.", + } + + return { + "ok": True, + "memory_id": memory_id, + "provenance": prov.to_dict(), + } + + +@app.get( + "/memory/{memory_id}/confidence", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Trust"], + summary="Get confidence envelope for a memory", +) +async def get_memory_confidence( + memory_id: str, + engine: HAIMEngine = Depends(get_engine), +): + """ + Return a structured confidence envelope for a memory, combining: + - Bayesian reliability (BayesianLTP posterior mean) + - access_count (evidence strength) + - staleness (days since last verification) + - source_type trust weight + - contradiction flag + + Level: high | medium | low | contradicted | stale + """ + from mnemocore.core.confidence import build_confidence_envelope + + node = await engine.get_memory(memory_id) + if not node: + raise MemoryNotFoundError(memory_id) + + prov = getattr(node, "provenance", None) + envelope = build_confidence_envelope(node, prov) + + return { + "ok": True, + "memory_id": memory_id, + "confidence": envelope, + } + + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5.0 — Agent 3 stub: Proactive Recall +# (Full implementation added by Agent 3 workstream) +# ───────────────────────────────────────────────────────────────────────────── + +@app.get( + "/proactive", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Autonomy"], + summary="Retrieve contextually relevant memories without explicit query", +) +async def get_proactive_memories( + agent_id: Optional[str] = None, + limit: int = 10, + engine: HAIMEngine = Depends(get_engine), +): + """ + Proactive recall stub (Phase 5.0 / Agent 3). + Returns the most recently active high-LTP memories as a stand-in + until the full ProactiveRecallDaemon is implemented. + """ + nodes = await engine.tier_manager.get_hot_snapshot() if hasattr(engine, "tier_manager") else [] + sorted_nodes = sorted(nodes, key=lambda n: n.ltp_strength, reverse=True)[:limit] + + from mnemocore.core.confidence import build_confidence_envelope + results = [] + for n in sorted_nodes: + prov = getattr(n, "provenance", None) + results.append({ + "id": n.id, + "content": n.content, + "ltp_strength": round(n.ltp_strength, 4), + "confidence": build_confidence_envelope(n, prov), + "tier": getattr(n, "tier", "hot"), + }) + + return {"ok": True, "proactive_results": results, "count": len(results)} + + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5.0 — Agent 2: Memory Lifecycle Endpoints +# ───────────────────────────────────────────────────────────────────────────── + +@app.get( + "/contradictions", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Lifecycle"], + summary="List active contradiction groups requiring resolution", +) +async def list_contradictions( + unresolved_only: bool = True, +): + """ + Returns all detected contradiction groups from the ContradictionRegistry. + By default only unresolved contradictions are returned. + """ + from mnemocore.core.contradiction import get_contradiction_detector + detector = get_contradiction_detector() + records = detector.registry.list_all(unresolved_only=unresolved_only) + return { + "ok": True, + "count": len(records), + "contradictions": [r.to_dict() for r in records], + } + + +class ResolveContradictionRequest(BaseModel): + note: Optional[str] = None + + +@app.post( + "/contradictions/{group_id}/resolve", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Lifecycle"], + summary="Mark a contradiction group as resolved", +) +async def resolve_contradiction(group_id: str, req: ResolveContradictionRequest): + """Manually resolve a detected contradiction.""" + from mnemocore.core.contradiction import get_contradiction_detector + detector = get_contradiction_detector() + success = detector.registry.resolve(group_id, note=req.note) + if not success: + raise HTTPException(status_code=404, detail=f"Contradiction group {group_id!r} not found.") + return {"ok": True, "resolved_group_id": group_id} + + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5.0 — Agent 3: Autonomous Cognition Endpoints +# ───────────────────────────────────────────────────────────────────────────── + +@app.get( + "/memory/{memory_id}/emotional-tag", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Autonomy"], + summary="Get emotional (valence/arousal) tag for a memory", +) +async def get_emotional_tag_ep( + memory_id: str, + engine: HAIMEngine = Depends(get_engine), +): + """Return the valence/arousal emotional metadata for a memory.""" + from mnemocore.core.emotional_tag import get_emotional_tag + node = await engine.get_memory(memory_id) + if not node: + raise MemoryNotFoundError(memory_id) + tag = get_emotional_tag(node) + return { + "ok": True, + "memory_id": memory_id, + "emotional_tag": { + "valence": tag.valence, + "arousal": tag.arousal, + "salience": round(tag.salience(), 4), + }, + } + + +class EmotionalTagPatchRequest(BaseModel): + valence: float + arousal: float + + +@app.patch( + "/memory/{memory_id}/emotional-tag", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Autonomy"], + summary="Attach or update emotional tag on a memory", +) +async def patch_emotional_tag( + memory_id: str, + req: EmotionalTagPatchRequest, + engine: HAIMEngine = Depends(get_engine), +): + from mnemocore.core.emotional_tag import EmotionalTag, attach_emotional_tag + node = await engine.get_memory(memory_id) + if not node: + raise MemoryNotFoundError(memory_id) + tag = EmotionalTag(valence=req.valence, arousal=req.arousal) + attach_emotional_tag(node, tag) + return {"ok": True, "memory_id": memory_id, "emotional_tag": tag.to_metadata_dict()} + + +# ───────────────────────────────────────────────────────────────────────────── +# Phase 5.0 — Agent 4: Prediction Endpoints +# ───────────────────────────────────────────────────────────────────────────── + +_prediction_store_instance = None + + +def _get_prediction_store(engine: HAIMEngine = Depends(get_engine)): + from mnemocore.core.prediction_store import PredictionStore + global _prediction_store_instance + if _prediction_store_instance is None: + _prediction_store_instance = PredictionStore(engine=engine) + return _prediction_store_instance + + +class CreatePredictionRequest(BaseModel): + content: str + confidence: float = 0.5 + deadline_days: Optional[float] = None + related_memory_ids: Optional[List[str]] = None + tags: Optional[List[str]] = None + + +class VerifyPredictionRequest(BaseModel): + success: bool + notes: Optional[str] = None + + +@app.post( + "/predictions", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Prediction"], + summary="Store a new forward-looking prediction", +) +async def create_prediction(req: CreatePredictionRequest): + from mnemocore.core.prediction_store import PredictionStore + global _prediction_store_instance + if _prediction_store_instance is None: + _prediction_store_instance = PredictionStore() + pred_id = _prediction_store_instance.create( + content=req.content, + confidence=req.confidence, + deadline_days=req.deadline_days, + related_memory_ids=req.related_memory_ids, + tags=req.tags, + ) + pred = _prediction_store_instance.get(pred_id) + return {"ok": True, "prediction": pred.to_dict()} + + +@app.get( + "/predictions", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Prediction"], + summary="List all predictions", +) +async def list_predictions(status: Optional[str] = None): + from mnemocore.core.prediction_store import PredictionStore + global _prediction_store_instance + if _prediction_store_instance is None: + _prediction_store_instance = PredictionStore() + return { + "ok": True, + "predictions": [ + p.to_dict() + for p in _prediction_store_instance.list_all(status=status) + ], + } + + +@app.post( + "/predictions/{pred_id}/verify", + dependencies=[Depends(get_api_key)], + tags=["Phase 5.0 — Prediction"], + summary="Verify or falsify a prediction", +) +async def verify_prediction(pred_id: str, req: VerifyPredictionRequest): + from mnemocore.core.prediction_store import PredictionStore + global _prediction_store_instance + if _prediction_store_instance is None: + _prediction_store_instance = PredictionStore() + pred = await _prediction_store_instance.verify(pred_id, success=req.success, notes=req.notes) + if pred is None: + raise HTTPException(status_code=404, detail=f"Prediction {pred_id!r} not found.") + return {"ok": True, "prediction": pred.to_dict()} + + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8100) diff --git a/src/mnemocore/core/agent_profile.py b/src/mnemocore/core/agent_profile.py new file mode 100644 index 0000000000000000000000000000000000000000..944d2fa5bdaf2da80dce3ecb5f5abdf313aa9b15 --- /dev/null +++ b/src/mnemocore/core/agent_profile.py @@ -0,0 +1,65 @@ +""" +Agent Profiles +============== +Persistent state encompassing quirks, long-term alignment details, and tooling preferences per individual actor. +Allows multiple independent agents to interact cleanly without memory namespace collisions. +""" + +from typing import Dict, List, Optional, Any +from dataclasses import dataclass, field +from datetime import datetime +import threading +import logging + +logger = logging.getLogger(__name__) + +@dataclass +class AgentProfile: + id: str + name: str + description: str + created_at: datetime + last_active: datetime + # Hard bounds over behavior: e.g. "Do not delete files without explicit prompt" + core_directives: List[str] = field(default_factory=list) + # Flexible learned preferences + preferences: dict[str, Any] = field(default_factory=dict) + # Agent-specific metrics + reliability_score: float = 1.0 + + +class AgentProfileService: + def __init__(self): + # Local state dict, should back out to SQLite or Redis + self._profiles: Dict[str, AgentProfile] = {} + self._lock = threading.RLock() + + def get_or_create_profile(self, agent_id: str, name: str = "Unknown Agent") -> AgentProfile: + """Retrieve the identity profile for an agent, constructing it if completely uninitialized.""" + with self._lock: + if agent_id not in self._profiles: + self._profiles[agent_id] = AgentProfile( + id=agent_id, + name=name, + description=f"Auto-generated profile for {agent_id}", + created_at=datetime.utcnow(), + last_active=datetime.utcnow() + ) + + profile = self._profiles[agent_id] + profile.last_active = datetime.utcnow() + return profile + + def update_preferences(self, agent_id: str, new_preferences: dict[str, Any]) -> None: + """Merge learned trait or task preferences into an agent's persistent identity.""" + with self._lock: + profile = self.get_or_create_profile(agent_id) + profile.preferences.update(new_preferences) + logger.debug(f"Updated preferences for agent {agent_id}.") + + def adjust_reliability(self, agent_id: str, points: float) -> None: + """Alter universal trust rating of the agent based on episodic action evaluations.""" + with self._lock: + profile = self.get_or_create_profile(agent_id) + profile.reliability_score = max(0.0, min(1.0, profile.reliability_score + points)) + diff --git a/src/mnemocore/core/anticipatory.py b/src/mnemocore/core/anticipatory.py new file mode 100644 index 0000000000000000000000000000000000000000..1857d6ab3858bf1e36b681616ded5f906b191076 --- /dev/null +++ b/src/mnemocore/core/anticipatory.py @@ -0,0 +1,51 @@ +from typing import List, Optional +from loguru import logger + +from .config import AnticipatoryConfig +from .synapse_index import SynapseIndex +from .tier_manager import TierManager +from .topic_tracker import TopicTracker + +class AnticipatoryEngine: + """ + Phase 13.2: Anticipatory Memory + Predicts which memories the user is likely to request next based on the + current topic trajectory and graph structure, and pre-loads them into the HOT tier. + """ + def __init__( + self, + config: AnticipatoryConfig, + synapse_index: SynapseIndex, + tier_manager: TierManager, + topic_tracker: TopicTracker + ): + self.config = config + self.synapse_index = synapse_index + self.tier_manager = tier_manager + self.topic_tracker = topic_tracker + + async def predict_and_preload(self, current_node_id: str) -> List[str]: + """ + Predicts surrounding context from the current node and ensures they are preloaded. + Uses the multi-hop network in the SynapseIndex to find likely next nodes. + """ + if not self.config.enabled: + return [] + + # Get neighbors up to predictive depth + # We use a relatively low depth to avoid flooding the HOT tier + neighbors = self.synapse_index.get_multi_hop_neighbors( + current_node_id, + depth=self.config.predictive_depth + ) + + # We'll just take the top 5 highest-weighted neighbors + # Sort by path weight (which multi-hop computes) + sorted_neighbors = sorted(neighbors.items(), key=lambda x: x[1], reverse=True)[:5] + target_ids = [nid for nid, weight in sorted_neighbors if nid != current_node_id] + + if target_ids: + logger.debug(f"Anticipatory engine pre-loading {len(target_ids)} predicted nodes.") + await self.tier_manager.anticipate(target_ids) + + return target_ids diff --git a/src/mnemocore/core/binary_hdv.py b/src/mnemocore/core/binary_hdv.py index 5da72c07a0f2fb19cd78e9b53bc0dbf6dcf97797..c39a2fc01e1628da51d626c6b0470e7e64d55b0f 100644 --- a/src/mnemocore/core/binary_hdv.py +++ b/src/mnemocore/core/binary_hdv.py @@ -22,6 +22,21 @@ import hashlib from typing import List, Optional, Tuple import numpy as np +import re + + +# Cached lookup table for popcount (bits set per byte value 0-255) +_POPCOUNT_TABLE: Optional[np.ndarray] = None + + +def _build_popcount_table() -> np.ndarray: + """Build or return cached popcount lookup table for bytes (0-255).""" + global _POPCOUNT_TABLE + if _POPCOUNT_TABLE is None: + _POPCOUNT_TABLE = np.array( + [bin(i).count("1") for i in range(256)], dtype=np.int32 + ) + return _POPCOUNT_TABLE class BinaryHDV: @@ -138,12 +153,13 @@ class BinaryHDV: """ Hamming distance: count of differing bits. - Uses np.unpackbits + sum for correctness. + Uses lookup table for speed (replacing unpackbits). Range: [0, dimension]. """ assert self.dimension == other.dimension xor_result = np.bitwise_xor(self.data, other.data) - return int(np.unpackbits(xor_result).sum()) + # Optimized: use precomputed popcount table instead of unpacking bits + return int(_build_popcount_table()[xor_result].sum()) def normalized_distance(self, other: "BinaryHDV") -> float: """Hamming distance normalized to [0.0, 1.0].""" @@ -210,7 +226,8 @@ class BinaryHDV: return cls(data=data, dimension=dimension) def __repr__(self) -> str: - popcount = int(np.unpackbits(self.data).sum()) + # Optimized: use precomputed popcount table + popcount = int(_build_popcount_table()[self.data].sum()) return f"BinaryHDV(dim={self.dimension}, popcount={popcount}/{self.dimension})" def __eq__(self, other: object) -> bool: @@ -383,26 +400,37 @@ class TextEncoder: """ Encode a text string to a binary HDV. - Tokenization: simple whitespace split + lowercasing. + Tokenization: simple whitespace split after normalization. Each token is bound with its position via XOR(token, permute(position_marker, i)). All position-bound tokens are bundled via majority vote. """ - tokens = text.lower().split() + # BUG-02 Fix: strip punctuation and normalize spaces + normalized = re.sub(r'[^\w\s]', '', text).lower() + tokens = normalized.split() if not tokens: return BinaryHDV.random(self.dimension) if len(tokens) == 1: return self.get_token_vector(tokens[0]) - # Build position-bound token vectors - bound_vectors = [] - for i, token in enumerate(tokens): - token_hdv = self.get_token_vector(token) - # Permute by position index for order encoding - positioned = token_hdv.permute(shift=i) - bound_vectors.append(positioned) + # Build position-bound token vectors (#27) + # Optimized: Batch process data instead of multiple object instantiations + token_hdvs = [self.get_token_vector(t) for t in tokens] + packed_data = np.stack([v.data for v in token_hdvs], axis=0) + all_bits = np.unpackbits(packed_data, axis=1) + + # Apply position-based permutations (roll) + for i in range(len(tokens)): + if i > 0: + all_bits[i] = np.roll(all_bits[i], i) - return majority_bundle(bound_vectors) + # Vectorized majority vote (equivalent to majority_bundle) + sums = all_bits.sum(axis=0) + threshold = len(tokens) / 2.0 + result_bits = np.zeros(self.dimension, dtype=np.uint8) + result_bits[sums > threshold] = 1 + + return BinaryHDV(data=np.packbits(result_bits), dimension=self.dimension) def encode_with_context( self, text: str, context_hdv: BinaryHDV @@ -415,21 +443,3 @@ class TextEncoder: """ content_hdv = self.encode(text) return content_hdv.xor_bind(context_hdv) - - -# ====================================================================== -# Internal helpers -# ====================================================================== - -# Cached lookup table for popcount (bits set per byte value 0-255) -_POPCOUNT_TABLE: Optional[np.ndarray] = None - - -def _build_popcount_table() -> np.ndarray: - """Build or return cached popcount lookup table for bytes (0-255).""" - global _POPCOUNT_TABLE - if _POPCOUNT_TABLE is None: - _POPCOUNT_TABLE = np.array( - [bin(i).count("1") for i in range(256)], dtype=np.int32 - ) - return _POPCOUNT_TABLE diff --git a/src/mnemocore/core/confidence.py b/src/mnemocore/core/confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..0cb911dba9665af59c03fb122b8abb038b866933 --- /dev/null +++ b/src/mnemocore/core/confidence.py @@ -0,0 +1,196 @@ +""" +Confidence Calibration Module (Phase 5.0) +========================================== +Generates structured confidence envelopes for retrieved memories, +combining all available trust signals into a single queryable object. + +Signals used: + - BayesianLTP reliability (mean of Beta posterior) + - access_count (low count → less evidence) + - staleness (days since last verification) + - source type (external ≤ user_correction vs observation) + - contradiction flag (from ProvenanceRecord) + +Output: a ConfidenceEnvelope dict appended to every query response, +enabling consuming agents to make trust-aware decisions. + +Public API: + env = ConfidenceEnvelopeGenerator.build(node, provenance) + level = env["level"] # "high" | "medium" | "low" | "contradicted" | "stale" +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from .node import MemoryNode + from .provenance import ProvenanceRecord + + +# ------------------------------------------------------------------ # +# Confidence levels (ordered by trust) # +# ------------------------------------------------------------------ # + +LEVEL_HIGH = "high" +LEVEL_MEDIUM = "medium" +LEVEL_LOW = "low" +LEVEL_CONTRADICTED = "contradicted" +LEVEL_STALE = "stale" + +# Thresholds +RELIABILITY_HIGH_THRESHOLD = 0.80 +RELIABILITY_MEDIUM_THRESHOLD = 0.50 +ACCESS_COUNT_MIN_EVIDENCE = 2 # Less than this → low confidence +ACCESS_COUNT_HIGH_EVIDENCE = 5 # At least this → supports high confidence +STALENESS_STALE_DAYS = 30 # Days without verification → stale + + +# ------------------------------------------------------------------ # +# Source-type trust weights # +# ------------------------------------------------------------------ # + +SOURCE_TRUST: Dict[str, float] = { + "observation": 1.0, + "inference": 0.8, + "external_sync": 0.75, + "dream": 0.6, + "consolidation": 0.85, + "prediction": 0.5, + "user_correction": 1.0, + "unknown": 0.5, +} + + +# ------------------------------------------------------------------ # +# Confidence Envelope Generator # +# ------------------------------------------------------------------ # + +class ConfidenceEnvelopeGenerator: + """ + Builds a confidence_envelope dict for a MemoryNode. + + Does NOT mutate the node — only reads fields. + Thread-safe; no shared state. + """ + + @staticmethod + def _reliability(node: "MemoryNode") -> float: + """ + Extract reliability float from the node. + Falls back to ltp_strength if no Bayesian state is attached. + """ + bayes = getattr(node, "_bayes", None) + if bayes is not None: + return float(bayes.mean) + return float(getattr(node, "ltp_strength", 0.5)) + + @staticmethod + def _staleness_days(node: "MemoryNode", provenance: Optional["ProvenanceRecord"]) -> float: + """Days since last verification, or days since last access.""" + if provenance: + # Find the most recent 'verified' event + for evt in reversed(provenance.lineage): + if evt.event == "verified" and evt.outcome is True: + try: + ts = datetime.fromisoformat(evt.timestamp) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - ts + return delta.total_seconds() / 86400.0 + except (ValueError, TypeError): + pass + + # Fall back to last_accessed on the node + last = getattr(node, "last_accessed", None) + if last is not None: + if getattr(last, "tzinfo", None) is None: + last = last.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - last + return delta.total_seconds() / 86400.0 + + return 0.0 + + @classmethod + def build( + cls, + node: "MemoryNode", + provenance: Optional["ProvenanceRecord"] = None, + ) -> Dict[str, Any]: + """ + Build a full confidence_envelope dict for the given node. + + Returns a dict suitable for direct JSON serialization. + """ + reliability = cls._reliability(node) + access_count: int = getattr(node, "access_count", 1) + staleness: float = cls._staleness_days(node, provenance) + + # Determine source type for trust weighting + source_type = "unknown" + if provenance: + source_type = provenance.origin.type + source_trust = SOURCE_TRUST.get(source_type, 0.5) + + # Contradiction check + is_contradicted = provenance.is_contradicted() if provenance else False + + # Last verified date (human-readable) + last_verified: Optional[str] = None + if provenance: + for evt in reversed(provenance.lineage): + if evt.event == "verified" and evt.outcome is True: + last_verified = evt.timestamp + break + + # ---- Determine level ------------------------------------ # + if is_contradicted: + level = LEVEL_CONTRADICTED + elif staleness > STALENESS_STALE_DAYS: + level = LEVEL_STALE + elif ( + reliability >= RELIABILITY_HIGH_THRESHOLD + and access_count >= ACCESS_COUNT_HIGH_EVIDENCE + and source_trust >= 0.75 + ): + level = LEVEL_HIGH + elif reliability >= RELIABILITY_MEDIUM_THRESHOLD and access_count >= ACCESS_COUNT_MIN_EVIDENCE: + level = LEVEL_MEDIUM + else: + level = LEVEL_LOW + + envelope: Dict[str, Any] = { + "level": level, + "reliability": round(reliability, 4), + "access_count": access_count, + "staleness_days": round(staleness, 1), + "source_type": source_type, + "source_trust": round(source_trust, 2), + "is_contradicted": is_contradicted, + } + if last_verified: + envelope["last_verified"] = last_verified + + return envelope + + +# ------------------------------------------------------------------ # +# Convenience function # +# ------------------------------------------------------------------ # + +def build_confidence_envelope( + node: "MemoryNode", + provenance: Optional["ProvenanceRecord"] = None, +) -> Dict[str, Any]: + """ + Module-level shortcut for ConfidenceEnvelopeGenerator.build(). + + Args: + node: MemoryNode to evaluate. + provenance: Optional ProvenanceRecord for the node. + + Returns: + confidence_envelope dict with level, reliability, staleness, etc. + """ + return ConfidenceEnvelopeGenerator.build(node, provenance) diff --git a/src/mnemocore/core/config.py b/src/mnemocore/core/config.py index 3c4b4f20a4ce6ee799a3165b6c22118fe4c26a80..99d4489df381770dfef71d2d6614cc07784ecd96 100644 --- a/src/mnemocore/core/config.py +++ b/src/mnemocore/core/config.py @@ -139,6 +139,37 @@ class EncodingConfig: token_method: str = "bundle" +@dataclass(frozen=True) +class SynapseConfig: + """Configuration for Phase 12.1: Aggressive Synapse Formation""" + similarity_threshold: float = 0.5 + auto_bind_on_store: bool = True + multi_hop_depth: int = 2 + + +@dataclass(frozen=True) +class ContextConfig: + """Configuration for Phase 12.2: Contextual Awareness""" + enabled: bool = True + shift_threshold: float = 0.3 + rolling_window_size: int = 5 + + +@dataclass(frozen=True) +class PreferenceConfig: + """Configuration for Phase 12.3: Preference Learning""" + enabled: bool = True + learning_rate: float = 0.1 + history_limit: int = 100 + + +@dataclass(frozen=True) +class AnticipatoryConfig: + """Configuration for Phase 13.2: Anticipatory Memory""" + enabled: bool = True + predictive_depth: int = 1 + + @dataclass(frozen=True) class DreamLoopConfig: """Configuration for the dream loop (subconscious background processing).""" @@ -194,11 +225,19 @@ class SubconsciousAIConfig: max_memories_per_cycle: int = 10 # Process at most N memories per pulse +@dataclass(frozen=True) +class PulseConfig: + """Configuration for Phase 5 AGI Pulse Loop orchestrator.""" + enabled: bool = True + interval_seconds: int = 30 + max_agents_per_tick: int = 50 + max_episodes_per_tick: int = 200 + @dataclass(frozen=True) class HAIMConfig: """Root configuration for the HAIM system.""" - version: str = "3.0" + version: str = "4.5" dimensionality: int = 16384 encoding: EncodingConfig = field(default_factory=EncodingConfig) tiers_hot: TierConfig = field( @@ -230,8 +269,13 @@ class HAIMConfig: paths: PathsConfig = field(default_factory=PathsConfig) consolidation: ConsolidationConfig = field(default_factory=ConsolidationConfig) attention_masking: AttentionMaskingConfig = field(default_factory=AttentionMaskingConfig) + synapse: SynapseConfig = field(default_factory=SynapseConfig) + context: ContextConfig = field(default_factory=ContextConfig) + preference: PreferenceConfig = field(default_factory=PreferenceConfig) + anticipatory: AnticipatoryConfig = field(default_factory=AnticipatoryConfig) dream_loop: DreamLoopConfig = field(default_factory=DreamLoopConfig) subconscious_ai: SubconsciousAIConfig = field(default_factory=SubconsciousAIConfig) + pulse: PulseConfig = field(default_factory=PulseConfig) def _env_override(key: str, default): @@ -347,15 +391,6 @@ def load_config(path: Optional[Path] = None) -> HAIMConfig: token_method=enc_raw.get("token_method", "bundle"), ) - # Build LTP config - ltp_raw = raw.get("ltp") or {} - ltp = LTPConfig( - initial_importance=ltp_raw.get("initial_importance", 0.5), - decay_lambda=ltp_raw.get("decay_lambda", 0.01), - permanence_threshold=ltp_raw.get("permanence_threshold", 0.95), - half_life_days=ltp_raw.get("half_life_days", 30.0), - ) - # Build paths config paths_raw = raw.get("paths") or {} paths = PathsConfig( @@ -499,6 +534,37 @@ def load_config(path: Optional[Path] = None) -> HAIMConfig: model=_env_override("DREAM_LOOP_MODEL", dream_raw.get("model", "gemma3:1b")), ) + # Build synapse config (Phase 12.1) + syn_raw = raw.get("synapse") or {} + synapse = SynapseConfig( + similarity_threshold=_env_override("SYNAPSE_SIMILARITY_THRESHOLD", syn_raw.get("similarity_threshold", 0.5)), + auto_bind_on_store=_env_override("SYNAPSE_AUTO_BIND_ON_STORE", syn_raw.get("auto_bind_on_store", True)), + multi_hop_depth=_env_override("SYNAPSE_MULTI_HOP_DEPTH", syn_raw.get("multi_hop_depth", 2)), + ) + + # Build context config (Phase 12.2) + ctx_raw = raw.get("context") or {} + context = ContextConfig( + enabled=_env_override("CONTEXT_ENABLED", ctx_raw.get("enabled", True)), + shift_threshold=_env_override("CONTEXT_SHIFT_THRESHOLD", ctx_raw.get("shift_threshold", 0.3)), + rolling_window_size=_env_override("CONTEXT_ROLLING_WINDOW_SIZE", ctx_raw.get("rolling_window_size", 5)), + ) + + # Build preference config (Phase 12.3) + pref_raw = raw.get("preference") or {} + preference = PreferenceConfig( + enabled=_env_override("PREFERENCE_ENABLED", pref_raw.get("enabled", True)), + learning_rate=_env_override("PREFERENCE_LEARNING_RATE", pref_raw.get("learning_rate", 0.1)), + history_limit=_env_override("PREFERENCE_HISTORY_LIMIT", pref_raw.get("history_limit", 100)), + ) + + # Build anticipatory config (Phase 13.2) + ant_raw = raw.get("anticipatory") or {} + anticipatory = AnticipatoryConfig( + enabled=_env_override("ANTICIPATORY_ENABLED", ant_raw.get("enabled", True)), + predictive_depth=_env_override("ANTICIPATORY_PREDICTIVE_DEPTH", ant_raw.get("predictive_depth", 1)), + ) + # Build subconscious AI config (Phase 4.4 BETA) sub_raw = raw.get("subconscious_ai") or {} subconscious_ai = SubconsciousAIConfig( @@ -524,8 +590,17 @@ def load_config(path: Optional[Path] = None) -> HAIMConfig: max_memories_per_cycle=_env_override("SUBCONSCIOUS_AI_MAX_MEMORIES_PER_CYCLE", sub_raw.get("max_memories_per_cycle", 10)), ) + # Build pulse config (Phase 5.0) + pulse_raw = raw.get("pulse") or {} + pulse = PulseConfig( + enabled=_env_override("PULSE_ENABLED", pulse_raw.get("enabled", True)), + interval_seconds=_env_override("PULSE_INTERVAL_SECONDS", pulse_raw.get("interval_seconds", 30)), + max_agents_per_tick=_env_override("PULSE_MAX_AGENTS_PER_TICK", pulse_raw.get("max_agents_per_tick", 50)), + max_episodes_per_tick=_env_override("PULSE_MAX_EPISODES_PER_TICK", pulse_raw.get("max_episodes_per_tick", 200)), + ) + return HAIMConfig( - version=raw.get("version", "3.0"), + version=raw.get("version", "4.5"), dimensionality=dimensionality, encoding=encoding, tiers_hot=_build_tier("hot", hot_raw), @@ -542,8 +617,13 @@ def load_config(path: Optional[Path] = None) -> HAIMConfig: paths=paths, consolidation=consolidation, attention_masking=attention_masking, + synapse=synapse, + context=context, + preference=preference, + anticipatory=anticipatory, dream_loop=dream_loop, subconscious_ai=subconscious_ai, + pulse=pulse, ) diff --git a/src/mnemocore/core/container.py b/src/mnemocore/core/container.py index 696570b5a6e548c33437940aa98ddf44203f0d22..5d8f21579baae2138d4d6ed964a02d0f7216ebbb 100644 --- a/src/mnemocore/core/container.py +++ b/src/mnemocore/core/container.py @@ -12,6 +12,14 @@ from .config import HAIMConfig from .async_storage import AsyncRedisStorage from .qdrant_store import QdrantStore +# Phase 5 AGI Services +from .working_memory import WorkingMemoryService +from .episodic_store import EpisodicStoreService +from .semantic_store import SemanticStoreService +from .procedural_store import ProceduralStoreService +from .meta_memory import MetaMemoryService +from .agent_profile import AgentProfileService + @dataclass class Container: @@ -21,6 +29,14 @@ class Container: config: HAIMConfig redis_storage: Optional[AsyncRedisStorage] = None qdrant_store: Optional[QdrantStore] = None + + # Phase 5 Services + working_memory: Optional[WorkingMemoryService] = None + episodic_store: Optional[EpisodicStoreService] = None + semantic_store: Optional[SemanticStoreService] = None + procedural_store: Optional[ProceduralStoreService] = None + meta_memory: Optional[MetaMemoryService] = None + agent_profiles: Optional[AgentProfileService] = None def build_container(config: HAIMConfig) -> Container: @@ -57,6 +73,14 @@ def build_container(config: HAIMConfig) -> Container: hnsw_ef_construct=config.qdrant.hnsw_ef_construct, ) + # Initialize Phase 5 AGI Services + container.working_memory = WorkingMemoryService() + container.episodic_store = EpisodicStoreService() + container.semantic_store = SemanticStoreService(qdrant_store=container.qdrant_store) + container.procedural_store = ProceduralStoreService() + container.meta_memory = MetaMemoryService() + container.agent_profiles = AgentProfileService() + return container diff --git a/src/mnemocore/core/contradiction.py b/src/mnemocore/core/contradiction.py new file mode 100644 index 0000000000000000000000000000000000000000..36877748ecd4e9efa013fe33a2e571bb78b70837 --- /dev/null +++ b/src/mnemocore/core/contradiction.py @@ -0,0 +1,336 @@ +""" +Contradiction Detection Module (Phase 5.0) +========================================== +Detects contradicting memories in MnemoCore using a two-stage pipeline: + +Stage 1: TextEncoder similarity search (fast, vector-based) + - At /store time: compare new memory against top-5 existing memories + - If similarity > SIMILARITY_THRESHOLD (0.80) → proceed to Stage 2 + +Stage 2: LLM-based semantic comparison (accurate, but heavier) + - Uses SubconsciousAI connector to evaluate if two memories actually contradict + - Avoids false positives from paraphrases (similarity doesn't mean contradiction) + +On confirmed contradiction: + - Both memories receive a 'contradiction_group_id' in their provenance lineage + - Both are flagged in their metadata + - The API returns an alert in the store response + - Entries are added to a ContradictionRegistry for the /contradictions endpoint + +Background scan: + - ContradictionDetector.scan(nodes) can be called from ConsolidationWorker + +Public API: + detector = ContradictionDetector(engine) + result = await detector.check_on_store(new_content, new_node, existing_nodes) + all = detector.registry.list_all() +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from loguru import logger + +if TYPE_CHECKING: + from .node import MemoryNode + + +# ------------------------------------------------------------------ # +# Thresholds # +# ------------------------------------------------------------------ # + +SIMILARITY_THRESHOLD: float = 0.80 # Above this → suspect contradiction +LLM_CONFIRM_MIN_SCORE: float = 0.70 # LLM contradiction confidence minimum + + +# ------------------------------------------------------------------ # +# ContradictionRecord # +# ------------------------------------------------------------------ # + +@dataclass +class ContradictionRecord: + """A detected contradiction between two memories.""" + group_id: str = field(default_factory=lambda: f"cg_{uuid.uuid4().hex[:12]}") + memory_a_id: str = "" + memory_b_id: str = "" + similarity_score: float = 0.0 + llm_confirmed: bool = False + detected_at: str = field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + resolved: bool = False + resolution_note: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "group_id": self.group_id, + "memory_a_id": self.memory_a_id, + "memory_b_id": self.memory_b_id, + "similarity_score": round(self.similarity_score, 4), + "llm_confirmed": self.llm_confirmed, + "detected_at": self.detected_at, + "resolved": self.resolved, + "resolution_note": self.resolution_note, + } + + +# ------------------------------------------------------------------ # +# ContradictionRegistry # +# ------------------------------------------------------------------ # + +class ContradictionRegistry: + """In-memory store of detected contradictions (survives until restart).""" + + def __init__(self) -> None: + self._records: Dict[str, ContradictionRecord] = {} + + def register(self, record: ContradictionRecord) -> None: + self._records[record.group_id] = record + + def resolve(self, group_id: str, note: Optional[str] = None) -> bool: + if group_id in self._records: + self._records[group_id].resolved = True + self._records[group_id].resolution_note = note + return True + return False + + def list_all(self, unresolved_only: bool = True) -> List[ContradictionRecord]: + recs = list(self._records.values()) + if unresolved_only: + recs = [r for r in recs if not r.resolved] + return sorted(recs, key=lambda r: r.detected_at, reverse=True) + + def list_for_memory(self, memory_id: str) -> List[ContradictionRecord]: + return [ + r for r in self._records.values() + if r.memory_a_id == memory_id or r.memory_b_id == memory_id + ] + + def __len__(self) -> int: + return len([r for r in self._records.values() if not r.resolved]) + + +# ------------------------------------------------------------------ # +# ContradictionDetector # +# ------------------------------------------------------------------ # + +class ContradictionDetector: + """ + Two-stage contradiction detector. + + Stage 1: Vector similarity via the engine's binary HDV comparison. + Stage 2: LLM semantic check via SubconsciousAI (optional). + """ + + def __init__( + self, + engine=None, # HAIMEngine — optional; if None, similarity check uses fallback + similarity_threshold: float = SIMILARITY_THRESHOLD, + top_k: int = 5, + use_llm: bool = True, + ) -> None: + self.engine = engine + self.similarity_threshold = similarity_threshold + self.top_k = top_k + self.use_llm = use_llm + self.registry = ContradictionRegistry() + + # ---- Similarity helpers -------------------------------------- # + + def _hamming_similarity(self, node_a: "MemoryNode", node_b: "MemoryNode") -> float: + """ + Compute binary HDV similarity between two nodes. + Similarity = 1 - normalized_hamming_distance. + """ + try: + import numpy as np + a = node_a.hdv.data + b = node_b.hdv.data + xor = np.bitwise_xor(a, b) + ham = float(bin(int.from_bytes(xor.tobytes(), "little")).count("1")) + dim = len(a) * 8 + return 1.0 - ham / dim + except Exception: + return 0.0 + + # ---- LLM contradiction check --------------------------------- # + + async def _llm_contradicts( + self, content_a: str, content_b: str + ) -> Tuple[bool, float]: + """ + Ask SubconsciousAI if two contents contradict each other. + Returns (is_contradiction, confidence_score). + Falls back to False if LLM is unavailable. + """ + if not self.engine or not self.use_llm: + return False, 0.0 + + try: + subcon = getattr(self.engine, "subconscious_ai", None) + if subcon is None: + return False, 0.0 + + prompt = ( + "Do the following two statements contradict each other? " + "Answer with a JSON object: {\"contradiction\": true/false, \"confidence\": 0.0-1.0}.\n\n" + f"Statement A: {content_a[:500]}\n" + f"Statement B: {content_b[:500]}" + ) + raw = await subcon.generate(prompt, max_tokens=64) + import json as _json + parsed = _json.loads(raw.strip()) + return bool(parsed.get("contradiction", False)), float(parsed.get("confidence", 0.0)) + except Exception as exc: + logger.debug(f"LLM contradiction check failed: {exc}") + return False, 0.0 + + # ---- Flag helpers ------------------------------------------- # + + def _flag_node(self, node: "MemoryNode", group_id: str) -> None: + """Attach contradiction metadata to a node's provenance and metadata fields.""" + node.metadata["contradiction_group_id"] = group_id + node.metadata["contradicted_at"] = datetime.now(timezone.utc).isoformat() + + prov = getattr(node, "provenance", None) + if prov is not None: + prov.mark_contradicted(group_id) + + # ---- Main API ------------------------------------------------ # + + async def check_on_store( + self, + new_node: "MemoryNode", + candidates: Optional[List["MemoryNode"]] = None, + ) -> Optional[ContradictionRecord]: + """ + Check a newly stored node against existing memories. + + Args: + new_node: The node just stored. + candidates: Optional pre-fetched list of nodes to compare against. + If None and engine is available, fetches via HDV search. + + Returns: + ContradictionRecord if a contradiction was detected, else None. + """ + # Fetch candidates if not provided + if candidates is None and self.engine is not None: + try: + results = await self.engine.query( + new_node.content, top_k=self.top_k + ) + nodes = [] + for mem_id, _score in results: + n = await self.engine.get_memory(mem_id) + if n and n.id != new_node.id: + nodes.append(n) + candidates = nodes + except Exception as e: + logger.debug(f"ContradictionDetector: candidate fetch failed: {e}") + candidates = [] + + if not candidates: + return None + + # Stage 1: similarity filter + high_sim_candidates = [] + for cand in candidates: + sim = self._hamming_similarity(new_node, cand) + if sim >= self.similarity_threshold: + high_sim_candidates.append((cand, sim)) + + if not high_sim_candidates: + return None + + # Stage 2: LLM confirmation for the highest-similarity candidate + high_sim_candidates.sort(key=lambda x: x[1], reverse=True) + top_cand, top_sim = high_sim_candidates[0] + + is_contradiction = False + llm_confirmed = False + + if self.use_llm: + is_contradiction, conf = await self._llm_contradicts( + new_node.content, top_cand.content + ) + llm_confirmed = is_contradiction and conf >= LLM_CONFIRM_MIN_SCORE + else: + # Without LLM, use high similarity as a soft contradiction signal + is_contradiction = top_sim >= 0.90 + llm_confirmed = False + + if not is_contradiction: + return None + + # Register the contradiction + record = ContradictionRecord( + memory_a_id=new_node.id, + memory_b_id=top_cand.id, + similarity_score=top_sim, + llm_confirmed=llm_confirmed, + ) + self.registry.register(record) + self._flag_node(new_node, record.group_id) + self._flag_node(top_cand, record.group_id) + + logger.warning( + f"⚠️ Contradiction detected: {new_node.id[:8]} ↔ {top_cand.id[:8]} " + f"(sim={top_sim:.3f}, llm_confirmed={llm_confirmed}, group={record.group_id})" + ) + return record + + async def scan(self, nodes: "List[MemoryNode]") -> List[ContradictionRecord]: + """ + Background scan: compare each node against its peers in the provided list. + Called periodically from ConsolidationWorker. + + Returns all newly detected contradiction records. + """ + found: List[ContradictionRecord] = [] + n = len(nodes) + for i in range(n): + for j in range(i + 1, n): + sim = self._hamming_similarity(nodes[i], nodes[j]) + if sim < self.similarity_threshold: + continue + is_contradiction, _ = await self._llm_contradicts( + nodes[i].content, nodes[j].content + ) + if not is_contradiction: + continue + record = ContradictionRecord( + memory_a_id=nodes[i].id, + memory_b_id=nodes[j].id, + similarity_score=sim, + llm_confirmed=True, + ) + self.registry.register(record) + self._flag_node(nodes[i], record.group_id) + self._flag_node(nodes[j], record.group_id) + found.append(record) + + if found: + logger.info(f"ContradictionDetector background scan: {len(found)} contradictions found in {n} nodes") + return found + + +# ------------------------------------------------------------------ # +# Module singleton # +# ------------------------------------------------------------------ # + +_DETECTOR: ContradictionDetector | None = None + + +def get_contradiction_detector(engine=None) -> ContradictionDetector: + """Return the shared ContradictionDetector singleton.""" + global _DETECTOR + if _DETECTOR is None: + _DETECTOR = ContradictionDetector(engine=engine) + elif engine is not None and _DETECTOR.engine is None: + _DETECTOR.engine = engine + return _DETECTOR diff --git a/src/mnemocore/core/cross_domain.py b/src/mnemocore/core/cross_domain.py new file mode 100644 index 0000000000000000000000000000000000000000..3c7606327b4b20c313386bcaafa612f6050c719b --- /dev/null +++ b/src/mnemocore/core/cross_domain.py @@ -0,0 +1,211 @@ +""" +Cross-Domain Association Builder (Phase 5.0 — Agent 3) +======================================================= +Automatically links memories across three semantic domains: + + strategic – goals, decisions, roadmaps, strategies + operational – code, bugs, documentation, tasks + personal – preferences, habits, relationships, context + +Cross-domain synapses improve holistic reasoning: when a strategic +goal changes, the system can surface related operational tasks or +personal context without being explicitly queried. + +Implementation: + - Each memory is tagged with a `domain` in its metadata (or inferred) + - CrossDomainSynapseBuilder monitors recently stored memories + - Co-occurrence within a time window → create a cross-domain synapse + - Synapse weight is damped (0.2×) relative to intra-domain (1.0×) + +Integration with RippleContext: + - ripple_context.py uses domain_weight when propagating context + - Cross-domain propagation uses CROSS_DOMAIN_WEIGHT as the multiplier + +Public API: + builder = CrossDomainSynapseBuilder(engine) + await builder.process_new_memory(node) # call after /store + pairs = await builder.scan_recent(hours=1) # background scan +""" + +from __future__ import annotations + +from datetime import datetime, timezone, timedelta +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple +from loguru import logger + +if TYPE_CHECKING: + from .node import MemoryNode + + +# ------------------------------------------------------------------ # +# Constants # +# ------------------------------------------------------------------ # + +DOMAINS = {"strategic", "operational", "personal"} +DEFAULT_DOMAIN = "operational" + +# Weight applied to cross-domain synapses (vs 1.0 for intra-domain) +CROSS_DOMAIN_WEIGHT: float = 0.2 + +# Time window for co-occurrence detection (hours) +COOCCURRENCE_WINDOW_HOURS: float = 2.0 + +# Keywords used to infer domain automatically if not tagged +DOMAIN_KEYWORDS: Dict[str, List[str]] = { + "strategic": [ + "goal", "strategy", "roadmap", "vision", "mission", "objective", + "decision", "priority", "kpi", "okr", "plan", "budget", "market", + ], + "personal": [ + "prefer", "habit", "feel", "emotion", "prefer", "like", "dislike", + "relationship", "trust", "colleague", "friend", "name", "remember me", + ], + "operational": [ + "code", "bug", "fix", "implement", "test", "deploy", "api", + "function", "class", "module", "error", "exception", "task", "ticket", + ], +} + + +# ------------------------------------------------------------------ # +# Domain inference # +# ------------------------------------------------------------------ # + +def infer_domain(content: str, metadata: Optional[Dict] = None) -> str: + """ + Infer the semantic domain of a memory from its content and metadata. + + Priority: + 1. metadata["domain"] if set + 2. keyword match in content (highest score wins) + 3. DEFAULT_DOMAIN ("operational") + """ + if metadata and "domain" in metadata: + d = metadata["domain"].lower() + return d if d in DOMAINS else DEFAULT_DOMAIN + + content_lower = content.lower() + best_domain = DEFAULT_DOMAIN + best_count = 0 + + for domain, keywords in DOMAIN_KEYWORDS.items(): + count = sum(1 for kw in keywords if kw in content_lower) + if count > best_count: + best_count = count + best_domain = domain + + return best_domain + + +# ------------------------------------------------------------------ # +# CrossDomainSynapseBuilder # +# ------------------------------------------------------------------ # + +class CrossDomainSynapseBuilder: + """ + Detects cross-domain co-occurrences and requests synapse creation. + + Works by maintaining a rolling buffer of recently stored memories, + then pairing memories from different domains that appeared within + COOCCURRENCE_WINDOW_HOURS of each other. + """ + + def __init__( + self, + engine=None, # HAIMEngine + window_hours: float = COOCCURRENCE_WINDOW_HOURS, + cross_domain_weight: float = CROSS_DOMAIN_WEIGHT, + ) -> None: + self.engine = engine + self.window = timedelta(hours=window_hours) + self.weight = cross_domain_weight + # Buffer: list of (node_id, domain, stored_at) + self._buffer: List[Tuple[str, str, datetime]] = [] + + # ---- Domain helpers ------------------------------------------ # + + def tag_domain(self, node: "MemoryNode") -> str: + """Infer and write domain tag to node.metadata. Returns domain string.""" + domain = infer_domain(node.content, getattr(node, "metadata", {})) + if hasattr(node, "metadata"): + node.metadata["domain"] = domain + return domain + + # ---- Synapse creation --------------------------------------- # + + async def _create_synapse(self, id_a: str, id_b: str) -> None: + """ + Request synapse creation between two nodes via the engine's synapse index. + Weight is damped by CROSS_DOMAIN_WEIGHT. + """ + if self.engine is None: + logger.debug(f"CrossDomain: no engine, skipping synapse {id_a[:8]} ↔ {id_b[:8]}") + return + try: + synapse_index = getattr(self.engine, "synapse_index", None) + if synapse_index is not None: + synapse_index.add_or_strengthen(id_a, id_b, delta=self.weight) + logger.debug( + f"CrossDomain synapse created: {id_a[:8]} ↔ {id_b[:8]} weight={self.weight}" + ) + except Exception as exc: + logger.debug(f"CrossDomain synapse creation failed: {exc}") + + # ---- Main API ----------------------------------------------- # + + async def process_new_memory(self, node: "MemoryNode") -> List[Tuple[str, str]]: + """ + Called after a new memory is stored. + Tags its domain and checks for cross-domain co-occurrences in the buffer. + + Returns list of (id_a, id_b) pairs for which synapses were created. + """ + domain = self.tag_domain(node) + now = datetime.now(timezone.utc) + + # Cut stale entries from buffer + cutoff = now - self.window + self._buffer = [(nid, d, ts) for nid, d, ts in self._buffer if ts >= cutoff] + + # Find cross-domain pairs with current node + pairs: List[Tuple[str, str]] = [] + already_seen: Set[str] = set() + for existing_id, existing_domain, _ts in self._buffer: + if existing_domain != domain and existing_id not in already_seen: + await self._create_synapse(node.id, existing_id) + pairs.append((node.id, existing_id)) + already_seen.add(existing_id) + + # Add current node to buffer + self._buffer.append((node.id, domain, now)) + + if pairs: + logger.info( + f"CrossDomain: {len(pairs)} cross-domain synapses created for node {node.id[:8]} (domain={domain})" + ) + return pairs + + async def scan_recent(self, hours: float = 1.0) -> List[Tuple[str, str]]: + """ + Scan the current buffer for any unpaired cross-domain co-occurrences. + Returns all cross-domain pairs. + """ + now = datetime.now(timezone.utc) + cutoff = now - timedelta(hours=hours) + recent = [(nid, d, ts) for nid, d, ts in self._buffer if ts >= cutoff] + + pairs: List[Tuple[str, str]] = [] + n = len(recent) + for i in range(n): + for j in range(i + 1, n): + id_i, dom_i, _ = recent[i] + id_j, dom_j, _ = recent[j] + if dom_i != dom_j: + await self._create_synapse(id_i, id_j) + pairs.append((id_i, id_j)) + + return pairs + + def clear_buffer(self) -> None: + """Reset the co-occurrence buffer.""" + self._buffer.clear() diff --git a/src/mnemocore/core/emotional_tag.py b/src/mnemocore/core/emotional_tag.py new file mode 100644 index 0000000000000000000000000000000000000000..fb2dc081092e2f6c70c48a948e86b738e44dfe8c --- /dev/null +++ b/src/mnemocore/core/emotional_tag.py @@ -0,0 +1,124 @@ +""" +Emotional Tagging Module (Phase 5.0 — Agent 3) +================================================ +Adds valence/arousal emotional metadata to MemoryNode storage. + +Based on affective computing research (Russell's circumplex model): + - emotional_valence: float in [-1.0, 1.0] + -1.0 = extremely negative (fear, grief) + 0.0 = neutral + +1.0 = extremely positive (joy, excitement) + + - emotional_arousal: float in [0.0, 1.0] + 0.0 = calm / low energy + 1.0 = highly activated / intense + +These signals are used by the SubconsciousAI dream cycle to prioritize +consolidation of high-valence, high-arousal memories (the most +biologically significant ones). + +Public API: + tag = EmotionalTag(valence=0.8, arousal=0.9) + meta = tag.to_metadata_dict() + tag_back = EmotionalTag.from_metadata(node.metadata) + score = tag.salience() # combined importance weight +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from .node import MemoryNode + + +# ------------------------------------------------------------------ # +# EmotionalTag # +# ------------------------------------------------------------------ # + +@dataclass +class EmotionalTag: + """ + Two-dimensional emotional metadata for a memory. + + valence ∈ [-1.0, 1.0] (-1 = very negative, +1 = very positive) + arousal ∈ [ 0.0, 1.0] ( 0 = calm, 1 = highly activated) + """ + + valence: float = 0.0 + arousal: float = 0.0 + + def __post_init__(self) -> None: + self.valence = float(max(-1.0, min(1.0, self.valence))) + self.arousal = float(max(0.0, min(1.0, self.arousal))) + + # ---- Salience ------------------------------------------------ # + + def salience(self) -> float: + """ + Combined salience score for dream cycle prioritization. + High |valence| AND high arousal = most memorable / worth consolidating. + + Returns a float in [0.0, 1.0]. + """ + return abs(self.valence) * self.arousal + + def is_emotionally_significant(self, threshold: float = 0.3) -> bool: + """True if the salience is above the given threshold.""" + return self.salience() >= threshold + + # ---- Serialization ------------------------------------------- # + + def to_metadata_dict(self) -> Dict[str, Any]: + return { + "emotional_valence": self.valence, + "emotional_arousal": self.arousal, + "emotional_salience": round(self.salience(), 4), + } + + @classmethod + def from_metadata(cls, metadata: Dict[str, Any]) -> "EmotionalTag": + """Extract an EmotionalTag from a MemoryNode's metadata dict.""" + return cls( + valence=float(metadata.get("emotional_valence", 0.0)), + arousal=float(metadata.get("emotional_arousal", 0.0)), + ) + + @classmethod + def from_node(cls, node: "MemoryNode") -> "EmotionalTag": + """Extract emotional tag directly from a MemoryNode.""" + return cls.from_metadata(getattr(node, "metadata", {})) + + # ---- Helpers -------------------------------------------------- # + + @classmethod + def neutral(cls) -> "EmotionalTag": + return cls(valence=0.0, arousal=0.0) + + @classmethod + def high_positive(cls) -> "EmotionalTag": + """Factory for highly positive, highly aroused tags (e.g. breakthrough).""" + return cls(valence=1.0, arousal=1.0) + + @classmethod + def high_negative(cls) -> "EmotionalTag": + """Factory for highly negative, highly aroused tags (e.g. critical failure).""" + return cls(valence=-1.0, arousal=1.0) + + def __repr__(self) -> str: + return f"EmotionalTag(valence={self.valence:+.2f}, arousal={self.arousal:.2f}, salience={self.salience():.2f})" + + +# ------------------------------------------------------------------ # +# Node helpers # +# ------------------------------------------------------------------ # + +def attach_emotional_tag(node: "MemoryNode", tag: EmotionalTag) -> None: + """Write emotional metadata into node.metadata in place.""" + node.metadata.update(tag.to_metadata_dict()) + + +def get_emotional_tag(node: "MemoryNode") -> EmotionalTag: + """Read the emotional tag from a node's metadata (returns neutral if absent).""" + return EmotionalTag.from_node(node) diff --git a/src/mnemocore/core/engine.py b/src/mnemocore/core/engine.py index 527ca8bac7aa829c49c65b63080cc8b28960cd6a..2e42ea7d1f98afba72264059664d09414d0f243d 100644 --- a/src/mnemocore/core/engine.py +++ b/src/mnemocore/core/engine.py @@ -39,6 +39,11 @@ from .gap_filler import GapFiller, GapFillerConfig from .synapse_index import SynapseIndex from .subconscious_ai import SubconsciousAIWorker +# Phase 5 AGI Stores +from .working_memory import WorkingMemoryService +from .episodic_store import EpisodicStoreService +from .semantic_store import SemanticStoreService + # Phase 4.5: Recursive Synthesis Engine from .recursive_synthesizer import RecursiveSynthesizer, SynthesizerConfig @@ -71,6 +76,9 @@ class HAIMEngine: persist_path: Optional[str] = None, config: Optional[HAIMConfig] = None, tier_manager: Optional[TierManager] = None, + working_memory: Optional[WorkingMemoryService] = None, + episodic_store: Optional[EpisodicStoreService] = None, + semantic_store: Optional[SemanticStoreService] = None, ): """ Initialize HAIMEngine with optional dependency injection. @@ -80,6 +88,9 @@ class HAIMEngine: persist_path: Path to memory persistence file. config: Configuration object. If None, uses global get_config(). tier_manager: TierManager instance. If None, creates a new one. + working_memory: Optional Phase 5 WM service. + episodic_store: Optional Phase 5 EM service. + semantic_store: Optional Phase 5 Semantic service. """ self.config = config or get_config() self.dimension = self.config.dimensionality @@ -89,6 +100,11 @@ class HAIMEngine: # Core Components self.tier_manager = tier_manager or TierManager(config=self.config) + + # Phase 5 Components + self.working_memory = working_memory + self.episodic_store = episodic_store + self.semantic_store = semantic_store self.binary_encoder = TextEncoder(self.dimension) # ── Phase 3.x: synapse raw dicts (kept for backward compat) ── @@ -132,6 +148,23 @@ class HAIMEngine: # ── Phase 4.5: recursive synthesizer ─────────────────────────── self._recursive_synthesizer: Optional[RecursiveSynthesizer] = None + + # ── Phase 12.2: Contextual Topic Tracker ─────────────────────── + from .topic_tracker import TopicTracker + self.topic_tracker = TopicTracker(self.config.context, self.dimension) + + # ── Phase 12.3: Preference Learning ──────────────────────────── + from .preference_store import PreferenceStore + self.preference_store = PreferenceStore(self.config.preference, self.dimension) + + # ── Phase 13.2: Anticipatory Memory ──────────────────────────── + from .anticipatory import AnticipatoryEngine + self.anticipatory_engine = AnticipatoryEngine( + self.config.anticipatory, + self._synapse_index, + self.tier_manager, + self.topic_tracker + ) # Conceptual Layer (VSA Soul) data_dir = self.config.paths.data_dir @@ -337,6 +370,24 @@ class HAIMEngine: """ _is_gap_fill = metadata.get("source") == "llm_gap_fill" + # Phase 12.1: Aggressive Synapse Formation (Auto-bind). + # Fix 4: collect all bindings first, persist synapses only once at the end. + if hasattr(self.config, 'synapse') and self.config.synapse.auto_bind_on_store: + similar_nodes = await self.query( + node.content, + top_k=3, + associative_jump=False, + track_gaps=False, + ) + bind_pairs = [ + (node.id, neighbor_id) + for neighbor_id, similarity in similar_nodes + if neighbor_id != node.id + and similarity >= self.config.synapse.similarity_threshold + ] + if bind_pairs: + await self._auto_bind_batch(bind_pairs) + self.subconscious_queue.append(node.id) if not _is_gap_fill: @@ -346,6 +397,9 @@ class HAIMEngine: # Main store() method - Orchestration only # ========================================================================== + # Maximum allowed content length (Fix 5: input validation) + _MAX_CONTENT_LENGTH: int = 100_000 + @timer(STORE_DURATION_SECONDS, labels={"tier": "hot"}) @traced("store_memory") async def store( @@ -359,20 +413,38 @@ class HAIMEngine: Store new memory with holographic encoding. This method orchestrates the memory storage pipeline: - 1. Encode input content - 2. Evaluate tier placement via EIG - 3. Persist to storage - 4. Trigger post-store processing + 1. Validate input + 2. Encode input content + 3. Evaluate tier placement via EIG + 4. Persist to storage + 5. Trigger post-store processing Args: - content: The text content to store. + content: The text content to store. Must be non-empty and ≤100 000 chars. metadata: Optional metadata dictionary. goal_id: Optional goal identifier for context binding. project_id: Optional project identifier for isolation masking (Phase 4.1). Returns: The unique identifier of the stored memory node. + + Raises: + ValueError: If content is empty or exceeds the maximum allowed length. + RuntimeError: If the engine has not been initialized via initialize(). """ + # Fix 5: Input validation + if not content or not content.strip(): + raise ValueError("Memory content cannot be empty or whitespace-only.") + if len(content) > self._MAX_CONTENT_LENGTH: + raise ValueError( + f"Memory content is too long ({len(content):,} chars). " + f"Maximum: {self._MAX_CONTENT_LENGTH:,}." + ) + if not self._initialized: + raise RuntimeError( + "HAIMEngine.initialize() must be awaited before calling store()." + ) + # 1. Encode input and bind goal context encoded_vec, updated_metadata = await self._encode_input(content, metadata, goal_id) @@ -387,6 +459,35 @@ class HAIMEngine: # 3. Create and persist memory node node = await self._persist_memory(content, encoded_vec, updated_metadata) + # Phase 5.1: If agent_id in metadata, push to Working Memory and log Episode event + agent_id = updated_metadata.get("agent_id") + if agent_id: + if self.working_memory: + from .memory_model import WorkingMemoryItem + self.working_memory.push_item( + agent_id, + WorkingMemoryItem( + id=f"wm_{node.id[:8]}", + agent_id=agent_id, + created_at=datetime.utcnow(), + ttl_seconds=3600, + content=content, + kind="observation", + importance=node.epistemic_value or 0.5, + tags=updated_metadata.get("tags", []), + hdv=encoded_vec + ) + ) + + episode_id = updated_metadata.get("episode_id") + if episode_id and self.episodic_store: + self.episodic_store.append_event( + episode_id=episode_id, + kind="observation", + content=content, + metadata=updated_metadata + ) + # 4. Trigger post-store processing await self._trigger_post_store(node, updated_metadata) @@ -412,24 +513,24 @@ class HAIMEngine: if node_id in self.subconscious_queue: self.subconscious_queue.remove(node_id) - # 3. Phase 4.0: clean up via SynapseIndex (O(k)) + # 3. Phase 4.0: clean up via SynapseIndex (O(k)). + # Fix 2: legacy dict rebuild removed — _synapse_index is authoritative. async with self.synapse_lock: removed_count = self._synapse_index.remove_node(node_id) - # Rebuild legacy dicts - self.synapses = dict(self._synapse_index.items()) - self.synapse_adjacency = {} - for syn in self._synapse_index.values(): - self.synapse_adjacency.setdefault(syn.neuron_a_id, []) - self.synapse_adjacency.setdefault(syn.neuron_b_id, []) - self.synapse_adjacency[syn.neuron_a_id].append(syn) - self.synapse_adjacency[syn.neuron_b_id].append(syn) - if removed_count: await self._save_synapses() return deleted + async def log_decision(self, context_text: str, outcome: float) -> None: + """ + Phase 12.3: Logs a user decision or feedback context to update preference vector. + Outcome should be positive (e.g. 1.0) or negative (e.g. -1.0). + """ + vec = await self._run_in_thread(self.binary_encoder.encode, context_text) + self.preference_store.log_decision(vec, outcome) + async def close(self): """Perform graceful shutdown of engine components.""" logger.info("Shutting down HAIMEngine...") @@ -461,6 +562,8 @@ class HAIMEngine: chrono_weight: bool = True, chrono_lambda: float = 0.0001, include_neighbors: bool = False, + metadata_filter: Optional[Dict[str, Any]] = None, + include_cold: bool = False, ) -> List[Tuple[str, float]]: """ Query memories using Hamming distance. @@ -480,10 +583,18 @@ class HAIMEngine: Formula: Final_Score = Semantic_Similarity * (1 / (1 + lambda * Time_Delta)) - chrono_lambda: Decay rate in seconds^-1 (default: 0.0001 ~ 2.7h half-life). - include_neighbors: Also fetch temporal neighbors (previous/next) for top results. + - include_cold: Include COLD tier in the search (bounded linear scan, default False). + + Fix 3: Triggers anticipatory preloading (Phase 13.2) as fire-and-forget after returning. """ # Encode Query query_vec = await self._run_in_thread(self.binary_encoder.encode, query_text) + # Phase 12.2: Context Tracking + is_shift, sim = self.topic_tracker.add_query(query_vec) + if is_shift: + logger.info(f"Context shifted during query. (sim {sim:.3f})") + # Phase 4.1: Apply project isolation mask to query if project_id: query_vec = self.isolation_masker.apply_mask(query_vec, project_id) @@ -494,6 +605,8 @@ class HAIMEngine: query_vec, top_k=top_k * 2, time_range=time_range, + metadata_filter=metadata_filter, + include_cold=include_cold, ) scores: Dict[str, float] = {} @@ -515,13 +628,38 @@ class HAIMEngine: if chrono_weight and score > 0: mem = mem_map.get(nid) if mem: - time_delta = now_ts - mem.created_at.timestamp() # seconds since creation + time_delta = max(0.0, now_ts - mem.created_at.timestamp()) # seconds since creation # Formula: Final = Semantic * (1 / (1 + lambda * time_delta)) decay_factor = 1.0 / (1.0 + chrono_lambda * time_delta) score = score * decay_factor + # Phase 12.3: Preference Learning Bias + if self.preference_store.config.enabled and self.preference_store.preference_vector is not None: + mem = mem_map.get(nid) + if not mem: + mem = await self.tier_manager.get_memory(nid) + if mem and mem.id not in mem_map: + mem_map[mem.id] = mem + if mem: + score = self.preference_store.bias_score(mem.hdv, score) + scores[nid] = score + # Phase 5.1: Boost context matching Working Memory + agent_id = metadata_filter.get("agent_id") if metadata_filter else None + if agent_id and self.working_memory: + wm_state = self.working_memory.get_state(agent_id) + if wm_state: + wm_texts = [item.content for item in wm_state.items] + if wm_texts: + # Very lightweight lexical boost for items currently in working memory + q_lower = query_text.lower() + for nid in scores: + mem = mem_map.get(nid) # Assuming already cached from chrono weighting + if mem and mem.content: + if any(w_text.lower() in mem.content.lower() for w_text in wm_texts): + scores[nid] *= 1.15 # 15% boost for WM overlap + # 2. Associative Spreading (via SynapseIndex for O(1) adjacency lookup) if associative_jump and self._synapse_index: top_seeds = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:3] @@ -540,6 +678,15 @@ class HAIMEngine: if neighbor not in augmented_scores: mem = await self.tier_manager.get_memory(neighbor) if mem: + if metadata_filter: + match = True + node_meta = mem.metadata or {} + for k, v in metadata_filter.items(): + if node_meta.get(k) != v: + match = False + break + if not match: + continue augmented_scores[neighbor] = query_vec.similarity(mem.hdv) if neighbor in augmented_scores: @@ -595,6 +742,15 @@ class HAIMEngine: if mem.previous_id: prev_mem = await self.tier_manager.get_memory(mem.previous_id) if prev_mem and prev_mem.id not in scores: + if metadata_filter: + match = True + p_meta = prev_mem.metadata or {} + for k, v in metadata_filter.items(): + if p_meta.get(k) != v: + match = False + break + if not match: + continue neighbor_ids.add(prev_mem.id) # Try to find the memory that follows this one (has this as previous_id). @@ -614,8 +770,35 @@ class HAIMEngine: # Re-sort after adding neighbors, but preserve query() top_k contract. top_results = sorted(top_results, key=lambda x: x[1], reverse=True)[:top_k] + # Phase 13.2 (Fix 3): Anticipatory preloading — fire-and-forget so it + # never blocks the caller. Only activated when the engine is fully warm. + if top_results and self._initialized and self.config.anticipatory.enabled: + asyncio.ensure_future( + self.anticipatory_engine.predict_and_preload(top_results[0][0]) + ) + return top_results + async def get_context_nodes(self, top_k: int = 3) -> List[Tuple[str, float]]: + """ + Phase 12.2: Contextual Awareness + Retrieves the top_k most relevant nodes relating to the current topic context vector. + Should be explicitly used by prompt builders before LLM logic injection. + """ + if not self.topic_tracker.config.enabled: + return [] + + ctx = self.topic_tracker.get_context() + if ctx is None: + return [] + + results = await self.tier_manager.search( + ctx, + top_k=top_k, + time_range=None, + metadata_filter=None, + ) + return results async def _background_dream(self, depth: int = 2): """ Passive Subconscious – strengthen synapses in idle cycles. @@ -668,12 +851,32 @@ class HAIMEngine: return sorted(active_nodes, key=score, reverse=True)[:max_collapse] - async def bind_memories(self, id_a: str, id_b: str, success: bool = True): + async def _auto_bind_batch( + self, + pairs: List[Tuple[str, str]], + success: bool = True, + weight: float = 1.0, + ) -> None: + """ + Fix 4: Bind multiple (id_a, id_b) pairs in one pass, saving synapses once. + + Used by auto-bind in _trigger_post_store() to avoid N disk writes per store. + """ + async with self.synapse_lock: + for id_a, id_b in pairs: + mem_a = await self.tier_manager.get_memory(id_a) + mem_b = await self.tier_manager.get_memory(id_b) + if mem_a and mem_b: + self._synapse_index.add_or_fire(id_a, id_b, success=success, weight=weight) + await self._save_synapses() + + async def bind_memories(self, id_a: str, id_b: str, success: bool = True, weight: float = 1.0): """ Bind two memories by ID. - Phase 4.0: delegates to SynapseIndex for O(1) insert/fire. - Also syncs legacy dicts for backward-compat. + Fix 2: delegates exclusively to SynapseIndex — legacy dict sync removed. + The legacy self.synapses / self.synapse_adjacency attributes remain for + backward compatibility but are only populated at startup from disk. """ mem_a = await self.tier_manager.get_memory(id_a) mem_b = await self.tier_manager.get_memory(id_b) @@ -682,17 +885,7 @@ class HAIMEngine: return async with self.synapse_lock: - syn = self._synapse_index.add_or_fire(id_a, id_b, success=success) - - # Keep legacy dict in sync for any external code still using it - synapse_key = tuple(sorted([id_a, id_b])) - self.synapses[synapse_key] = syn - self.synapse_adjacency.setdefault(synapse_key[0], []) - self.synapse_adjacency.setdefault(synapse_key[1], []) - if syn not in self.synapse_adjacency[synapse_key[0]]: - self.synapse_adjacency[synapse_key[0]].append(syn) - if syn not in self.synapse_adjacency[synapse_key[1]]: - self.synapse_adjacency[synapse_key[1]].append(syn) + self._synapse_index.add_or_fire(id_a, id_b, success=success, weight=weight) await self._save_synapses() @@ -712,26 +905,17 @@ class HAIMEngine: Also syncs any legacy dict entries into the index before compacting. """ async with self.synapse_lock: - # Sync legacy dict → SynapseIndex via the public register() API - # (handles tests / external code that injects into self.synapses directly) - for key, syn in list(self.synapses.items()): + # Retain legacy→index sync so tests that write to self.synapses directly + # still get their entries registered (Fix 2: sync only in this direction). + for syn in list(self.synapses.values()): if self._synapse_index.get(syn.neuron_a_id, syn.neuron_b_id) is None: self._synapse_index.register(syn) removed = self._synapse_index.compact(threshold) - if removed: - # Rebuild legacy dicts from the index - self.synapses = dict(self._synapse_index.items()) - self.synapse_adjacency = {} - for syn in self._synapse_index.values(): - self.synapse_adjacency.setdefault(syn.neuron_a_id, []) - self.synapse_adjacency.setdefault(syn.neuron_b_id, []) - self.synapse_adjacency[syn.neuron_a_id].append(syn) - self.synapse_adjacency[syn.neuron_b_id].append(syn) - - logger.info(f"cleanup_decay: pruned {removed} synapses below {threshold}") - await self._save_synapses() + if removed: + logger.info(f"cleanup_decay: pruned {removed} synapses below {threshold}") + await self._save_synapses() async def get_stats(self) -> Dict[str, Any]: """Aggregate statistics from engine components.""" @@ -936,18 +1120,9 @@ class HAIMEngine: def _load(): self._synapse_index.load_from_file(self.synapse_path) + # Fix 2: _synapse_index is authoritative — legacy dicts no longer rebuilt. await self._run_in_thread(_load) - # Rebuild legacy dicts from SynapseIndex for backward compat - async with self.synapse_lock: - self.synapses = dict(self._synapse_index.items()) - self.synapse_adjacency = {} - for syn in self._synapse_index.values(): - self.synapse_adjacency.setdefault(syn.neuron_a_id, []) - self.synapse_adjacency.setdefault(syn.neuron_b_id, []) - self.synapse_adjacency[syn.neuron_a_id].append(syn) - self.synapse_adjacency[syn.neuron_b_id].append(syn) - async def _save_synapses(self): """ Save synapses to disk in JSONL format. diff --git a/src/mnemocore/core/episodic_store.py b/src/mnemocore/core/episodic_store.py new file mode 100644 index 0000000000000000000000000000000000000000..118c38a30729aace196f880ddf60aaf8f170f802 --- /dev/null +++ b/src/mnemocore/core/episodic_store.py @@ -0,0 +1,144 @@ +""" +Episodic Store Service +====================== +Manages sequences of events (Episodes), chaining them chronologically. +Provides the foundation for episodic recall and narrative tracking over time. +""" + +from typing import Dict, List, Optional, Any +from datetime import datetime +import threading +import uuid +import logging + +from .memory_model import Episode, EpisodeEvent +from .tier_manager import TierManager + +logger = logging.getLogger(__name__) + + +class EpisodicStoreService: + def __init__(self, tier_manager: Optional[TierManager] = None): + self._tier_manager = tier_manager + # In-memory index of active episodes; eventually backed by SQLite/Qdrant + self._active_episodes: Dict[str, Episode] = {} + # Simple backward index map from agent to sorted list of historical episodes + self._agent_history: Dict[str, List[Episode]] = {} + self._lock = threading.RLock() + + def start_episode( + self, agent_id: str, goal: Optional[str] = None, context: Optional[str] = None + ) -> str: + with self._lock: + ep_id = f"ep_{uuid.uuid4().hex[:12]}" + + # Find previous absolute episode for this agent to populate links_prev + prev_links = [] + if agent_id in self._agent_history and self._agent_history[agent_id]: + last_ep = self._agent_history[agent_id][-1] + prev_links.append(last_ep.id) + + new_ep = Episode( + id=ep_id, + agent_id=agent_id, + started_at=datetime.utcnow(), + ended_at=None, + goal=goal, + context=context, + events=[], + outcome="in_progress", + reward=None, + links_prev=prev_links, + links_next=[], + ltp_strength=0.0, + reliability=1.0, + ) + + # Link the previous episode forward + if prev_links: + last_ep_id = prev_links[0] + last_ep = self._get_historical_ep(agent_id, last_ep_id) + if last_ep and new_ep.id not in last_ep.links_next: + last_ep.links_next.append(new_ep.id) + + self._active_episodes[ep_id] = new_ep + return ep_id + + def append_event( + self, + episode_id: str, + kind: str, + content: str, + metadata: Optional[dict[str, Any]] = None, + ) -> None: + with self._lock: + ep = self._active_episodes.get(episode_id) + if not ep: + logger.warning(f"Attempted to append event to inactive or not found episode: {episode_id}") + return + + event = EpisodeEvent( + timestamp=datetime.utcnow(), + kind=kind, # type: ignore + content=content, + metadata=metadata or {}, + ) + ep.events.append(event) + + def end_episode( + self, episode_id: str, outcome: str, reward: Optional[float] = None + ) -> None: + with self._lock: + ep = self._active_episodes.pop(episode_id, None) + if not ep: + logger.warning(f"Attempted to end inactive or not found episode: {episode_id}") + return + + ep.ended_at = datetime.utcnow() + ep.outcome = outcome # type: ignore + ep.reward = reward + + agent_history = self._agent_history.setdefault(ep.agent_id, []) + agent_history.append(ep) + + # Sort by start time just to ensure chronological order is preserved + agent_history.sort(key=lambda x: x.started_at) + + logger.debug(f"Ended episode {episode_id} with outcome {outcome}") + + def get_episode(self, episode_id: str) -> Optional[Episode]: + with self._lock: + # Check active first + if episode_id in self._active_episodes: + return self._active_episodes[episode_id] + # Then check history + for history in self._agent_history.values(): + for ep in history: + if ep.id == episode_id: + return ep + return None + + def get_recent( + self, agent_id: str, limit: int = 5, context: Optional[str] = None + ) -> List[Episode]: + with self._lock: + history = self._agent_history.get(agent_id, []) + + # Active episodes count too + active = [ep for ep in self._active_episodes.values() if ep.agent_id == agent_id] + + combined = history + active + combined.sort(key=lambda x: x.started_at, reverse=True) + + if context: + combined = [ep for ep in combined if ep.context == context] + + return combined[:limit] + + def _get_historical_ep(self, agent_id: str, episode_id: str) -> Optional[Episode]: + history = self._agent_history.get(agent_id, []) + for ep in history: + if ep.id == episode_id: + return ep + return None + diff --git a/src/mnemocore/core/forgetting_curve.py b/src/mnemocore/core/forgetting_curve.py new file mode 100644 index 0000000000000000000000000000000000000000..f9a54255bca35fa0d88b822be7c2e048b965668a --- /dev/null +++ b/src/mnemocore/core/forgetting_curve.py @@ -0,0 +1,233 @@ +""" +Forgetting Curve Manager (Phase 5.0) +===================================== +Implements Ebbinghaus-based spaced repetition scheduling for MnemoCore. + +The ForgettingCurveManager layers on top of AdaptiveDecayEngine to: + 1. Schedule "review" events at optimal intervals (spaced repetition) + 2. Decide whether low-retention memories should be consolidated vs. deleted + 3. Work collaboratively with the ConsolidationWorker + +Key idea: at each review interval, the system re-evaluates a memory's EIG. + - High EIG + low retention → CONSOLIDATE (absorb into a stronger anchor) + - Low EIG + low retention → ARCHIVE / EVICT + +The review scheduling uses the SuperMemo-inspired interval: + next_review_days = S_i * ln(1 / TARGET_RETENTION)^-1 + +where TARGET_RETENTION = 0.70 (retain 70% at next review point). + +Public API: + manager = ForgettingCurveManager(engine) + await manager.run_once(nodes) # scan HOT/WARM nodes, schedule reviews + schedule = manager.get_schedule() # sorted list of upcoming reviews +""" + +from __future__ import annotations + +import asyncio +import math +from dataclasses import dataclass, field +from datetime import datetime, timezone, timedelta +from typing import TYPE_CHECKING, Dict, List, Optional + +from loguru import logger + +from .temporal_decay import AdaptiveDecayEngine, get_adaptive_decay_engine + +if TYPE_CHECKING: + from .node import MemoryNode + + +# ------------------------------------------------------------------ # +# Constants # +# ------------------------------------------------------------------ # + +TARGET_RETENTION: float = 0.70 # Retention level at which we schedule the next review +MIN_EIG_TO_CONSOLIDATE: float = 0.3 # Minimum epistemic value to consolidate instead of evict + + +# ------------------------------------------------------------------ # +# Review Schedule Entry # +# ------------------------------------------------------------------ # + +@dataclass +class ReviewEntry: + """A scheduled review for a single memory.""" + memory_id: str + due_at: datetime # When to review + current_retention: float # Retention at scheduling time + stability: float # S_i at scheduling time + action: str = "review" # "review" | "consolidate" | "evict" + + def to_dict(self) -> Dict: + return { + "memory_id": self.memory_id, + "due_at": self.due_at.isoformat(), + "current_retention": round(self.current_retention, 4), + "stability": round(self.stability, 4), + "action": self.action, + } + + +# ------------------------------------------------------------------ # +# Forgetting Curve Manager # +# ------------------------------------------------------------------ # + +class ForgettingCurveManager: + """ + Schedules spaced-repetition review events for MemoryNodes. + + Attach to a running HAIMEngine to enable automatic review scheduling. + Works in concert with AdaptiveDecayEngine and ConsolidationWorker. + """ + + def __init__( + self, + engine=None, # HAIMEngine – typed as Any to avoid circular import + decay_engine: Optional[AdaptiveDecayEngine] = None, + target_retention: float = TARGET_RETENTION, + min_eig_to_consolidate: float = MIN_EIG_TO_CONSOLIDATE, + ) -> None: + self.engine = engine + self.decay = decay_engine or get_adaptive_decay_engine() + self.target_retention = target_retention + self.min_eig_to_consolidate = min_eig_to_consolidate + self._schedule: List[ReviewEntry] = [] + + # ---- Interval calculation ------------------------------------ # + + def next_review_days(self, node: "MemoryNode") -> float: + """ + Days until the next review should be scheduled. + + Derived from: TARGET_RETENTION = e^(-next_days / S_i) + → next_days = -S_i * ln(TARGET_RETENTION) + + Example: S_i=5, target=0.70 → next_days = 5 × 0.357 ≈ 1.78 days + """ + s_i = self.decay.stability(node) + # Protect against math domain errors + target = max(1e-6, min(self.target_retention, 0.999)) + return -s_i * math.log(target) + + def _determine_action(self, node: "MemoryNode", retention: float) -> str: + """ + Decide what to do with a low-retention memory: + - consolidate: has historical importance (epistemic_value > threshold) + - evict: low value, low retention + - review: needs attention but not critical yet + """ + if self.decay.should_evict(node): + eig = getattr(node, "epistemic_value", 0.0) + if eig >= self.min_eig_to_consolidate: + return "consolidate" + return "evict" + return "review" + + # ---- Scan and schedule --------------------------------------- # + + def schedule_reviews(self, nodes: "List[MemoryNode]") -> List[ReviewEntry]: + """ + Scan the provided nodes and build a schedule of upcoming reviews. + Nodes with retention ≤ REVIEW_THRESHOLD are immediately flagged. + + Returns the new ReviewEntry objects added to the schedule. + """ + now = datetime.now(timezone.utc) + new_entries: List[ReviewEntry] = [] + + for node in nodes: + retention = self.decay.retention(node) + s_i = self.decay.stability(node) + + # Always update review_candidate flag on the node itself + self.decay.update_review_candidate(node) + + # Schedule next review based on spaced repetition interval + days_until = self.next_review_days(node) + due_at = now + timedelta(days=days_until) + action = self._determine_action(node, retention) + + entry = ReviewEntry( + memory_id=node.id, + due_at=due_at, + current_retention=retention, + stability=s_i, + action=action, + ) + new_entries.append(entry) + + # Merge into the schedule (replace existing entries for same memory_id) + existing_ids = {e.memory_id for e in self._schedule} + self._schedule = [ + e for e in self._schedule if e.memory_id not in {n.id for n in nodes} + ] + self._schedule.extend(new_entries) + self._schedule.sort(key=lambda e: e.due_at) + + logger.info( + f"ForgettingCurveManager: scheduled {len(new_entries)} reviews for {len(nodes)} nodes. " + f"Total scheduled: {len(self._schedule)}" + ) + return new_entries + + def get_schedule(self) -> List[ReviewEntry]: + """Return the current review schedule sorted by due_at.""" + return sorted(self._schedule, key=lambda e: e.due_at) + + def get_due_reviews(self) -> List[ReviewEntry]: + """Return entries that are due now (due_at <= now).""" + now = datetime.now(timezone.utc) + return [e for e in self._schedule if e.due_at <= now] + + def get_actions_by_type(self, action: str) -> List[ReviewEntry]: + """Filter schedule by action type: 'review', 'consolidate', or 'evict'.""" + return [e for e in self._schedule if e.action == action] + + def remove_entry(self, memory_id: str) -> None: + """Remove a memory from the review schedule (e.g., it was evicted).""" + self._schedule = [e for e in self._schedule if e.memory_id != memory_id] + + # ---- Engine integration ------------------------------------- # + + async def run_once(self) -> Dict: + """ + Run a full scan over HOT + WARM nodes and update the review schedule. + + Returns a stats dict with counts per action. + """ + if self.engine is None: + logger.warning("ForgettingCurveManager: no engine attached, cannot scan tiers.") + return {} + + nodes: List["MemoryNode"] = [] + try: + hot = await self.engine.tier_manager.get_hot_snapshot() + nodes.extend(hot) + except Exception as e: + logger.warning(f"ForgettingCurveManager: could not fetch HOT nodes: {e}") + + try: + warm = await self.engine.tier_manager.list_warm(max_results=1000) + nodes.extend(warm) + except (AttributeError, Exception) as e: + logger.debug(f"ForgettingCurveManager: WARM fetch skipped: {e}") + + entries = self.schedule_reviews(nodes) + + # Count actions + from collections import Counter + action_counts = dict(Counter(e.action for e in entries)) + + logger.info(f"ForgettingCurveManager scan: {action_counts}") + return { + "nodes_scanned": len(nodes), + "entries_scheduled": len(entries), + "action_counts": action_counts, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + +# Convenience import alias +from typing import Dict # noqa: E402 (already imported above, just ensuring type hint works) diff --git a/src/mnemocore/core/hnsw_index.py b/src/mnemocore/core/hnsw_index.py index 1d49accfc1ea31f4abaf13c975ee509ed2becc54..e0c05903fe0067c4670872c8a1d53c5ae234f3f4 100644 --- a/src/mnemocore/core/hnsw_index.py +++ b/src/mnemocore/core/hnsw_index.py @@ -50,18 +50,32 @@ FLAT_THRESHOLD: int = 256 # use flat index below this hop count # HNSW Index Manager # # ------------------------------------------------------------------ # +import json +from pathlib import Path +from threading import Lock +from .config import get_config + class HNSWIndexManager: """ Manages a FAISS HNSW binary ANN index for the HOT tier. + Thread-safe singleton with disk persistence. Automatically switches between: - IndexBinaryFlat (N < FLAT_THRESHOLD — exact, faster for small N) - IndexBinaryHNSW (N ≥ FLAT_THRESHOLD — approx, faster for large N) - - The index is rebuilt from scratch when switching modes (rare operation). - All operations are synchronous (called from within asyncio.Lock context). """ + _instance: "HNSWIndexManager | None" = None + _singleton_lock: Lock = Lock() + + def __new__(cls, *args, **kwargs) -> "HNSWIndexManager": + with cls._singleton_lock: + if cls._instance is None: + obj = super().__new__(cls) + obj._initialized = False + cls._instance = obj + return cls._instance + def __init__( self, dimension: int = 16384, @@ -69,52 +83,70 @@ class HNSWIndexManager: ef_construction: int = DEFAULT_EF_CONSTRUCTION, ef_search: int = DEFAULT_EF_SEARCH, ): + if getattr(self, "_initialized", False): + return + self.dimension = dimension self.m = m self.ef_construction = ef_construction self.ef_search = ef_search - # ID maps - self._id_map: Dict[int, str] = {} # faiss_int_id → node_id - self._node_map: Dict[str, int] = {} # node_id → faiss_int_id - self._next_id: int = 1 - self._use_hnsw: bool = False + self._write_lock = Lock() - # FAISS index (initialised below) + self._id_map: List[Optional[str]] = [] + self._vector_store: List[np.ndarray] = [] + self._use_hnsw = False + self._stale_count = 0 self._index = None + + config = get_config() + data_dir = Path(config.paths.data_dir if hasattr(config, 'paths') else "./data") + data_dir.mkdir(parents=True, exist_ok=True) + + self.INDEX_PATH = data_dir / "mnemocore_hnsw.faiss" + self.IDMAP_PATH = data_dir / "mnemocore_hnsw_idmap.json" + self.VECTOR_PATH = data_dir / "mnemocore_hnsw_vectors.npy" if FAISS_AVAILABLE: - self._build_flat_index() - else: - logger.warning("HNSWIndexManager running WITHOUT faiss — linear fallback only.") + if self.INDEX_PATH.exists() and self.IDMAP_PATH.exists() and self.VECTOR_PATH.exists(): + self._load() + else: + self._build_flat_index() + + self._initialized = True # ---- Index construction -------------------------------------- # def _build_flat_index(self) -> None: """Create a fresh IndexBinaryFlat (exact Hamming ANN).""" - base = faiss.IndexBinaryFlat(self.dimension) - self._index = faiss.IndexBinaryIDMap(base) + self._index = faiss.IndexBinaryFlat(self.dimension) self._use_hnsw = False logger.debug(f"Built FAISS flat binary index (dim={self.dimension})") - def _build_hnsw_index(self, existing_nodes: Optional[List[Tuple[int, np.ndarray]]] = None) -> None: + def _build_hnsw_index(self) -> None: """ Build an HNSW binary index and optionally re-populate with existing vectors. - - Note: FAISS IndexBinaryHNSW does NOT support IDMap natively, so we use a - custom double-mapping approach: HNSW indices map 1-to-1 to our _id_map. - We rebuild as IndexBinaryHNSW and re-add all existing vectors. """ hnsw = faiss.IndexBinaryHNSW(self.dimension, self.m) hnsw.hnsw.efConstruction = self.ef_construction hnsw.hnsw.efSearch = self.ef_search - if existing_nodes: - # Batch add in order of faiss_int_id so positions are deterministic - existing_nodes.sort(key=lambda x: x[0]) - vecs = np.stack([v for _, v in existing_nodes]) - hnsw.add(vecs) - logger.debug(f"HNSW index rebuilt with {len(existing_nodes)} existing vectors") + if self._vector_store: + # Compact the index to remove None entries + compact_ids = [] + compact_vecs = [] + for i, node_id in enumerate(self._id_map): + if node_id is not None: + compact_ids.append(node_id) + compact_vecs.append(self._vector_store[i]) + + if compact_vecs: + vecs = np.stack(compact_vecs) + hnsw.add(vecs) + + self._id_map = compact_ids + self._vector_store = compact_vecs + self._stale_count = 0 self._index = hnsw self._use_hnsw = True @@ -125,43 +157,18 @@ class HNSWIndexManager: def _maybe_upgrade_to_hnsw(self) -> None: """Upgrade to HNSW index if HOT tier has grown large enough.""" - if not FAISS_AVAILABLE: - return - if self._use_hnsw: + if not FAISS_AVAILABLE or self._use_hnsw: return - if len(self._id_map) < FLAT_THRESHOLD: + active_count = len(self._id_map) - self._stale_count + if active_count < FLAT_THRESHOLD: return logger.info( - f"HOT tier size ({len(self._id_map)}) ≥ threshold ({FLAT_THRESHOLD}) " + f"HOT tier size ({active_count}) ≥ threshold ({FLAT_THRESHOLD}) " "— upgrading to HNSW index." ) - # NOTE: For HNSW without IDMap we maintain position-based mapping. - # We rebuild from the current flat index contents. - # Collect all existing (local_pos → node_vector) pairs. - # - # For simplicity in this transition we do a full rebuild from scratch: - # the upgrade happens at most once per process lifetime (HOT usually stays - # under threshold or once it crosses, it stays crossed). - existing: List[Tuple[int, np.ndarray]] = [] - for fid, node_id in self._id_map.items(): - # We can't reconstruct vectors from IndexBinaryIDMap cheaply, - # so we store them in a shadow cache while using the flat index. - if node_id in self._vector_cache: - existing.append((fid, self._vector_cache[node_id])) - - self._build_hnsw_index(existing) - - # ---- Vector shadow cache (needed for HNSW rebuild) ----------- # - # HNSW indices don't support IDMap; we cache raw vectors separately - # so we can rebuild on threshold-crossing. - - @property - def _vector_cache(self) -> Dict[str, np.ndarray]: - if not hasattr(self, "_vcache"): - object.__setattr__(self, "_vcache", {}) - return self._vcache # type: ignore[attr-defined] + self._build_hnsw_index() # ---- Public API --------------------------------------------- # @@ -176,77 +183,66 @@ class HNSWIndexManager: if not FAISS_AVAILABLE or self._index is None: return - fid = self._next_id - self._next_id += 1 - self._id_map[fid] = node_id - self._node_map[node_id] = fid - self._vector_cache[node_id] = hdv_data.copy() - - vec = np.expand_dims(hdv_data, axis=0) + vec = np.ascontiguousarray(np.expand_dims(hdv_data, axis=0)) - try: - if self._use_hnsw: - # HNSW.add() — position is implicit (sequential) + with self._write_lock: + try: self._index.add(vec) - else: - ids = np.array([fid], dtype="int64") - self._index.add_with_ids(vec, ids) - except Exception as exc: - logger.error(f"HNSW/FAISS add failed for {node_id}: {exc}") - return + self._id_map.append(node_id) + self._vector_store.append(hdv_data.copy()) + except Exception as exc: + logger.error(f"HNSW/FAISS add failed for {node_id}: {repr(exc)}") + return - # Check if we should upgrade to HNSW - self._maybe_upgrade_to_hnsw() + self._maybe_upgrade_to_hnsw() + self._save() def remove(self, node_id: str) -> None: """ Remove a node from the index. - - For HNSW (no IDMap), we mark the node as deleted in our bookkeeping - and rebuild the index lazily when the deletion rate exceeds 20%. + Marks node as deleted and rebuilds index lazily when the deletion rate exceeds 20%. """ if not FAISS_AVAILABLE or self._index is None: return - fid = self._node_map.pop(node_id, None) - if fid is None: - return - - self._id_map.pop(fid, None) - self._vector_cache.pop(node_id, None) - - if not self._use_hnsw: + with self._write_lock: try: - ids = np.array([fid], dtype="int64") - self._index.remove_ids(ids) - except Exception as exc: - logger.error(f"FAISS flat remove failed for {node_id}: {exc}") - else: - # HNSW doesn't support removal; track stale fraction and rebuild when needed - if not hasattr(self, "_stale_count"): - object.__setattr__(self, "_stale_count", 0) - self._stale_count += 1 # type: ignore[attr-defined] - - total = max(len(self._id_map) + self._stale_count, 1) - stale_fraction = self._stale_count / total - if stale_fraction > 0.20 and len(self._id_map) > 0: - logger.info(f"HNSW stale fraction {stale_fraction:.1%} — rebuilding index.") - existing = [ - (fid2, self._vector_cache[nid]) - for fid2, nid in self._id_map.items() - if nid in self._vector_cache - ] - self._build_hnsw_index(existing) - self._stale_count = 0 + fid = self._id_map.index(node_id) + self._id_map[fid] = None + self._stale_count += 1 + + total = max(len(self._id_map), 1) + stale_fraction = self._stale_count / total + + if stale_fraction > 0.20 and len(self._id_map) > 0: + logger.info(f"HNSW stale fraction {stale_fraction:.1%} — rebuilding index.") + if self._use_hnsw: + self._build_hnsw_index() + else: + self._build_flat_index() + if self._vector_store: + compact_ids = [] + compact_vecs = [] + for i, nid in enumerate(self._id_map): + if nid is not None: + compact_ids.append(nid) + compact_vecs.append(self._vector_store[i]) + if compact_vecs: + vecs = np.ascontiguousarray(np.stack(compact_vecs)) + self._index.add(vecs) + self._id_map = compact_ids + self._vector_store = compact_vecs + self._stale_count = 0 + + self._save() + except ValueError: + pass + def search(self, query_data: np.ndarray, top_k: int = 10) -> List[Tuple[str, float]]: """ Search for top-k nearest neighbours. - Args: - query_data: Packed uint8 query array (D/8 bytes). - top_k: Number of results to return. - Returns: List of (node_id, similarity_score) sorted by descending similarity. similarity = 1 - normalised_hamming_distance ∈ [0, 1]. @@ -254,8 +250,25 @@ class HNSWIndexManager: if not FAISS_AVAILABLE or self._index is None or not self._id_map: return [] - k = min(top_k, len(self._id_map)) - q = np.expand_dims(query_data, axis=0) + # Fetch more to account for deleted (None) entries + k = min(top_k + self._stale_count, len(self._id_map)) + if k <= 0: + return [] + + index_dimension = int(getattr(self._index, "d", self.dimension) or self.dimension) + query_bytes = np.ascontiguousarray(query_data, dtype=np.uint8).reshape(-1) + expected_bytes = index_dimension // 8 + if expected_bytes > 0 and query_bytes.size != expected_bytes: + logger.warning( + f"HNSW query dimension mismatch: index={index_dimension} bits ({expected_bytes} bytes), " + f"query={query_bytes.size} bytes. Adjusting query to index dimension." + ) + if query_bytes.size > expected_bytes: + query_bytes = query_bytes[:expected_bytes] + else: + query_bytes = np.pad(query_bytes, (0, expected_bytes - query_bytes.size), mode="constant") + + q = np.expand_dims(query_bytes, axis=0) try: distances, ids = self._index.search(q, k) @@ -265,43 +278,59 @@ class HNSWIndexManager: results: List[Tuple[str, float]] = [] for dist, idx in zip(distances[0], ids[0]): - if idx == -1: + if idx < 0 or idx >= len(self._id_map): continue - - if self._use_hnsw: - # HNSW returns 0-based position indices; map back through insertion order - node_id = self._position_to_node_id(int(idx)) - else: - node_id = self._id_map.get(int(idx)) - - if node_id: - sim = 1.0 - float(dist) / self.dimension + + node_id = self._id_map[idx] + if node_id is not None: + sim = 1.0 - float(dist) / max(index_dimension, 1) + sim = float(np.clip(sim, 0.0, 1.0)) results.append((node_id, sim)) + if len(results) >= top_k: + break return results - def _position_to_node_id(self, position: int) -> Optional[str]: - """ - Map HNSW sequential position back to node_id. - Positions correspond to insertion order; we track this via _position_map. - """ - if not hasattr(self, "_position_map"): - object.__setattr__(self, "_position_map", {}) - pm: Dict[int, str] = self._position_map # type: ignore[attr-defined] - - # Rebuild position map if needed (after index rebuild) - if len(pm) < len(self._id_map): - pm.clear() - for pos, (fid, nid) in enumerate( - sorted(self._id_map.items(), key=lambda x: x[0]) - ): - pm[pos] = nid - - return pm.get(position) + def _save(self): + try: + faiss.write_index_binary(self._index, str(self.INDEX_PATH)) + with open(self.IDMAP_PATH, "w") as f: + json.dump({ + "id_map": self._id_map, + "use_hnsw": self._use_hnsw, + "stale_count": self._stale_count + }, f) + if self._vector_store: + np.save(str(self.VECTOR_PATH), np.stack(self._vector_store)) + except Exception as e: + logger.error(f"Failed to save HNSW index state: {e}") + + def _load(self): + try: + self._index = faiss.read_index_binary(str(self.INDEX_PATH)) + index_dimension = int(getattr(self._index, "d", self.dimension) or self.dimension) + if index_dimension != self.dimension: + logger.warning( + f"HNSW index dimension mismatch on load: config={self.dimension}, index={index_dimension}. " + "Using index dimension." + ) + self.dimension = index_dimension + with open(self.IDMAP_PATH, "r") as f: + state = json.load(f) + self._id_map = state.get("id_map", []) + self._use_hnsw = state.get("use_hnsw", False) + self._stale_count = state.get("stale_count", 0) + + vecs = np.load(str(self.VECTOR_PATH)) + self._vector_store = list(vecs) + logger.info("Loaded HNSW persistent state from disk") + except Exception as e: + logger.error(f"Failed to load HNSW index state: {e}") + self._build_flat_index() @property def size(self) -> int: - return len(self._id_map) + return len([x for x in self._id_map if x is not None]) @property def index_type(self) -> str: @@ -318,4 +347,5 @@ class HNSWIndexManager: "ef_construction": self.ef_construction if self._use_hnsw else None, "ef_search": self.ef_search if self._use_hnsw else None, "faiss_available": FAISS_AVAILABLE, + "stale_count": self._stale_count } diff --git a/src/mnemocore/core/memory_model.py b/src/mnemocore/core/memory_model.py new file mode 100644 index 0000000000000000000000000000000000000000..22d332544b7cfac22c3b71549520897387577bdc --- /dev/null +++ b/src/mnemocore/core/memory_model.py @@ -0,0 +1,132 @@ +""" +Memory Models +============= +Data classes mapping the Cognitive Architecture Phase 5 entities: +Working Memory (WM), Episodic Memory (EM), Semantic Memory (SM), +Procedural Memory (PM), and Meta-Memory (MM). +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Literal, Optional, List + +from .binary_hdv import BinaryHDV + + +# --- Working Memory (WM) --- + +@dataclass +class WorkingMemoryItem: + id: str + agent_id: str + created_at: datetime + ttl_seconds: int + content: str + kind: Literal["thought", "observation", "goal", "plan_step", "action", "meta"] + importance: float + tags: List[str] + hdv: Optional[BinaryHDV] = None + + +@dataclass +class WorkingMemoryState: + agent_id: str + max_items: int + items: List[WorkingMemoryItem] = field(default_factory=list) + + +# --- Episodic Memory (EM) --- + +@dataclass +class EpisodeEvent: + timestamp: datetime + kind: Literal["observation", "action", "thought", "reward", "error", "system"] + content: str + metadata: dict[str, Any] + hdv: Optional[BinaryHDV] = None + + +@dataclass +class Episode: + id: str + agent_id: str + started_at: datetime + ended_at: Optional[datetime] + goal: Optional[str] + context: Optional[str] + events: List[EpisodeEvent] + outcome: Literal["success", "failure", "partial", "unknown", "in_progress"] + reward: Optional[float] + links_prev: List[str] + links_next: List[str] + ltp_strength: float + reliability: float + + @property + def is_active(self) -> bool: + return self.ended_at is None + + +# --- Semantic Memory (SM) --- + +@dataclass +class SemanticConcept: + id: str + label: str + description: str + tags: List[str] + prototype_hdv: BinaryHDV + support_episode_ids: List[str] + reliability: float + last_updated_at: datetime + metadata: dict[str, Any] + + +# --- Procedural Memory (PM) --- + +@dataclass +class ProcedureStep: + order: int + instruction: str + code_snippet: Optional[str] = None + tool_call: Optional[dict[str, Any]] = None + + +@dataclass +class Procedure: + id: str + name: str + description: str + created_by_agent: Optional[str] + created_at: datetime + updated_at: datetime + steps: List[ProcedureStep] + trigger_pattern: str + success_count: int + failure_count: int + reliability: float + tags: List[str] + + +# --- Meta-Memory (MM) --- + +@dataclass +class SelfMetric: + name: str + value: float + window: str # e.g. "5m", "1h", "24h" + updated_at: datetime + + +@dataclass +class SelfImprovementProposal: + id: str + created_at: datetime + author: Literal["system", "agent", "human"] + title: str + description: str + rationale: str + expected_effect: str + status: Literal["pending", "accepted", "rejected", "implemented"] + metadata: dict[str, Any] + diff --git a/src/mnemocore/core/meta_memory.py b/src/mnemocore/core/meta_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..01f1f608766cc4e8a4fafd06d2c72aee2bdbcfc1 --- /dev/null +++ b/src/mnemocore/core/meta_memory.py @@ -0,0 +1,70 @@ +""" +Meta Memory Service +=================== +Maintains a self-model of the memory substrate, gathering metrics and surfacing self-improvement proposals. +Plays a crucial role in enabling an AGI system to observe and upgrade its own thinking architectures over time. +""" + +from typing import Dict, List, Optional +import threading +import logging +from datetime import datetime + +from .memory_model import SelfMetric, SelfImprovementProposal + +logger = logging.getLogger(__name__) + + +class MetaMemoryService: + def __init__(self): + self._metrics: List[SelfMetric] = [] + self._proposals: Dict[str, SelfImprovementProposal] = {} + self._lock = threading.RLock() + + def record_metric(self, name: str, value: float, window: str) -> None: + """Log a new performance or algorithmic metric reading.""" + with self._lock: + # We strictly bind this to metrics history for Subconscious AI trend analysis. + metric = SelfMetric( + name=name, value=value, window=window, updated_at=datetime.utcnow() + ) + self._metrics.append(metric) + + # Cap local metrics storage bounds + if len(self._metrics) > 10000: + self._metrics = self._metrics[-5000:] + + logger.debug(f"Recorded meta-metric: {name}={value} ({window})") + + def list_metrics(self, limit: int = 100, window: Optional[str] = None) -> List[SelfMetric]: + """Fetch historical metric footprints.""" + with self._lock: + filtered = [m for m in self._metrics if (not window) or m.window == window] + filtered.sort(key=lambda x: x.updated_at, reverse=True) + return filtered[:limit] + + def create_proposal(self, proposal: SelfImprovementProposal) -> str: + """Inject a formally modeled improvement prompt into the queue.""" + with self._lock: + self._proposals[proposal.id] = proposal + logger.info(f"New self-improvement proposal created by {proposal.author}: {proposal.title}") + return proposal.id + + def update_proposal_status(self, proposal_id: str, status: str) -> None: + """Mark a proposal as accepted, rejected, or implemented by the oversight entity.""" + with self._lock: + proposal = self._proposals.get(proposal_id) + if not proposal: + logger.warning(f"Could not update unknown proposal ID: {proposal_id}") + return + + proposal.status = status # type: ignore + logger.info(f"Proposal {proposal_id} status escalated to: {status}") + + def list_proposals(self, status: Optional[str] = None) -> List[SelfImprovementProposal]: + """Retrieve proposals matching a given state.""" + with self._lock: + if status: + return [p for p in self._proposals.values() if p.status == status] + return list(self._proposals.values()) + diff --git a/src/mnemocore/core/node.py b/src/mnemocore/core/node.py index f07f8179cb39fa688f71c523c709a4d8ffa9824c..5d8d244b95893b753606686e8c34b9c1bf52ae1f 100644 --- a/src/mnemocore/core/node.py +++ b/src/mnemocore/core/node.py @@ -1,11 +1,16 @@ +from __future__ import annotations + from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Dict, Any, Optional +from typing import TYPE_CHECKING, Dict, Any, Optional import math from .binary_hdv import BinaryHDV from .config import get_config +if TYPE_CHECKING: + from .provenance import ProvenanceRecord + @dataclass class MemoryNode: @@ -35,6 +40,15 @@ class MemoryNode: # Phase 4.3: Episodic Chaining - links to temporally adjacent memories previous_id: Optional[str] = None # UUID of the memory created immediately before this one + # Phase 5.0 — Agent 1: Trust & Provenance + provenance: Optional["ProvenanceRecord"] = field(default=None, repr=False) + + # Phase 5.0 — Agent 2: Adaptive Temporal Decay + # Per-memory stability: S_i = S_base * (1 + k * access_count) + # Starts at 1.0; increases logarithmically on access. + stability: float = 1.0 + review_candidate: bool = False # Set by ForgettingCurveManager when near decay threshold + def access(self, update_weights: bool = True): """Retrieve memory (reconsolidation)""" now = datetime.now(timezone.utc) @@ -46,6 +60,11 @@ class MemoryNode: # We recalculate based on new access count self.calculate_ltp() + # Phase 5.0: update per-memory stability on each successful access + # S_i grows logarithmically so older frequently-accessed memories are more stable + import math as _math + self.stability = max(1.0, 1.0 + _math.log1p(self.access_count) * 0.5) + # Legacy updates self.epistemic_value *= 1.01 self.epistemic_value = min(self.epistemic_value, 1.0) diff --git a/src/mnemocore/core/prediction_store.py b/src/mnemocore/core/prediction_store.py new file mode 100644 index 0000000000000000000000000000000000000000..424c12cdbf64fd79acc924736156a7d1526cca85 --- /dev/null +++ b/src/mnemocore/core/prediction_store.py @@ -0,0 +1,294 @@ +""" +Prediction Memory Store (Phase 5.0 — Agent 4) +============================================== +Stores explicitly made predictions about future events and tracks their outcomes. + +A prediction has a lifecycle: + pending → verified (correct) OR falsified (wrong) OR expired (deadline passed) + +Key behaviors: + - Verified predictions STRENGTHEN related strategic memories via synaptic binding + - Falsified predictions REDUCE confidence on related memories + generate a + "lesson learned" via SubconsciousAI + - Expired predictions are flagged for manual review + +Backed by a lightweight in-memory + provenance-attached store. +For persistence, predictions are serialized to node.metadata["prediction"] and +stored as regular MemoryNodes in the HOT tier with a special tag. + +Public API: + store = PredictionStore() + pred_id = store.create(content="...", confidence=0.7, deadline_days=90) + store.verify(pred_id, success=True, notes="EU AI Act enforced") + due = store.get_due() # predictions past their deadline +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone, timedelta +from typing import Any, Dict, List, Optional + +from loguru import logger + + +# ------------------------------------------------------------------ # +# Prediction status constants # +# ------------------------------------------------------------------ # + +STATUS_PENDING = "pending" +STATUS_VERIFIED = "verified" +STATUS_FALSIFIED = "falsified" +STATUS_EXPIRED = "expired" + + +# ------------------------------------------------------------------ # +# PredictionRecord # +# ------------------------------------------------------------------ # + +@dataclass +class PredictionRecord: + """A single forward-looking prediction stored in MnemoCore.""" + + id: str = field(default_factory=lambda: f"pred_{uuid.uuid4().hex[:16]}") + content: str = "" + predicted_at: str = field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + verification_deadline: Optional[str] = None # ISO datetime string + confidence_at_creation: float = 0.5 + status: str = STATUS_PENDING + outcome: Optional[bool] = None # True=verified, False=falsified + verification_notes: Optional[str] = None + verified_at: Optional[str] = None + related_memory_ids: List[str] = field(default_factory=list) + tags: List[str] = field(default_factory=list) + + def is_expired(self) -> bool: + """True if the deadline has passed and status is still pending.""" + if self.status != STATUS_PENDING or self.verification_deadline is None: + return False + deadline = datetime.fromisoformat(self.verification_deadline) + if deadline.tzinfo is None: + deadline = deadline.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) > deadline + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "content": self.content, + "predicted_at": self.predicted_at, + "verification_deadline": self.verification_deadline, + "confidence_at_creation": round(self.confidence_at_creation, 4), + "status": self.status, + "outcome": self.outcome, + "verification_notes": self.verification_notes, + "verified_at": self.verified_at, + "related_memory_ids": self.related_memory_ids, + "tags": self.tags, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "PredictionRecord": + return cls( + id=d.get("id", f"pred_{uuid.uuid4().hex[:16]}"), + content=d.get("content", ""), + predicted_at=d.get("predicted_at", datetime.now(timezone.utc).isoformat()), + verification_deadline=d.get("verification_deadline"), + confidence_at_creation=d.get("confidence_at_creation", 0.5), + status=d.get("status", STATUS_PENDING), + outcome=d.get("outcome"), + verification_notes=d.get("verification_notes"), + verified_at=d.get("verified_at"), + related_memory_ids=d.get("related_memory_ids", []), + tags=d.get("tags", []), + ) + + +# ------------------------------------------------------------------ # +# PredictionStore # +# ------------------------------------------------------------------ # + +class PredictionStore: + """ + In-memory store for PredictionRecords with lifecycle management. + + For production use, wire to an engine so verified/falsified predictions + can update related MemoryNode synapses and generate LLM insights. + """ + + def __init__(self, engine=None) -> None: + self.engine = engine + self._records: Dict[str, PredictionRecord] = {} + + # ---- CRUD ---------------------------------------------------- # + + def create( + self, + content: str, + confidence: float = 0.5, + deadline_days: Optional[float] = None, + deadline: Optional[datetime] = None, + related_memory_ids: Optional[List[str]] = None, + tags: Optional[List[str]] = None, + ) -> str: + """ + Store a new prediction. + + Args: + content: The prediction statement. + confidence: Confidence at creation time [0, 1]. + deadline_days: Days from now until deadline (alternative to deadline). + deadline: Explicit deadline datetime (overrides deadline_days). + related_memory_ids: IDs of memories this prediction relates to. + tags: Optional classification tags. + + Returns: + The prediction ID. + """ + deadline_iso: Optional[str] = None + if deadline is not None: + if deadline.tzinfo is None: + deadline = deadline.replace(tzinfo=timezone.utc) + deadline_iso = deadline.isoformat() + elif deadline_days is not None: + deadline_iso = ( + datetime.now(timezone.utc) + timedelta(days=deadline_days) + ).isoformat() + + rec = PredictionRecord( + content=content, + confidence_at_creation=max(0.0, min(1.0, confidence)), + verification_deadline=deadline_iso, + related_memory_ids=related_memory_ids or [], + tags=tags or [], + ) + self._records[rec.id] = rec + logger.info( + f"Prediction created: {rec.id} | confidence={confidence:.2f} | " + f"deadline={deadline_iso or 'none'}" + ) + return rec.id + + def get(self, pred_id: str) -> Optional[PredictionRecord]: + return self._records.get(pred_id) + + def list_all(self, status: Optional[str] = None) -> List[PredictionRecord]: + """Return all predictions, optionally filtered by status.""" + recs = list(self._records.values()) + if status: + recs = [r for r in recs if r.status == status] + return sorted(recs, key=lambda r: r.predicted_at, reverse=True) + + def get_due(self) -> List[PredictionRecord]: + """Return pending predictions that have passed their deadline.""" + return [r for r in self._records.values() if r.is_expired()] + + # ---- Lifecycle ----------------------------------------------- # + + async def verify( + self, + pred_id: str, + success: bool, + notes: Optional[str] = None, + ) -> Optional[PredictionRecord]: + """ + Verify or falsify a prediction. + + Side effects: + - Verified: strengthens related memories via synaptic binding + - Falsified: reduces confidence on related memories + lesson learned + """ + rec = self._records.get(pred_id) + if rec is None: + logger.warning(f"PredictionStore.verify: unknown id {pred_id!r}") + return None + + rec.status = STATUS_VERIFIED if success else STATUS_FALSIFIED + rec.outcome = success + rec.verification_notes = notes + rec.verified_at = datetime.now(timezone.utc).isoformat() + + logger.info( + f"Prediction {pred_id} → {rec.status} | notes={notes or '—'}" + ) + + if self.engine is not None: + if success: + await self._strengthen_related(rec) + else: + await self._weaken_related(rec) + await self._generate_lesson(rec) + + return rec + + async def expire_due(self) -> List[PredictionRecord]: + """Mark overdue pending predictions as expired. Returns expired list.""" + due = self.get_due() + for rec in due: + rec.status = STATUS_EXPIRED + logger.info(f"Prediction {rec.id} expired (deadline passed).") + return due + + # ---- Engine integration -------------------------------------- # + + async def _strengthen_related(self, rec: PredictionRecord) -> None: + """Verified prediction → strengthen synapses on related memories.""" + for mem_id in rec.related_memory_ids: + try: + node = await self.engine.get_memory(mem_id) + if node: + si = getattr(self.engine, "synapse_index", None) + if si: + si.add_or_strengthen(rec.id, mem_id, delta=0.15) + logger.debug(f"Prediction {rec.id}: strengthened memory {mem_id[:8]}") + except Exception as exc: + logger.debug(f"Prediction strengthen failed for {mem_id}: {exc}") + + async def _weaken_related(self, rec: PredictionRecord) -> None: + """Falsified prediction → reduce confidence on related memories.""" + for mem_id in rec.related_memory_ids: + try: + node = await self.engine.get_memory(mem_id) + if node: + from .bayesian_ltp import get_bayesian_updater + updater = get_bayesian_updater() + updater.observe_node_retrieval(node, helpful=False, eig_signal=0.5) + logger.debug(f"Prediction {rec.id}: weakened memory {mem_id[:8]}") + except Exception as exc: + logger.debug(f"Prediction weaken failed for {mem_id}: {exc}") + + async def _generate_lesson(self, rec: PredictionRecord) -> None: + """Ask SubconsciousAI to synthesize a 'lesson learned' for a falsified prediction.""" + try: + subcon = getattr(self.engine, "subconscious_ai", None) + if subcon is None: + return + prompt = ( + f"The following prediction was FALSIFIED: '{rec.content}'. " + f"Confidence at creation: {rec.confidence_at_creation:.2f}. " + f"Notes: {rec.verification_notes or 'none'}. " + "In 1-2 sentences, what is the key lesson learned from this failure?" + ) + lesson = await subcon.generate(prompt, max_tokens=128) + # Store the lesson as a new memory + await self.engine.store( + lesson.strip(), + metadata={ + "type": "lesson_learned", + "source_prediction_id": rec.id, + "domain": "strategic", + } + ) + logger.info(f"Lesson learned generated for falsified prediction {rec.id}") + except Exception as exc: + logger.debug(f"Lesson generation failed: {exc}") + + # ---- Serialization ------------------------------------------- # + + def to_list(self) -> List[Dict[str, Any]]: + return [r.to_dict() for r in self._records.values()] + + def __len__(self) -> int: + return len(self._records) diff --git a/src/mnemocore/core/preference_store.py b/src/mnemocore/core/preference_store.py new file mode 100644 index 0000000000000000000000000000000000000000..35994f3376b0ee956765d89bed3b1a6beb2ca471 --- /dev/null +++ b/src/mnemocore/core/preference_store.py @@ -0,0 +1,53 @@ +import numpy as np +from loguru import logger +from typing import List, Optional + +from .binary_hdv import BinaryHDV, majority_bundle +from .config import PreferenceConfig + +class PreferenceStore: + """ + Phase 12.3: Preference Learning + Maintains a persistent vector representing implicit user preferences + based on logged decisions or positive feedback. + """ + def __init__(self, config: PreferenceConfig, dimension: int): + self.config = config + self.dimension = dimension + # The preference vector represents the "ideal" or "preferred" region + self.preference_vector: Optional[BinaryHDV] = None + self.decision_history: List[BinaryHDV] = [] + + def log_decision(self, context_hdv: BinaryHDV, outcome: float) -> None: + """ + Logs a decision or feedback event. + `outcome`: positive value (e.g. 1.0) for good feedback, negative (-1.0) for bad feedback. + If outcome is positive, the preference vector shifts slightly toward `context_hdv`. + If outcome is negative, the preference vector shifts away (invert context_hdv). + """ + if not self.config.enabled: + return + + target_hdv = context_hdv if outcome >= 0 else context_hdv.invert() + self.decision_history.append(target_hdv) + + # Maintain history size + if len(self.decision_history) > self.config.history_limit: + self.decision_history.pop(0) + + # Update preference vector via majority bundling of recent positive shifts + self.preference_vector = majority_bundle(self.decision_history) + logger.debug(f"Logged decision (outcome={outcome:.2f}). Preference vector updated.") + + def bias_score(self, target_hdv: BinaryHDV, base_score: float) -> float: + """ + Biases a retrieval score using the preference vector if one exists. + Formula: new_score = base_score + (learning_rate * preference_similarity) + """ + if not self.config.enabled or self.preference_vector is None: + return base_score + + pref_sim = self.preference_vector.similarity(target_hdv) + + # We apply the learning_rate as the max potential boost an item can get from mapping exactly to preferences + return base_score + (self.config.learning_rate * pref_sim) diff --git a/src/mnemocore/core/procedural_store.py b/src/mnemocore/core/procedural_store.py new file mode 100644 index 0000000000000000000000000000000000000000..fcd296c52560c3b045a685a1657b8fc6d2b991de --- /dev/null +++ b/src/mnemocore/core/procedural_store.py @@ -0,0 +1,77 @@ +""" +Procedural Store Service +======================== +Manages actionable skills, procedural routines, and agentic workflows. +Validates triggering patterns and tracks execution success rates dynamically. +""" + +from typing import Dict, List, Optional +import threading +import logging +from datetime import datetime + +from .memory_model import Procedure + +logger = logging.getLogger(__name__) + + +class ProceduralStoreService: + def __init__(self): + # Local dictionary for Procedures mapping by ID + # Would typically be serialized to SQLite, JSON, or Qdrant for retrieval. + self._procedures: Dict[str, Procedure] = {} + self._lock = threading.RLock() + + def store_procedure(self, proc: Procedure) -> None: + """Save a new or refined procedure into memory.""" + with self._lock: + proc.updated_at = datetime.utcnow() + self._procedures[proc.id] = proc + logger.info(f"Stored procedure {proc.id} ('{proc.name}')") + + def get_procedure(self, proc_id: str) -> Optional[Procedure]: + """Retrieve a procedure by exact ID.""" + with self._lock: + return self._procedures.get(proc_id) + + def find_applicable_procedures( + self, query: str, agent_id: Optional[str] = None, top_k: int = 5 + ) -> List[Procedure]: + """ + Find procedures whose trigger tags or trigger pattern matches the user intent. + Simple local text-matching for the prototype layout. + """ + with self._lock: + q_lower = query.lower() + results = [] + for proc in self._procedures.values(): + # Prefer procedures meant directly for this agent, or system globals + if proc.created_by_agent is not None and agent_id and proc.created_by_agent != agent_id: + continue + + if proc.trigger_pattern.lower() in q_lower or any(t.lower() in q_lower for t in proc.tags): + results.append(proc) + + # Sort by reliability and usage history to surface most competent tools + results.sort(key=lambda p: (p.reliability, p.success_count), reverse=True) + return results[:top_k] + + def record_procedure_outcome(self, proc_id: str, success: bool) -> None: + """Update procedure success metrics, affecting overall reliability.""" + with self._lock: + proc = self._procedures.get(proc_id) + if not proc: + return + + proc.updated_at = datetime.utcnow() + if success: + proc.success_count += 1 + # Increase reliability slightly on success + proc.reliability = min(1.0, proc.reliability + 0.05) + else: + proc.failure_count += 1 + # Decrease reliability heavily on failure + proc.reliability = max(0.0, proc.reliability - 0.1) + + logger.debug(f"Procedure {proc_id} outcome recorded: success={success}, new rel={proc.reliability:.2f}") + diff --git a/src/mnemocore/core/provenance.py b/src/mnemocore/core/provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..d2b290c05ab16ec4489f181c915703561ef0ae3d --- /dev/null +++ b/src/mnemocore/core/provenance.py @@ -0,0 +1,297 @@ +""" +Provenance Tracking Module (Phase 5.0) +======================================= +W3C PROV-inspired source tracking for MnemoCore memories. + +Tracks the full lifecycle of every MemoryNode: + - origin: where/how the memory was created + - lineage: ordered list of transformation events + - version: incremented on each significant mutation + +This is the foundation for: + - Trust & audit trails (AI Governance) + - Contradiction resolution + - Memory-as-a-Service lineage API + - Source reliability scoring + +Public API: + record = ProvenanceRecord.new(origin_type="observation", agent_id="agent-001") + record.add_event("consolidated", source_memories=["mem_a", "mem_b"]) + serialized = record.to_dict() + restored = ProvenanceRecord.from_dict(serialized) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + + +# ------------------------------------------------------------------ # +# Origin types # +# ------------------------------------------------------------------ # + +ORIGIN_TYPES = { + "observation", # Direct input from agent or user + "inference", # Derived/reasoned by LLM or engine + "dream", # Produced by SubconsciousAI dream cycle + "consolidation", # Result of SemanticConsolidation merge + "external_sync", # Fetched from external source (RSS, API, etc.) + "user_correction", # Explicit user override + "prediction", # Stored as a future prediction +} + + +# ------------------------------------------------------------------ # +# Lineage event # +# ------------------------------------------------------------------ # + +@dataclass +class LineageEvent: + """ + A single step in a memory's transformation history. + + Examples: + created – initial storage + accessed – retrieved by a query + consolidated – merged into or from a proto-memory cluster + verified – reliability confirmed externally + contradicted – flagged as contradicting another memory + updated – content or metadata modified + archived – moved to COLD tier + expired – TTL reached or evicted + """ + event: str + timestamp: str # ISO 8601 + actor: Optional[str] = None # agent_id, "system", "user", etc. + source_memories: List[str] = field(default_factory=list) # for consolidation + outcome: Optional[bool] = None # for verification events + notes: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "event": self.event, + "timestamp": self.timestamp, + } + if self.actor is not None: + d["actor"] = self.actor + if self.source_memories: + d["source_memories"] = self.source_memories + if self.outcome is not None: + d["outcome"] = self.outcome + if self.notes: + d["notes"] = self.notes + if self.extra: + d["extra"] = self.extra + return d + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "LineageEvent": + return cls( + event=d["event"], + timestamp=d["timestamp"], + actor=d.get("actor"), + source_memories=d.get("source_memories", []), + outcome=d.get("outcome"), + notes=d.get("notes"), + extra=d.get("extra", {}), + ) + + +# ------------------------------------------------------------------ # +# Origin # +# ------------------------------------------------------------------ # + +@dataclass +class ProvenanceOrigin: + """Where/how a memory was first created.""" + + type: str # One of ORIGIN_TYPES + agent_id: Optional[str] = None + session_id: Optional[str] = None + source_url: Optional[str] = None # For external_sync + timestamp: str = field( + default_factory=lambda: datetime.now(timezone.utc).isoformat() + ) + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "type": self.type, + "timestamp": self.timestamp, + } + if self.agent_id: + d["agent_id"] = self.agent_id + if self.session_id: + d["session_id"] = self.session_id + if self.source_url: + d["source_url"] = self.source_url + return d + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "ProvenanceOrigin": + return cls( + type=d.get("type", "observation"), + agent_id=d.get("agent_id"), + session_id=d.get("session_id"), + source_url=d.get("source_url"), + timestamp=d.get("timestamp", datetime.now(timezone.utc).isoformat()), + ) + + +# ------------------------------------------------------------------ # +# ProvenanceRecord — the full provenance object on a MemoryNode # +# ------------------------------------------------------------------ # + +@dataclass +class ProvenanceRecord: + """ + Full provenance object attached to a MemoryNode. + + Designed to be serialized into node.metadata["provenance"] for + backward compatibility with existing storage layers. + """ + + origin: ProvenanceOrigin + lineage: List[LineageEvent] = field(default_factory=list) + version: int = 1 + confidence_source: str = "bayesian_ltp" # How the confidence score is derived + + # ---- Factory methods ------------------------------------------ # + + @classmethod + def new( + cls, + origin_type: str = "observation", + agent_id: Optional[str] = None, + session_id: Optional[str] = None, + source_url: Optional[str] = None, + actor: Optional[str] = None, + ) -> "ProvenanceRecord": + """Create a fresh ProvenanceRecord and log the 'created' event.""" + now = datetime.now(timezone.utc).isoformat() + origin = ProvenanceOrigin( + type=origin_type if origin_type in ORIGIN_TYPES else "observation", + agent_id=agent_id, + session_id=session_id, + source_url=source_url, + timestamp=now, + ) + record = cls(origin=origin) + record.add_event( + event="created", + actor=actor or agent_id or "system", + ) + return record + + # ---- Mutation ------------------------------------------------- # + + def add_event( + self, + event: str, + actor: Optional[str] = None, + source_memories: Optional[List[str]] = None, + outcome: Optional[bool] = None, + notes: Optional[str] = None, + **extra: Any, + ) -> "ProvenanceRecord": + """Append a new lineage event and bump the version counter.""" + evt = LineageEvent( + event=event, + timestamp=datetime.now(timezone.utc).isoformat(), + actor=actor, + source_memories=source_memories or [], + outcome=outcome, + notes=notes, + extra=extra, + ) + self.lineage.append(evt) + self.version += 1 + return self + + def mark_consolidated( + self, + source_memory_ids: List[str], + actor: str = "consolidation_worker", + ) -> "ProvenanceRecord": + """Convenience wrapper for consolidation events.""" + return self.add_event( + event="consolidated", + actor=actor, + source_memories=source_memory_ids, + ) + + def mark_verified( + self, + success: bool, + actor: str = "system", + notes: Optional[str] = None, + ) -> "ProvenanceRecord": + """Record a verification outcome.""" + return self.add_event( + event="verified", + actor=actor, + outcome=success, + notes=notes, + ) + + def mark_contradicted( + self, + contradiction_group_id: str, + actor: str = "contradiction_detector", + ) -> "ProvenanceRecord": + """Flag this memory as contradicted.""" + return self.add_event( + event="contradicted", + actor=actor, + contradiction_group_id=contradiction_group_id, + ) + + # ---- Serialization -------------------------------------------- # + + def to_dict(self) -> Dict[str, Any]: + return { + "origin": self.origin.to_dict(), + "lineage": [e.to_dict() for e in self.lineage], + "version": self.version, + "confidence_source": self.confidence_source, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "ProvenanceRecord": + return cls( + origin=ProvenanceOrigin.from_dict(d.get("origin", {"type": "observation"})), + lineage=[LineageEvent.from_dict(e) for e in d.get("lineage", [])], + version=d.get("version", 1), + confidence_source=d.get("confidence_source", "bayesian_ltp"), + ) + + # ---- Helpers -------------------------------------------------- # + + @property + def created_at(self) -> str: + """ISO timestamp of the creation event.""" + for event in self.lineage: + if event.event == "created": + return event.timestamp + return self.origin.timestamp + + @property + def last_event(self) -> Optional[LineageEvent]: + """Most recent lineage event.""" + return self.lineage[-1] if self.lineage else None + + def is_contradicted(self) -> bool: + return any(e.event == "contradicted" for e in self.lineage) + + def is_verified(self) -> bool: + return any( + e.event == "verified" and e.outcome is True for e in self.lineage + ) + + def __repr__(self) -> str: + return ( + f"ProvenanceRecord(origin_type={self.origin.type!r}, " + f"version={self.version}, events={len(self.lineage)})" + ) diff --git a/src/mnemocore/core/pulse.py b/src/mnemocore/core/pulse.py new file mode 100644 index 0000000000000000000000000000000000000000..2722de06912fea9ba24cab599b01cd2cbaa2367f --- /dev/null +++ b/src/mnemocore/core/pulse.py @@ -0,0 +1,110 @@ +""" +Pulse Heartbeat Loop +==================== +The central background orchestrator that binds together the AGI cognitive cycles. +Triggers working memory maintenance, episodic sequence linking, gap tracking, and subconscious inferences. +""" + +from typing import Optional +from enum import Enum +import threading +import asyncio +import logging +import traceback +from datetime import datetime + +logger = logging.getLogger(__name__) + + +class PulseTick(Enum): + WM_MAINTENANCE = "wm_maintenance" + EPISODIC_CHAINING = "episodic_chaining" + SEMANTIC_REFRESH = "semantic_refresh" + GAP_DETECTION = "gap_detection" + INSIGHT_GENERATION = "insight_generation" + PROCEDURE_REFINEMENT = "procedure_refinement" + META_SELF_REFLECTION = "meta_self_reflection" + + +class PulseLoop: + def __init__(self, container, config): + """ + Args: + container: The fully built DI Container containing all memory sub-services. + config: Specifically the `config.pulse` section settings. + """ + self.container = container + self.config = config + self._running = False + self._task: Optional[asyncio.Task] = None + + async def start(self) -> None: + """Begin the background pulse orchestrator.""" + if not getattr(self.config, "enabled", False): + logger.info("Pulse loop is disabled via configuration.") + return + + self._running = True + interval = getattr(self.config, "interval_seconds", 30) + logger.info(f"Starting AGI Pulse Loop (interval={interval}s).") + + while self._running: + start_time = datetime.utcnow() + try: + await self.tick() + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Error during Pulse tick: {e}", exc_info=True) + + elapsed = (datetime.utcnow() - start_time).total_seconds() + sleep_time = max(0.1, interval - elapsed) + await asyncio.sleep(sleep_time) + + def stop(self) -> None: + """Gracefully interrupt and unbind the Pulse loop.""" + self._running = False + if self._task and not self._task.done(): + self._task.cancel() + logger.info("AGI Pulse Loop stopped.") + + async def tick(self) -> None: + """Execute a full iteration across the cognitive architecture planes.""" + await self._wm_maintenance() + await self._episodic_chaining() + await self._semantic_refresh() + await self._gap_detection() + await self._insight_generation() + await self._procedure_refinement() + await self._meta_self_reflection() + + async def _wm_maintenance(self) -> None: + """Prune overloaded short-term buffers and cull expired items.""" + if hasattr(self.container, "working_memory") and self.container.working_memory: + self.container.working_memory.prune_all() + logger.debug(f"Pulse: [{PulseTick.WM_MAINTENANCE.value}] Executed.") + + async def _episodic_chaining(self) -> None: + """Retroactively verify event streams and apply temporal links between episodic contexts.""" + logger.debug(f"Pulse: [{PulseTick.EPISODIC_CHAINING.value}] Stubbed.") + + async def _semantic_refresh(self) -> None: + """Prompt Qdrant abstractions or run `semantic_consolidation` loops over episodic data.""" + logger.debug(f"Pulse: [{PulseTick.SEMANTIC_REFRESH.value}] Stubbed.") + + async def _gap_detection(self) -> None: + """Unearth missing knowledge vectors (GapDetector integration).""" + logger.debug(f"Pulse: [{PulseTick.GAP_DETECTION.value}] Stubbed.") + + async def _insight_generation(self) -> None: + """Forward memory patterns to LLM for spontaneous inference generation.""" + logger.debug(f"Pulse: [{PulseTick.INSIGHT_GENERATION.value}] Stubbed.") + + async def _procedure_refinement(self) -> None: + """Modify procedure reliabilities directly depending on episode occurrences.""" + logger.debug(f"Pulse: [{PulseTick.PROCEDURE_REFINEMENT.value}] Stubbed.") + + async def _meta_self_reflection(self) -> None: + """Collate macro anomalies and submit SelfImprovementProposals.""" + logger.debug(f"Pulse: [{PulseTick.META_SELF_REFLECTION.value}] Stubbed.") + diff --git a/src/mnemocore/core/qdrant_store.py b/src/mnemocore/core/qdrant_store.py index 34d6de3558e06a52fcc71244be65ed88d2e047ed..687865470ba81427bace336724c294df98c94fb7 100644 --- a/src/mnemocore/core/qdrant_store.py +++ b/src/mnemocore/core/qdrant_store.py @@ -6,7 +6,7 @@ Provides async access to Qdrant for vector storage and similarity search. Phase 4.3: Temporal Recall - supports time-based filtering and indexing. """ -from typing import List, Any, Optional, Tuple +from typing import List, Any, Optional, Tuple, Dict from datetime import datetime import asyncio @@ -100,7 +100,7 @@ class QdrantStore: collection_name=self.collection_hot, vectors_config=models.VectorParams( size=self.dim, - distance=models.Distance.COSINE, + distance=models.Distance.DOT, on_disk=False ), quantization_config=quantization_config, @@ -118,7 +118,7 @@ class QdrantStore: collection_name=self.collection_warm, vectors_config=models.VectorParams( size=self.dim, - distance=models.Distance.MANHATTAN, + distance=models.Distance.DOT, on_disk=True ), quantization_config=quantization_config, @@ -169,6 +169,7 @@ class QdrantStore: limit: int = 5, score_threshold: float = 0.0, time_range: Optional[Tuple[datetime, datetime]] = None, + metadata_filter: Optional[Dict[str, Any]] = None, ) -> List[models.ScoredPoint]: """ Async semantic search. @@ -189,30 +190,51 @@ class QdrantStore: as search failures should not crash the calling code. """ try: - # Build time filter if provided (Phase 4.3) - query_filter = None + must_conditions = [] if time_range: start_ts = int(time_range[0].timestamp()) end_ts = int(time_range[1].timestamp()) - query_filter = models.Filter( - must=[ - models.FieldCondition( - key="unix_timestamp", - range=models.Range( - gte=start_ts, - lte=end_ts, - ), + must_conditions.append( + models.FieldCondition( + key="unix_timestamp", + range=models.Range( + gte=start_ts, + lte=end_ts, ), - ] + ) + ) + + if metadata_filter: + for k, v in metadata_filter.items(): + must_conditions.append( + models.FieldCondition( + key=k, + match=models.MatchValue(value=v) + ) + ) + + if must_conditions: + query_filter = models.Filter(must=must_conditions) + + # Support for Binary Quantization rescoring (BUG-04) + search_params = None + if self.binary_quantization: + search_params = models.SearchParams( + quantization=models.QuantizationSearchParams( + ignore=False, + rescore=True, + oversampling=2.0 + ) ) return await qdrant_breaker.call( - self.client.search, + self.client.query_points, collection_name=collection, - query_vector=query_vector, + query=query_vector, limit=limit, score_threshold=score_threshold, query_filter=query_filter, + search_params=search_params, ) except CircuitOpenError: logger.warning(f"Qdrant search blocked for {collection}: circuit breaker open") diff --git a/src/mnemocore/core/semantic_consolidation.py b/src/mnemocore/core/semantic_consolidation.py index a3cd7086682552ce50a90a79d9c5cec6395ae32b..8fa9759e9c6c28bfa39de881ce164bca12072824 100644 --- a/src/mnemocore/core/semantic_consolidation.py +++ b/src/mnemocore/core/semantic_consolidation.py @@ -37,6 +37,7 @@ from loguru import logger from .binary_hdv import BinaryHDV, majority_bundle from .config import get_config from .node import MemoryNode +from .provenance import ProvenanceRecord # ------------------------------------------------------------------ # @@ -266,6 +267,18 @@ class SemanticConsolidationWorker: medoid_node.metadata["proto_updated_at"] = datetime.now(timezone.utc).isoformat() proto_count += 1 + # Phase 5.0: record consolidation in provenance lineage + source_ids = [n.id for n in member_nodes if n.id != medoid_node.id] + if medoid_node.provenance is None: + medoid_node.provenance = ProvenanceRecord.new( + origin_type="consolidation", + actor="consolidation_worker", + ) + medoid_node.provenance.mark_consolidated( + source_memory_ids=source_ids, + actor="consolidation_worker", + ) + elapsed = time.monotonic() - t0 self.last_run = datetime.now(timezone.utc) self.stats = { diff --git a/src/mnemocore/core/semantic_store.py b/src/mnemocore/core/semantic_store.py new file mode 100644 index 0000000000000000000000000000000000000000..1476818bcbb313318324fcb4a40a9086a13c74e6 --- /dev/null +++ b/src/mnemocore/core/semantic_store.py @@ -0,0 +1,73 @@ +""" +Semantic Store Service +====================== +Manages semantic concepts, abstractions, and long-term general knowledge. +Integrates tightly with Qdrant/vector storage for conceptual search and retrieval. +""" + +from typing import Dict, List, Optional +import threading +import logging + +from .memory_model import SemanticConcept +from .binary_hdv import BinaryHDV + +logger = logging.getLogger(__name__) + + +class SemanticStoreService: + def __init__(self, qdrant_store=None): + """ + Args: + qdrant_store: The underlying QdrantStore instance for HDV similarity searches. + """ + self._qdrant_store = qdrant_store + + # Local cache of semantic concepts, usually backed by proper storage + # In a full implementation, `upsert_concept` also stores to Qdrant/disk. + self._concepts: Dict[str, SemanticConcept] = {} + self._lock = threading.RLock() + + def upsert_concept(self, concept: SemanticConcept) -> None: + """Add or update a semantic concept and ensure it is indexed for search.""" + with self._lock: + self._concepts[concept.id] = concept + logger.debug(f"Upserted semantic concept {concept.id} ({concept.label})") + + # Note: A complete implementation would push the prototype HDV + # and concept metadata into Qdrant for semantic search here. + pass + + def get_concept(self, concept_id: str) -> Optional[SemanticConcept]: + """Retrieve a specific concept by its ID.""" + with self._lock: + return self._concepts.get(concept_id) + + def find_nearby_concepts( + self, hdv: BinaryHDV, top_k: int = 5, min_similarity: float = 0.5 + ) -> List[SemanticConcept]: + """ + Find conceptually similar semantic anchors. + Delegates to Qdrant or performs local brute-force if in minimal mode. + """ + with self._lock: + # Temporary local-only similarity threshold search + results = [] + for concept in self._concepts.values(): + sim = hdv.similarity(concept.prototype_hdv) + if sim >= min_similarity: + results.append((sim, concept)) + + # Sort descending by similarity, then ascending by creation/id + results.sort(key=lambda x: (-x[0], x[1].id)) + return [c for _, c in results[:top_k]] + + def adjust_concept_reliability(self, concept_id: str, delta: float) -> None: + """Bump or decay the reliability of a concept (e.g. via Pulse immunology loops).""" + with self._lock: + concept = self._concepts.get(concept_id) + if not concept: + return + + concept.reliability = max(0.0, min(1.0, concept.reliability + delta)) + diff --git a/src/mnemocore/core/synapse.py b/src/mnemocore/core/synapse.py index 60ced6ebc22be07fa0af86d564f15a230866a878..eb9697a9badac4945a0420380592f92c0b124bab 100644 --- a/src/mnemocore/core/synapse.py +++ b/src/mnemocore/core/synapse.py @@ -20,34 +20,36 @@ class SynapticConnection: self.last_fired = datetime.now(timezone.utc) self.fire_count = 0 self.success_count = 0 # For Hebbian learning + # Cache half_life at creation to avoid get_config() on every strength call. + self._half_life_days: float = get_config().ltp.half_life_days - def fire(self, success: bool = True): + def fire(self, success: bool = True, weight: float = 1.0): """Activate synapse (strengthen if successful)""" self.last_fired = datetime.now(timezone.utc) self.fire_count += 1 if success: - # Hebbian learning: fire together, wire together - # Logistic-like growth or simple fractional approach - self.strength += 0.1 * (1 - self.strength) # Cap at 1.0 implicitly + # Phase 12.1: Allow aggressive weight multiplier for co-occurrence + self.strength += (0.1 * weight) * (1 - self.strength) + if self.strength > 1.0: + self.strength = 1.0 self.success_count += 1 def get_current_strength(self) -> float: """ Ebbinghaus forgetting curve - Returns decayed strength based on age and config half-life. + Returns decayed strength based on age and cached half-life. """ - config = get_config() age_seconds = (datetime.now(timezone.utc) - self.last_fired).total_seconds() age_days = age_seconds / 86400.0 # Exponential decay: exp(-λ * t) # Half-life formula: N(t) = N0 * (1/2)^(t / t_half) # Which is equivalent to N0 * exp(-ln(2) * t / t_half) - half_life = config.ltp.half_life_days - + half_life = self._half_life_days + if half_life <= 0: - return self.strength # No decay + return self.strength # No decay decay = math.exp(-(math.log(2) / half_life) * age_days) diff --git a/src/mnemocore/core/synapse_index.py b/src/mnemocore/core/synapse_index.py index 8c5d5de4e9831a474612b397be718c45fece34c8..d92cd6bb4bdfe8392f875b14af2043b232d172b1 100644 --- a/src/mnemocore/core/synapse_index.py +++ b/src/mnemocore/core/synapse_index.py @@ -27,6 +27,7 @@ from __future__ import annotations import json import os +import math from dataclasses import dataclass from datetime import datetime, timezone from typing import Dict, Iterator, List, Optional, Set, Tuple @@ -83,7 +84,7 @@ class SynapseIndex: self._adj.setdefault(key[0], set()).add(key[1]) self._adj.setdefault(key[1], set()).add(key[0]) - def add_or_fire(self, id_a: str, id_b: str, success: bool = True) -> "SynapticConnection": + def add_or_fire(self, id_a: str, id_b: str, success: bool = True, weight: float = 1.0) -> "SynapticConnection": """ Create a synapse if it doesn't exist, then fire it. @@ -104,7 +105,7 @@ class SynapseIndex: upd = _get_bayesian_updater() upd.observe_synapse(syn, success=success) # Also call the Hebbian fire for backward compat (updates fire_count etc.) - syn.fire(success=success) + syn.fire(success=success, weight=weight) return syn @@ -126,6 +127,36 @@ class SynapseIndex: result.append(syn) return result + def get_multi_hop_neighbors(self, node_id: str, depth: int = 2) -> Dict[str, float]: + """ + Phase 12.1: Traverse graph up to `depth` hops away. + Returns a mapping of node_id -> maximum cumulative connection strength path. + """ + visited = {node_id: 1.0} + current_layer = {node_id: 1.0} + + for _ in range(depth): + next_layer = {} + for curr_node, cum_weight in current_layer.items(): + for syn in self.neighbours(curr_node): + neighbor_id = syn.neuron_b_id if syn.neuron_a_id == curr_node else syn.neuron_a_id + if neighbor_id == node_id: + continue + + edge_weight = syn.get_current_strength() + new_weight = cum_weight * edge_weight + + # Store the strongest path to the node + if neighbor_id not in visited or new_weight > visited[neighbor_id]: + visited[neighbor_id] = new_weight + + if neighbor_id not in next_layer or new_weight > next_layer[neighbor_id]: + next_layer[neighbor_id] = new_weight + current_layer = next_layer + + visited.pop(node_id, None) + return visited + def neighbour_ids(self, node_id: str) -> Set[str]: """O(1) set of connected node IDs.""" return self._adj.get(node_id, set()).copy() @@ -209,10 +240,11 @@ class SynapseIndex: Returns 1.0 for isolated nodes. """ - boost = 1.0 - for syn in self.neighbours(node_id): - boost *= (1.0 + syn.get_current_strength()) - return boost + # Phase 4.5 Hotfix (Robin's Score Bug e+195): + # Instead of exponential product scaling which explodes for hub nodes, + # we aggregate the strengths and bound the multiplier logarithmically. + total_strength = sum(syn.get_current_strength() for syn in self.neighbours(node_id)) + return 1.0 + math.log1p(total_strength) def __len__(self) -> int: return len(self._edges) diff --git a/src/mnemocore/core/temporal_decay.py b/src/mnemocore/core/temporal_decay.py new file mode 100644 index 0000000000000000000000000000000000000000..56e0883c38d8914f8daf12e6ac2598e9db7a225f --- /dev/null +++ b/src/mnemocore/core/temporal_decay.py @@ -0,0 +1,197 @@ +""" +Adaptive Temporal Decay Module (Phase 5.0) +========================================== +Replaces the global λ decay parameter with per-memory adaptive decay +based on individual stability S_i, using the Ebbinghaus-inspired formula: + + R = e^(-T / S_i) + +where: + R = retention score in [0, 1] + T = time since last access (days) + S_i = memory stability = S_base × (1 + k × access_count) + +S_i increases with every successful retrieval, making frequently-accessed +memories much more decay-resistant, while rarely-used memories decay faster. + +This module provides: + - AdaptiveDecayEngine: computes retention and updates stability + - Background scan logic surfaced to TierManager for eviction decisions + - Review candidate flagging (≥ REVIEW_THRESHOLD) for spaced repetition + +Designed to be dependency-injected into TierManager in Phase 5.0. + +Public API: + engine = AdaptiveDecayEngine() + retention = engine.retention(node) # float in [0, 1] + engine.update_after_access(node) # call after each retrieval + candidates = engine.scan_review_candidates(nodes, threshold=0.3) +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from typing import TYPE_CHECKING, List + +from loguru import logger + +if TYPE_CHECKING: + from .node import MemoryNode + + +# ------------------------------------------------------------------ # +# Constants # +# ------------------------------------------------------------------ # + +# Base stability (days before first retrieval triggers significant decay) +S_BASE: float = 1.0 + +# Growth coefficient: how fast stability increases per access +# stability = S_base * (1 + K_GROWTH * log(1 + access_count)) +K_GROWTH: float = 0.5 + +# Retention threshold below which a memory becomes a review candidate +REVIEW_THRESHOLD: float = 0.40 + +# Minimum retention before a memory is considered for eviction +EVICTION_THRESHOLD: float = 0.10 + + +# ------------------------------------------------------------------ # +# Core engine # +# ------------------------------------------------------------------ # + +class AdaptiveDecayEngine: + """ + Computes per-memory adaptive retention scores and manages stability growth. + + Stateless: all state lives in MemoryNode.stability and MemoryNode.access_count. + Safe to instantiate multiple times; no shared mutable state. + """ + + def __init__( + self, + s_base: float = S_BASE, + k_growth: float = K_GROWTH, + review_threshold: float = REVIEW_THRESHOLD, + eviction_threshold: float = EVICTION_THRESHOLD, + ) -> None: + self.s_base = s_base + self.k_growth = k_growth + self.review_threshold = review_threshold + self.eviction_threshold = eviction_threshold + + # ---- Core math ----------------------------------------------- # + + def stability(self, node: "MemoryNode") -> float: + """ + Compute S_i for a node. + + S_i = S_base * (1 + K_GROWTH * log(1 + access_count)) + + This grows logarithmically so stability never explodes but keeps + compounding with regular access. + """ + ac = max(getattr(node, "access_count", 1), 1) + return self.s_base * (1.0 + self.k_growth * math.log1p(ac)) + + def retention(self, node: "MemoryNode") -> float: + """ + Compute current retention R = e^(-T / S_i). + + T is measured in days since last access. + Returns a value in (0, 1]. + """ + # Days since last access + last = getattr(node, "last_accessed", None) + if last is None: + last = getattr(node, "created_at", datetime.now(timezone.utc)) + if getattr(last, "tzinfo", None) is None: + last = last.replace(tzinfo=timezone.utc) + + t_days = (datetime.now(timezone.utc) - last).total_seconds() / 86400.0 + s_i = self.stability(node) + + r = math.exp(-t_days / max(s_i, 0.01)) + return float(r) + + # ---- Mutation helpers ---------------------------------------- # + + def update_after_access(self, node: "MemoryNode") -> None: + """ + Increase per-memory stability after a successful retrieval. + Writes back to node.stability. + """ + node.stability = self.stability(node) + node.review_candidate = False + logger.debug( + f"Node {getattr(node, 'id', '?')[:8]} stability updated → {node.stability:.3f}" + ) + + def update_review_candidate(self, node: "MemoryNode") -> bool: + """ + Flag a node as review_candidate if its retention is below the threshold. + Returns True if flagged. + """ + r = self.retention(node) + if r <= self.review_threshold: + node.review_candidate = True + logger.debug( + f"Node {getattr(node, 'id', '?')[:8]} flagged review_candidate " + f"(retention={r:.3f}, threshold={self.review_threshold})" + ) + return True + node.review_candidate = False + return False + + def should_evict(self, node: "MemoryNode") -> bool: + """ + True if a node's retention has fallen below the eviction threshold. + TierManager should use this instead of the global decay_lambda. + """ + return self.retention(node) <= self.eviction_threshold + + # ---- Batch scanning ------------------------------------------ # + + def scan_review_candidates( + self, nodes: "List[MemoryNode]" + ) -> "List[MemoryNode]": + """ + Scan a list of nodes and flag those that are review candidates. + Returns the sub-list of candidates. + """ + candidates: List["MemoryNode"] = [] + for node in nodes: + if self.update_review_candidate(node): + candidates.append(node) + + logger.info( + f"AdaptiveDecay scan: {len(candidates)}/{len(nodes)} " + f"nodes flagged as review candidates" + ) + return candidates + + def eviction_candidates( + self, nodes: "List[MemoryNode]" + ) -> "List[MemoryNode]": + """ + Return nodes whose retention has fallen below EVICTION_THRESHOLD. + TierManager calls this to decide which nodes to demote or remove. + """ + return [n for n in nodes if self.should_evict(n)] + + +# ------------------------------------------------------------------ # +# Module-level singleton # +# ------------------------------------------------------------------ # + +_ENGINE: AdaptiveDecayEngine | None = None + + +def get_adaptive_decay_engine() -> AdaptiveDecayEngine: + """Return the shared AdaptiveDecayEngine singleton.""" + global _ENGINE + if _ENGINE is None: + _ENGINE = AdaptiveDecayEngine() + return _ENGINE diff --git a/src/mnemocore/core/tier_manager.py b/src/mnemocore/core/tier_manager.py index d3e9783816bc85cf1b17a92e38545a7edc422f3b..8caa6d85200c52fc7d0654daa2403c6810533a8e 100644 --- a/src/mnemocore/core/tier_manager.py +++ b/src/mnemocore/core/tier_manager.py @@ -23,7 +23,7 @@ import json from datetime import datetime, timezone from itertools import islice from pathlib import Path -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Any if TYPE_CHECKING: from .qdrant_store import QdrantStore @@ -86,6 +86,9 @@ class TierManager: # HOT Tier: In-memory dictionary self.hot: Dict[str, MemoryNode] = {} + # Phase 13.2: O(1) inverted index for episodic chain – previous_id → node_id. + # Maintained in sync with self.hot so get_next_in_chain() avoids O(N) scans. + self._next_chain: Dict[str, str] = {} # WARM Tier: Qdrant (injected) or fallback to filesystem self.qdrant = qdrant_store @@ -179,6 +182,9 @@ class TierManager: async with self.lock: self.hot[node.id] = node self._add_to_faiss(node) + # Maintain inverted chain index (Fix 7: O(1) get_next_in_chain) + if node.previous_id: + self._next_chain[node.previous_id] = node.id # Check if we need to evict - decide under lock, execute outside if len(self.hot) > self.config.tiers_hot.max_memories: @@ -263,7 +269,11 @@ class TierManager: await self._promote_to_hot(warm_node) return warm_node - return None + # Fix 1: Fall back to COLD tier (read-only archive scan). + cold_node = await self._load_from_cold(node_id) + if cold_node: + logger.debug(f"Retrieved {node_id} from COLD tier.") + return cold_node async def get_memories_batch(self, node_ids: List[str]) -> List[Optional[MemoryNode]]: """ @@ -288,10 +298,33 @@ class TierManager: return [result_by_id.get(nid) for nid in node_ids] + async def anticipate(self, node_ids: List[str]) -> None: + """ + Phase 13.2: Anticipatory Memory + Pre-loads specific nodes into the HOT active tier (working memory). + This forces the nodes out of WARM/COLD and into RAM for near-zero latency retrieval. + """ + for nid in set(node_ids): + # Check if already in HOT + in_hot = False + async with self.lock: + if nid in self.hot: + in_hot = True + + if not in_hot: + # Load from WARM + node = await self._load_from_warm(nid) + if node: + # Force promote to HOT + await self._promote_to_hot(node) async def delete_memory(self, node_id: str): """Robust delete from all tiers.""" async with self.lock: if node_id in self.hot: + _node = self.hot[node_id] + # Maintain inverted chain index before removal + if _node.previous_id: + self._next_chain.pop(_node.previous_id, None) del self.hot[node_id] self._remove_from_faiss(node_id) logger.debug(f"Deleted {node_id} from HOT") @@ -385,8 +418,10 @@ class TierManager: victim = min(candidates, key=lambda n: n.ltp_strength) logger.info(f"Evicting {victim.id} from HOT to WARM (LTP: {victim.ltp_strength:.4f})") - - # Remove from HOT structure + + # Remove from HOT structure (maintain inverted chain index) + if victim.previous_id: + self._next_chain.pop(victim.previous_id, None) del self.hot[victim.id] self._remove_from_faiss(victim.id) @@ -621,6 +656,9 @@ class TierManager: node.tier = "hot" self.hot[node.id] = node self._add_to_faiss(node) + # Maintain inverted chain index + if node.previous_id: + self._next_chain[node.previous_id] = node.id # Check if we need to evict - prepare under lock, execute outside if len(self.hot) > self.config.tiers_hot.max_memories: @@ -743,12 +781,11 @@ class TierManager: The next MemoryNode in the chain, or None if not found / Qdrant unavailable. """ - # 1. Check HOT tier first (fast Linear Scan) - # This ensures we find recently created links that haven't been demoted yet. + # 1. Check HOT tier via O(1) inverted index (Fix 7). async with self.lock: - for node in self.hot.values(): - if node.previous_id == node_id: - return node + next_id = self._next_chain.get(node_id) + if next_id and next_id in self.hot: + return self.hot[next_id] # 2. Check WARM tier (Qdrant) if not self.use_qdrant or not self.qdrant: @@ -834,26 +871,46 @@ class TierManager: query_vec: BinaryHDV, top_k: int = 5, time_range: Optional[Tuple[datetime, datetime]] = None, + metadata_filter: Optional[Dict[str, Any]] = None, + include_cold: bool = False, ) -> List[Tuple[str, float]]: """ Global search across all tiers. - Combines FAISS (HOT) and Qdrant (WARM). + Combines HNSW/FAISS (HOT), Qdrant/FS (WARM), and optionally COLD. Phase 4.3: time_range filters results to memories within the given datetime range. + Fix 1: include_cold=True enables a bounded linear scan of the COLD archive. """ # 1. Search HOT via FAISS (time filtering done post-hoc for in-memory) hot_results = self.search_hot(query_vec, top_k) - # Apply time filter to HOT results if needed - if time_range: - start_ts = time_range[0].timestamp() - end_ts = time_range[1].timestamp() + # Apply time filter and metadata filter to HOT results if needed + if time_range or metadata_filter: + filtered_hot = [] async with self.lock: - hot_results = [ - (nid, score) for nid, score in hot_results - if nid in self.hot and - start_ts <= self.hot[nid].created_at.timestamp() <= end_ts - ] + for nid, score in hot_results: + node = self.hot.get(nid) + if not node: + continue + + if time_range: + start_ts = time_range[0].timestamp() + end_ts = time_range[1].timestamp() + if not (start_ts <= node.created_at.timestamp() <= end_ts): + continue + + if metadata_filter: + match = True + node_meta = node.metadata or {} + for k, v in metadata_filter.items(): + if node_meta.get(k) != v: + match = False + break + if not match: + continue + + filtered_hot.append((nid, score)) + hot_results = filtered_hot # 2. Search WARM via Qdrant warm_results = [] @@ -866,17 +923,25 @@ class TierManager: query_vector=q_vec, limit=top_k, time_range=time_range, # Phase 4.3: Pass time filter to Qdrant + metadata_filter=metadata_filter, # BUG-09: Agent Isolation ) warm_results = [(hit.id, hit.score) for hit in hits] except Exception as e: logger.error(f"WARM tier search failed: {e}") - # 3. Combine and Sort + # 3. Optionally search COLD tier (Fix 1: bounded linear scan) + cold_results: List[Tuple[str, float]] = [] + if include_cold: + cold_results = await self.search_cold(query_vec, top_k) + + # 4. Combine and Sort (HOT scores take precedence over WARM/COLD for same ID) combined = {} for nid, score in hot_results: combined[nid] = score for nid, score in warm_results: combined[nid] = max(combined.get(nid, 0), score) + for nid, score in cold_results: + combined[nid] = max(combined.get(nid, 0), score) sorted_results = sorted(combined.items(), key=lambda x: x[1], reverse=True) return sorted_results[:top_k] @@ -921,6 +986,115 @@ class TierManager: scores.sort(key=lambda x: x[1], reverse=True) return scores[:top_k] + async def _load_from_cold(self, node_id: str) -> Optional[MemoryNode]: + """ + Scan COLD archive files for a specific node (Fix 1: COLD read path). + + Archives are gzip JSONL, sorted newest-first for early-exit on recent data. + Returns None if not found or on error. + """ + def _scan(): + for archive_file in sorted( + self.cold_path.glob("archive_*.jsonl.gz"), reverse=True + ): + try: + with gzip.open(archive_file, "rt", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + if rec.get("id") == node_id: + return rec + except json.JSONDecodeError: + continue + except Exception: + continue + return None + + rec = await self._run_in_thread(_scan) + if rec is None: + return None + + try: + raw_vec = rec.get("hdv_vector") + dim = rec.get("dimension", self.config.dimensionality) + if raw_vec: + hdv_data = np.array(raw_vec, dtype=np.uint8) + hdv = BinaryHDV(data=hdv_data, dimension=dim) + else: + hdv = BinaryHDV.zeros(dim) + + node = MemoryNode( + id=rec["id"], + hdv=hdv, + content=rec.get("content", ""), + metadata=rec.get("metadata", {}), + tier="cold", + ltp_strength=rec.get("ltp_strength", 0.0), + previous_id=rec.get("previous_id"), + ) + if "created_at" in rec: + node.created_at = datetime.fromisoformat(rec["created_at"]) + return node + except Exception as e: + logger.error(f"Failed to reconstruct node {node_id} from COLD: {e}") + return None + + async def search_cold( + self, + query_vec: BinaryHDV, + top_k: int = 5, + max_scan: int = 1000, + ) -> List[Tuple[str, float]]: + """ + Linear similarity scan over COLD archive (Fix 1: COLD search path). + + Bounded by max_scan records to keep latency predictable. + Returns results sorted by descending similarity. + """ + config_dim = self.config.dimensionality + + def _scan(): + candidates: List[Tuple[str, float]] = [] + scanned = 0 + for archive_file in sorted( + self.cold_path.glob("archive_*.jsonl.gz"), reverse=True + ): + if scanned >= max_scan: + break + try: + with gzip.open(archive_file, "rt", encoding="utf-8") as f: + for line in f: + if scanned >= max_scan: + break + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + raw_vec = rec.get("hdv_vector") + if not raw_vec: + continue + dim = rec.get("dimension", config_dim) + hdv = BinaryHDV( + data=np.array(raw_vec, dtype=np.uint8), + dimension=dim, + ) + sim = query_vec.similarity(hdv) + candidates.append((rec["id"], sim)) + scanned += 1 + except Exception: + continue + except Exception: + continue + return candidates + + candidates = await self._run_in_thread(_scan) + candidates.sort(key=lambda x: x[1], reverse=True) + return candidates[:top_k] + async def _write_to_cold(self, record: dict): """Write a record to the cold archive.""" record["tier"] = "cold" diff --git a/src/mnemocore/core/topic_tracker.py b/src/mnemocore/core/topic_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..403a0d6bf7c5e6a5f300a680671eed634cab0f46 --- /dev/null +++ b/src/mnemocore/core/topic_tracker.py @@ -0,0 +1,62 @@ +import numpy as np +from loguru import logger +from typing import List, Optional, Tuple + +from .binary_hdv import BinaryHDV, majority_bundle +from .config import ContextConfig + +class TopicTracker: + """ + Phase 12.2: Contextual Awareness + Tracks the rolling conversational context using an HDV moving average. + Detects topic shifts and resets the context when appropriate. + """ + def __init__(self, config: ContextConfig, dimension: int): + self.config = config + self.dimension = dimension + self.context_vector: Optional[BinaryHDV] = None + self.history: List[BinaryHDV] = [] + + def add_query(self, query_hdv: BinaryHDV) -> Tuple[bool, float]: + """ + Adds a new query to the tracker. + Returns (is_shift, similarity). + If is_shift is True, it means a topic boundary was detected. + """ + if not self.config.enabled: + return False, 1.0 + + if self.context_vector is None: + self.context_vector = query_hdv + self.history = [query_hdv] + return False, 1.0 + + similarity = self.context_vector.similarity(query_hdv) + + # Detect shift + if similarity < self.config.shift_threshold: + logger.info(f"Topic shift detected! Similarity {similarity:.3f} < {self.config.shift_threshold}") + self.reset(query_hdv) + return True, similarity + + # Update rolling context + self.history.append(query_hdv) + if len(self.history) > self.config.rolling_window_size: + self.history.pop(0) + + # Recompute the majority bundle for the current window + self.context_vector = majority_bundle(self.history) + return False, similarity + + def reset(self, new_context: Optional[BinaryHDV] = None): + """Resets the topic tracker, optionally seeding it with a new query.""" + if new_context: + self.context_vector = new_context + self.history = [new_context] + else: + self.context_vector = None + self.history = [] + + def get_context(self) -> Optional[BinaryHDV]: + """Returns the current topic context vector.""" + return self.context_vector diff --git a/src/mnemocore/core/working_memory.py b/src/mnemocore/core/working_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..5d71b82e82a95fd41bb1738a8aa35eeec803ce6d --- /dev/null +++ b/src/mnemocore/core/working_memory.py @@ -0,0 +1,104 @@ +""" +Working Memory Service +====================== +Manages short-term operational state (STM/WM) for individual agents or sessions. +Provides fast caching, item eviction (via LRU + importance), and contextual focus. +""" + +from typing import Dict, List, Optional +from datetime import datetime, timedelta +import threading +import logging + +from .memory_model import WorkingMemoryItem, WorkingMemoryState + +logger = logging.getLogger(__name__) + + +class WorkingMemoryService: + def __init__(self, max_items_per_agent: int = 64): + self.max_items_per_agent = max_items_per_agent + self._states: Dict[str, WorkingMemoryState] = {} + self._lock = threading.RLock() + + def _get_or_create_state(self, agent_id: str) -> WorkingMemoryState: + with self._lock: + if agent_id not in self._states: + self._states[agent_id] = WorkingMemoryState( + agent_id=agent_id, max_items=self.max_items_per_agent + ) + return self._states[agent_id] + + def push_item(self, agent_id: str, item: WorkingMemoryItem) -> None: + """Push a new item into the agent's working memory, pruning if necessary.""" + with self._lock: + state = self._get_or_create_state(agent_id) + # Prevent exact duplicate IDs from being appended twice + existing_idx = next( + (i for i, x in enumerate(state.items) if x.id == item.id), None + ) + if existing_idx is not None: + state.items[existing_idx] = item + else: + state.items.append(item) + + self._prune(agent_id) + + def get_state(self, agent_id: str) -> Optional[WorkingMemoryState]: + """Retrieve the current working memory state for an agent.""" + with self._lock: + return self._states.get(agent_id) + + def clear(self, agent_id: str) -> None: + """Clear the working memory for a specific agent.""" + with self._lock: + if agent_id in self._states: + self._states[agent_id].items.clear() + + def prune_all(self) -> None: + """Prune TTL-expired items and overflows across all agents. Typically called by Pulse.""" + with self._lock: + for agent_id in list(self._states.keys()): + self._prune(agent_id) + + def _prune(self, agent_id: str) -> None: + """Internal method to prune a specific agent's state based on TTL and capacity limits.""" + state = self._states.get(agent_id) + if not state: + return + + now = datetime.utcnow() + active_items = [] + + # 1. Filter out expired items based on TTL + for item in state.items: + expected_expiry = item.created_at + timedelta(seconds=item.ttl_seconds) + if now < expected_expiry: + active_items.append(item) + + # 2. If still over capacity, sort by importance (descending) then by freshness (descending) + if len(active_items) > state.max_items: + # We want to keep the highest importance / newest items + active_items.sort( + key=lambda x: (x.importance, x.created_at.timestamp()), reverse=True + ) + active_items = active_items[: state.max_items] + + # Re-sort temporally for the final state list (oldest to newest) + active_items.sort(key=lambda x: x.created_at.timestamp()) + + state.items = active_items + + def promote_item(self, agent_id: str, item_id: str, bonus: float = 0.1) -> None: + """Locally boost the importance of an item and refresh its creation time if accessed.""" + with self._lock: + state = self._states.get(agent_id) + if not state: + return + for item in state.items: + if item.id == item_id: + item.importance = min(1.0, item.importance + bonus) + # Extend TTL window by effectively refreshing created_at (LRU-like behavior) + item.created_at = datetime.utcnow() + break + diff --git a/src/mnemocore/mcp/adapters/api_adapter.py b/src/mnemocore/mcp/adapters/api_adapter.py index 84791a3ab29237a145d5e8c2acf532eb5665c2dc..4799900072692ec16b00de4c4ec4ab2c2752f6ca 100644 --- a/src/mnemocore/mcp/adapters/api_adapter.py +++ b/src/mnemocore/mcp/adapters/api_adapter.py @@ -82,3 +82,14 @@ class MnemoCoreAPIAdapter: def health(self) -> Dict[str, Any]: return self._request("GET", "/health") + + # --- Phase 5: Cognitive Client Adapters --- + + def observe_context(self, payload: Dict[str, Any]) -> Dict[str, Any]: + return self._request("POST", "/wm/observe", payload) + + def get_working_context(self, agent_id: str, limit: int = 16) -> Dict[str, Any]: + return self._request("GET", f"/wm/context/{agent_id}?limit={limit}") + + def start_episode(self, payload: Dict[str, Any]) -> Dict[str, Any]: + return self._request("POST", "/episodes/start", payload) diff --git a/src/mnemocore/mcp/schemas.py b/src/mnemocore/mcp/schemas.py index e31019ededbfe95f9dea4a09f793f6871eb170cc..23af7e15ef8d2c6411ecc86b5082c227b51baf22 100644 --- a/src/mnemocore/mcp/schemas.py +++ b/src/mnemocore/mcp/schemas.py @@ -22,6 +22,25 @@ class MemoryIdInput(BaseModel): memory_id: str = Field(..., min_length=1, max_length=256) +# --- Phase 5: Cognitive Client Schemas --- + +class ObserveToolInput(BaseModel): + agent_id: str = Field(..., min_length=1, max_length=256) + content: str = Field(..., min_length=1, max_length=100_000) + kind: str = Field(default="observation", max_length=64) + importance: float = Field(default=0.5, ge=0.0, le=1.0) + tags: Optional[list[str]] = None + +class ContextToolInput(BaseModel): + agent_id: str = Field(..., min_length=1, max_length=256) + limit: int = Field(default=16, ge=1, le=100) + +class EpisodeToolInput(BaseModel): + agent_id: str = Field(..., min_length=1, max_length=256) + goal: str = Field(..., min_length=1, max_length=10_000) + context: Optional[str] = None + + class ToolResult(BaseModel): ok: bool data: Optional[Dict[str, Any]] = None diff --git a/src/mnemocore/mcp/server.py b/src/mnemocore/mcp/server.py index a2e14957321c0f15bf04b17c51757754b843ed95..8cd7252229295bdd4aaf180f0189c2b5bdb9d46b 100644 --- a/src/mnemocore/mcp/server.py +++ b/src/mnemocore/mcp/server.py @@ -9,7 +9,10 @@ from loguru import logger from mnemocore.core.config import get_config, HAIMConfig from mnemocore.mcp.adapters.api_adapter import MnemoCoreAPIAdapter, MnemoCoreAPIError -from mnemocore.mcp.schemas import StoreToolInput, QueryToolInput, MemoryIdInput +from mnemocore.mcp.schemas import ( + StoreToolInput, QueryToolInput, MemoryIdInput, + ObserveToolInput, ContextToolInput, EpisodeToolInput +) from mnemocore.core.exceptions import ( DependencyMissingError, UnsupportedTransportError, @@ -110,12 +113,45 @@ def build_server(config: HAIMConfig | None = None): def memory_health() -> Dict[str, Any]: return with_error_handling(adapter.health) + # --- Phase 5: Cognitive Client Tools --- + + def register_store_observation() -> None: + @server.tool() + def store_observation( + agent_id: str, + content: str, + kind: str = "observation", + importance: float = 0.5, + tags: list[str] | None = None + ) -> Dict[str, Any]: + payload = ObserveToolInput( + agent_id=agent_id, content=content, kind=kind, importance=importance, tags=tags + ).model_dump(exclude_none=True) + return with_error_handling(lambda: adapter.observe_context(payload)) + + def register_recall_context() -> None: + @server.tool() + def recall_context(agent_id: str, limit: int = 16) -> Dict[str, Any]: + data = ContextToolInput(agent_id=agent_id, limit=limit) + return with_error_handling(lambda: adapter.get_working_context(data.agent_id, data.limit)) + + def register_start_episode() -> None: + @server.tool() + def start_episode(agent_id: str, goal: str, context: str | None = None) -> Dict[str, Any]: + payload = EpisodeToolInput( + agent_id=agent_id, goal=goal, context=context + ).model_dump(exclude_none=True) + return with_error_handling(lambda: adapter.start_episode(payload)) + register_tool("memory_store", register_memory_store) register_tool("memory_query", register_memory_query) register_tool("memory_get", register_memory_get) register_tool("memory_delete", register_memory_delete) register_tool("memory_stats", register_memory_stats) register_tool("memory_health", register_memory_health) + register_tool("store_observation", register_store_observation) + register_tool("recall_context", register_recall_context) + register_tool("start_episode", register_start_episode) return server diff --git a/tests/conftest.py b/tests/conftest.py index 28d5b87396ab12fbbd80bf10e2cc54a47a995274..c0a7aec49389e865396849a4b76a3aac417f1509 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,17 @@ +import sys +from pathlib import Path + import pytest from unittest.mock import MagicMock, patch, AsyncMock +# Ensure local src/ package imports work without editable install. +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + + # ============================================================================= # Mock Infrastructure Fixtures (Phase 3.5 - Offline Testing Support) # ============================================================================= diff --git a/tests/test_agent_interface.py b/tests/test_agent_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..a8f6f782fa4e1c7e7457e05711be2a7370500776 --- /dev/null +++ b/tests/test_agent_interface.py @@ -0,0 +1,89 @@ +import pytest +import asyncio +from unittest.mock import MagicMock +from mnemocore.agent_interface import CognitiveMemoryClient +from mnemocore.core.engine import HAIMEngine +from mnemocore.core.working_memory import WorkingMemoryService +from mnemocore.core.episodic_store import EpisodicStoreService +from mnemocore.core.semantic_store import SemanticStoreService +from mnemocore.core.procedural_store import ProceduralStoreService +from mnemocore.core.meta_memory import MetaMemoryService + +@pytest.fixture +def mock_engine(): + engine = MagicMock(spec=HAIMEngine) + engine.encoder = MagicMock() + return engine + +def test_cognitive_client_observe_and_context(mock_engine): + wm = WorkingMemoryService() + client = CognitiveMemoryClient( + engine=mock_engine, + wm=wm, + episodic=MagicMock(), + semantic=MagicMock(), + procedural=MagicMock(), + meta=MagicMock() + ) + + agent_id = "agent-alpha" + client.observe(agent_id, content="User said hi", importance=0.9) + client.observe(agent_id, content="User asked about weather", importance=0.7) + + ctx = client.get_working_context(agent_id) + assert len(ctx) == 2 + assert ctx[0].content == "User said hi" + +def test_cognitive_client_episodic(mock_engine): + episodic = EpisodicStoreService() + client = CognitiveMemoryClient( + engine=mock_engine, + wm=MagicMock(), + episodic=episodic, + semantic=MagicMock(), + procedural=MagicMock(), + meta=MagicMock() + ) + + agent_id = "agent-beta" + ep_id = client.start_episode(agent_id, goal="Greet user") + client.append_event(ep_id, kind="action", content="Said hello") + client.end_episode(ep_id, outcome="Success") + + recent = episodic.get_recent(agent_id) + assert len(recent) == 1 + assert recent[0].goal == "Greet user" + +@pytest.mark.asyncio +async def test_cognitive_client_recall(mock_engine): + episodic = EpisodicStoreService() + client = CognitiveMemoryClient( + engine=mock_engine, + wm=MagicMock(), + episodic=episodic, + semantic=MagicMock(), + procedural=MagicMock(), + meta=MagicMock() + ) + + agent_id = "agent-gamma" + ep_id = client.start_episode(agent_id, goal="Buy milk") + client.end_episode(ep_id, outcome="Success") + + # Mock engine query + mock_engine.query.return_value = [("mem-1", 0.9)] + + mock_node = MagicMock() + mock_node.content = "Semantic info about milk" + + async def mock_get_memory(mem_id): + return mock_node + + mock_engine.tier_manager = MagicMock() + mock_engine.tier_manager.get_memory = mock_get_memory + + results = await client.recall(agent_id, query="milk", modes=("episodic", "semantic")) + assert len(results) == 2 + sources = [r["source"] for r in results] + assert "episodic" in sources + assert "semantic/engine" in sources diff --git a/tests/test_anticipatory.py b/tests/test_anticipatory.py new file mode 100644 index 0000000000000000000000000000000000000000..04e818950764e303ac9edd47dfae1cfece25830e --- /dev/null +++ b/tests/test_anticipatory.py @@ -0,0 +1,78 @@ +import pytest +import pytest_asyncio +import os +import asyncio +from mnemocore.core.config import get_config, reset_config +from mnemocore.core.engine import HAIMEngine + +@pytest.fixture +def test_engine(tmp_path): + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None + reset_config() + + data_dir = tmp_path / "data" + data_dir.mkdir() + + os.environ["HAIM_DATA_DIR"] = str(data_dir) + os.environ["HAIM_ENCODING_MODE"] = "binary" + os.environ["HAIM_DIMENSIONALITY"] = "1024" + os.environ["HAIM_ANTICIPATORY_ENABLED"] = "True" + os.environ["HAIM_ANTICIPATORY_PREDICTIVE_DEPTH"] = "1" + os.environ["HAIM_MEMORY_FILE"] = str(tmp_path / "memory.jsonl") + os.environ["HAIM_LTP_INITIAL_IMPORTANCE"] = "1.0" + + reset_config() + engine = HAIMEngine() + yield engine + + del os.environ["HAIM_DATA_DIR"] + del os.environ["HAIM_ENCODING_MODE"] + del os.environ["HAIM_DIMENSIONALITY"] + del os.environ["HAIM_ANTICIPATORY_ENABLED"] + del os.environ["HAIM_ANTICIPATORY_PREDICTIVE_DEPTH"] + del os.environ["HAIM_MEMORY_FILE"] + del os.environ["HAIM_LTP_INITIAL_IMPORTANCE"] + reset_config() + +@pytest.mark.asyncio +async def test_anticipatory_memory(test_engine): + await test_engine.initialize() + + node_a = await test_engine.store("I love learning about Machine Learning and AI algorithms.") + node_b = await test_engine.store("Neural networks and deep learning models are very powerful.") + + # Force a connection between them by querying both + # or just relying on their semantic similarity + # Let's forcefully bind them + await test_engine.bind_memories(node_a, node_b, weight=10.0) + + # Demote node_b to WARM to test preloading + mem_b_obj = await test_engine.tier_manager.get_memory(node_b) + assert mem_b_obj is not None + # We spoof its ltp to demote it, then call promote/demote + mem_b_obj.ltp_strength = -1.0 + mem_b_obj.tier = "hot" + # Actually wait get_memory doesn't demote instantly unless threshold checks out, let's just forcefuly delete from hot + async with test_engine.tier_manager.lock: + if node_b in test_engine.tier_manager.hot: + del test_engine.tier_manager.hot[node_b] + test_engine.tier_manager._remove_from_faiss(node_b) + await test_engine.tier_manager._save_to_warm(mem_b_obj) + + # Verify node_b is in WARM + assert node_b not in test_engine.tier_manager.hot + + # Query for something exact to node_a to guarantee it ranks first + results = await test_engine.query("I love learning about Machine Learning and AI algorithms.", top_k=2) + # The Anticipatory Engine should spawn a background task to preload node_b + assert len(results) > 0 + assert results[0][0] == node_a + + # Wait a tiny bit for the async preloading task to complete + await asyncio.sleep(0.1) + + # Check if node_b is back in HOT + assert node_b in test_engine.tier_manager.hot, "Anticipatory engine failed to preload node_b." + + await test_engine.close() diff --git a/tests/test_api_functional.py b/tests/test_api_functional.py index 883fef890dedf2cfa18c36e76b001108e06adecd..f8506497528d6947f172f9111d69a4ae5fd16206 100644 --- a/tests/test_api_functional.py +++ b/tests/test_api_functional.py @@ -21,8 +21,8 @@ mock_container.redis_storage.check_health = AsyncMock(return_value=True) mock_container.qdrant_store = MagicMock() # Patch before import -patcher1 = patch("src.api.main.HAIMEngine", mock_engine_cls) -patcher2 = patch("src.api.main.build_container", return_value=mock_container) +patcher1 = patch("mnemocore.api.main.HAIMEngine", mock_engine_cls) +patcher2 = patch("mnemocore.api.main.build_container", return_value=mock_container) patcher1.start() patcher2.start() diff --git a/tests/test_api_security.py b/tests/test_api_security.py index e1e04c3b09df6808af7e0767d067b419e6172791..01512861b0a1b0495d9fddf49229c2f6c4964bb8 100644 --- a/tests/test_api_security.py +++ b/tests/test_api_security.py @@ -47,8 +47,8 @@ mock_redis_client.pipeline.return_value = mock_pipeline mock_container.redis_storage.redis_client = mock_redis_client # Patch before import -patcher1 = patch("src.api.main.HAIMEngine", mock_engine_cls) -patcher2 = patch("src.api.main.build_container", return_value=mock_container) +patcher1 = patch("mnemocore.api.main.HAIMEngine", mock_engine_cls) +patcher2 = patch("mnemocore.api.main.build_container", return_value=mock_container) patcher1.start() patcher2.start() diff --git a/tests/test_api_security_limits.py b/tests/test_api_security_limits.py index 5fd8938430c3fe53343a07774c2c0f0820c5fbb6..42ac9b57ab659beecd123d40a569d26e923775db 100644 --- a/tests/test_api_security_limits.py +++ b/tests/test_api_security_limits.py @@ -55,8 +55,8 @@ mock_redis_client.pipeline.return_value = mock_pipeline mock_container.redis_storage.redis_client = mock_redis_client # Patch before import -patcher1 = patch("src.api.main.HAIMEngine", mock_engine_cls) -patcher2 = patch("src.api.main.build_container", return_value=mock_container) +patcher1 = patch("mnemocore.api.main.HAIMEngine", mock_engine_cls) +patcher2 = patch("mnemocore.api.main.build_container", return_value=mock_container) patcher1.start() patcher2.start() diff --git a/tests/test_auto_bind.py b/tests/test_auto_bind.py new file mode 100644 index 0000000000000000000000000000000000000000..8857c599ae4284107d955f0a3f7019514e9790a4 --- /dev/null +++ b/tests/test_auto_bind.py @@ -0,0 +1,62 @@ +import pytest +import pytest_asyncio +import os +from mnemocore.core.config import get_config, reset_config +from mnemocore.core.engine import HAIMEngine + +@pytest.fixture +def test_engine(tmp_path): + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None + reset_config() + + data_dir = tmp_path / "data" + data_dir.mkdir() + + os.environ["HAIM_DATA_DIR"] = str(data_dir) + os.environ["HAIM_ENCODING_MODE"] = "binary" + os.environ["HAIM_DIMENSIONALITY"] = "1024" + os.environ["HAIM_SYNAPSE_SIMILARITY_THRESHOLD"] = "0.4" + os.environ["HAIM_SYNAPSE_AUTO_BIND_ON_STORE"] = "True" + os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] = "0.01" + os.environ["HAIM_LTP_INITIAL_IMPORTANCE"] = "0.8" + os.environ["HAIM_MEMORY_FILE"] = str(data_dir / "memory.jsonl") + os.environ["HAIM_CODEBOOK_FILE"] = str(data_dir / "codebook.json") + os.environ["HAIM_SYNAPSES_FILE"] = str(data_dir / "synapses.json") + os.environ["HAIM_WARM_MMAP_DIR"] = str(data_dir / "warm") + os.environ["HAIM_COLD_ARCHIVE_DIR"] = str(data_dir / "cold") + + reset_config() + engine = HAIMEngine() + yield engine + + del os.environ["HAIM_DATA_DIR"] + del os.environ["HAIM_ENCODING_MODE"] + del os.environ["HAIM_DIMENSIONALITY"] + del os.environ["HAIM_SYNAPSE_SIMILARITY_THRESHOLD"] + del os.environ["HAIM_SYNAPSE_AUTO_BIND_ON_STORE"] + del os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] + del os.environ["HAIM_LTP_INITIAL_IMPORTANCE"] + for key in ["HAIM_MEMORY_FILE", "HAIM_CODEBOOK_FILE", "HAIM_SYNAPSES_FILE", "HAIM_WARM_MMAP_DIR", "HAIM_COLD_ARCHIVE_DIR"]: + os.environ.pop(key, None) + reset_config() + +@pytest.mark.asyncio +async def test_auto_bind_on_store(test_engine): + await test_engine.initialize() + + # Store a memory + concept = "Python Memory Management and Garbage Collection" + mid1 = await test_engine.store(concept) + + # Store a very similar memory + concept2 = "Python Garbage Collection and Memory Management" + mid2 = await test_engine.store(concept2) + + # Check if a synapse was automatically formed + async with test_engine.synapse_lock: + syn = test_engine._synapse_index.get(mid1, mid2) + assert syn is not None, "Synapse should be auto-created between similar concepts" + assert syn.fire_count >= 1 + + await test_engine.close() diff --git a/tests/test_batch_ops.py b/tests/test_batch_ops.py index e2d5461c02b84a99ffd96a752c9346ed0a6ac13e..0fb4b44bf2c52a9a7a1793074de1bcae71b68c5f 100644 --- a/tests/test_batch_ops.py +++ b/tests/test_batch_ops.py @@ -20,7 +20,7 @@ class TestBatchOps(unittest.TestCase): def test_cpu_device_selection(self): """Verify fallback to CPU when GPU unavailable.""" - with patch("src.core.batch_ops.torch") as mock_torch: + with patch("mnemocore.core.batch_ops.torch") as mock_torch: mock_torch.cuda.is_available.return_value = False mock_torch.backends.mps.is_available.return_value = False bp = BatchProcessor(use_gpu=True) @@ -28,8 +28,8 @@ class TestBatchOps(unittest.TestCase): def test_gpu_device_selection(self): """Verify selection of CUDA when available.""" - with patch("src.core.batch_ops.torch") as mock_torch, \ - patch("src.core.batch_ops.TORCH_AVAILABLE", True): + with patch("mnemocore.core.batch_ops.torch") as mock_torch, \ + patch("mnemocore.core.batch_ops.TORCH_AVAILABLE", True): mock_torch.cuda.is_available.return_value = True mock_torch.backends.mps.is_available.return_value = False bp = BatchProcessor(use_gpu=True) @@ -71,7 +71,7 @@ class TestBatchOps(unittest.TestCase): self.assertEqual(dists[0, 1], 0) # q vs t2 (identical) self.assertGreater(dists[0, 0], 0) # q vs t1 (random) - @patch("src.core.batch_ops.torch") + @patch("mnemocore.core.batch_ops.torch") def test_search_gpu_mock(self, mock_torch): """Test GPU search logic flow (mocked tensor operations).""" # Configure mock torch behavior diff --git a/tests/test_confidence.py b/tests/test_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..c2698067e27d8b44b22ea041a2cb03d1da6b72bf --- /dev/null +++ b/tests/test_confidence.py @@ -0,0 +1,144 @@ +""" +Tests for Phase 5.0 Confidence Calibration Module. +Tests ConfidenceEnvelopeGenerator for all confidence levels and edge cases. +""" + +import pytest +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock + +from mnemocore.core.confidence import ( + ConfidenceEnvelopeGenerator, + build_confidence_envelope, + LEVEL_HIGH, + LEVEL_MEDIUM, + LEVEL_LOW, + LEVEL_CONTRADICTED, + LEVEL_STALE, +) +from mnemocore.core.provenance import ProvenanceRecord + + +# ------------------------------------------------------------------ # +# Helpers # +# ------------------------------------------------------------------ # + +def _make_node( + ltp_strength: float = 0.9, + access_count: int = 10, + days_old: float = 5.0, + bayes_mean: float | None = None, +): + """Create a minimal mock MemoryNode.""" + node = MagicMock() + node.ltp_strength = ltp_strength + node.access_count = access_count + now = datetime.now(timezone.utc) + node.last_accessed = now - timedelta(days=days_old) + # Optionally inject a Bayesian state mock + if bayes_mean is not None: + bayes = MagicMock() + bayes.mean = bayes_mean + node._bayes = bayes + else: + # Remove _bayes attribute so hasattr returns False + del node._bayes + return node + + +# ------------------------------------------------------------------ # +# ConfidenceEnvelopeGenerator # +# ------------------------------------------------------------------ # + +class TestConfidenceEnvelopeGenerator: + def test_high_confidence(self): + node = _make_node(ltp_strength=0.92, access_count=8, days_old=3) + prov = ProvenanceRecord.new(origin_type="observation") + env = ConfidenceEnvelopeGenerator.build(node, prov) + assert env["level"] == LEVEL_HIGH + assert env["reliability"] >= 0.80 + + def test_medium_confidence_low_reliability(self): + node = _make_node(ltp_strength=0.65, access_count=3, days_old=5) + env = ConfidenceEnvelopeGenerator.build(node) + assert env["level"] == LEVEL_MEDIUM + + def test_low_confidence_insufficient_access(self): + node = _make_node(ltp_strength=0.75, access_count=1, days_old=2) + env = ConfidenceEnvelopeGenerator.build(node) + assert env["level"] == LEVEL_LOW + + def test_low_confidence_poor_reliability(self): + node = _make_node(ltp_strength=0.35, access_count=10, days_old=2) + env = ConfidenceEnvelopeGenerator.build(node) + assert env["level"] == LEVEL_LOW + + def test_stale_overrides_high_reliability(self): + """A memory last verified 40 days ago should be STALE even if reliable.""" + node = _make_node(ltp_strength=0.98, access_count=20, days_old=40) + env = ConfidenceEnvelopeGenerator.build(node) + assert env["level"] == LEVEL_STALE + assert env["staleness_days"] >= 30 + + def test_contradicted_overrides_everything(self): + """Contradicted memories take priority over any reliability score.""" + node = _make_node(ltp_strength=0.99, access_count=100, days_old=1) + prov = ProvenanceRecord.new(origin_type="observation") + prov.mark_contradicted("cg_999") + env = ConfidenceEnvelopeGenerator.build(node, prov) + assert env["level"] == LEVEL_CONTRADICTED + assert env["is_contradicted"] is True + + def test_source_type_observation(self): + node = _make_node(ltp_strength=0.9, access_count=6) + prov = ProvenanceRecord.new(origin_type="observation") + env = ConfidenceEnvelopeGenerator.build(node, prov) + assert env["source_type"] == "observation" + assert env["source_trust"] == 1.0 + + def test_source_type_dream_lower_trust(self): + node = _make_node(ltp_strength=0.9, access_count=6) + prov = ProvenanceRecord.new(origin_type="dream") + env = ConfidenceEnvelopeGenerator.build(node, prov) + assert env["source_type"] == "dream" + # Dream trust is 0.6, should not reach HIGH level + assert env["level"] != LEVEL_HIGH + + def test_verified_event_resets_staleness(self): + """A fresh verification event should make staleness very short.""" + node = _make_node(ltp_strength=0.9, access_count=8, days_old=50) + prov = ProvenanceRecord.new(origin_type="observation") + prov.mark_verified(success=True) + env = ConfidenceEnvelopeGenerator.build(node, prov) + # Verified just now → staleness should be near 0 + assert env["staleness_days"] < 1.0 + assert env["level"] != LEVEL_STALE + + def test_bayesian_state_used_over_ltp(self): + """If node has _bayes, reliability = bayes.mean, not ltp_strength.""" + node = _make_node(ltp_strength=0.3, access_count=6, bayes_mean=0.95) + env = ConfidenceEnvelopeGenerator.build(node) + assert env["reliability"] == pytest.approx(0.95, abs=0.01) + + def test_no_provenance_uses_last_accessed(self): + node = _make_node(ltp_strength=0.85, access_count=7, days_old=5) + env = ConfidenceEnvelopeGenerator.build(node, provenance=None) + assert env["source_type"] == "unknown" + assert "level" in env + + def test_envelope_keys_present(self): + node = _make_node() + env = build_confidence_envelope(node) + expected_keys = { + "level", "reliability", "access_count", + "staleness_days", "source_type", "source_trust", "is_contradicted", + } + assert expected_keys.issubset(env.keys()) + + def test_module_shortcut_same_result(self): + node = _make_node(ltp_strength=0.88, access_count=6) + prov = ProvenanceRecord.new() + r1 = ConfidenceEnvelopeGenerator.build(node, prov) + r2 = build_confidence_envelope(node, prov) + assert r1["level"] == r2["level"] + assert r1["reliability"] == r2["reliability"] diff --git a/tests/test_config.py b/tests/test_config.py index b91262af748fda3be58f164bc368d6253d7d74d7..75699912b1b5a58c7831cc50b0d6b9fc2a70dd62 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -80,7 +80,7 @@ class TestLoadConfig: missing_path = tmp_path / "nonexistent.yaml" config = load_config(missing_path) assert config.dimensionality == 16384 - assert config.version == "3.0" + assert config.version == "4.5" def test_dimensionality_must_be_multiple_of_64(self, tmp_path): bad_config = {"haim": {"dimensionality": 100}} diff --git a/tests/test_consolidation_worker.py b/tests/test_consolidation_worker.py index be8200a46abc2110452f007ba0e0c13d47ce90aa..62dac9222f80f2e030c62f9c71487d3914518ab0 100644 --- a/tests/test_consolidation_worker.py +++ b/tests/test_consolidation_worker.py @@ -14,9 +14,9 @@ class TestConsolidationWorker(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): # Patch dependencies - self.storage_patcher = patch('src.core.consolidation_worker.AsyncRedisStorage') - self.tier_manager_patcher = patch('src.core.consolidation_worker.TierManager') - self.config_patcher = patch('src.core.consolidation_worker.get_config') + self.storage_patcher = patch('mnemocore.core.consolidation_worker.AsyncRedisStorage') + self.tier_manager_patcher = patch('mnemocore.core.consolidation_worker.TierManager') + self.config_patcher = patch('mnemocore.core.consolidation_worker.get_config') self.MockStorage = self.storage_patcher.start() self.MockTierManager = self.tier_manager_patcher.start() diff --git a/tests/test_context_awareness.py b/tests/test_context_awareness.py new file mode 100644 index 0000000000000000000000000000000000000000..efa6b7f85d0778e3a8b47a9c2215a0e4b2ffd666 --- /dev/null +++ b/tests/test_context_awareness.py @@ -0,0 +1,66 @@ +import pytest +import pytest_asyncio +import os +from mnemocore.core.config import get_config, reset_config +from mnemocore.core.engine import HAIMEngine + +@pytest.fixture +def test_engine(tmp_path): + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None + reset_config() + + data_dir = tmp_path / "data" + data_dir.mkdir() + + os.environ["HAIM_DATA_DIR"] = str(data_dir) + os.environ["HAIM_ENCODING_MODE"] = "binary" + os.environ["HAIM_DIMENSIONALITY"] = "1024" + os.environ["HAIM_CONTEXT_ENABLED"] = "True" + # Unrelated binary vectors have similarity ~0.5. + os.environ["HAIM_CONTEXT_SHIFT_THRESHOLD"] = "0.6" + os.environ["HAIM_CONTEXT_ROLLING_WINDOW_SIZE"] = "3" + os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] = "0.01" + + reset_config() + engine = HAIMEngine() + yield engine + + del os.environ["HAIM_DATA_DIR"] + del os.environ["HAIM_ENCODING_MODE"] + del os.environ["HAIM_DIMENSIONALITY"] + del os.environ["HAIM_CONTEXT_ENABLED"] + del os.environ["HAIM_CONTEXT_SHIFT_THRESHOLD"] + del os.environ["HAIM_CONTEXT_ROLLING_WINDOW_SIZE"] + del os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] + reset_config() + +@pytest.mark.asyncio +async def test_topic_tracker_and_context(test_engine): + await test_engine.initialize() + + # Store some nodes for retrieval + await test_engine.store("I love learning about Machine Learning and AI algorithms.") + await test_engine.store("Neural networks and deep learning models are very powerful.") + await test_engine.store("Pineapples belong on pizza. Yes, it is true.") + + # 1. Querying related to AI multiple times to build context + await test_engine.query("Tell me about Machine Learning.") + await test_engine.query("How do Neural Networks work?") + + # Check context nodes for AI context + ctx_nodes1 = await test_engine.get_context_nodes(top_k=2) + assert len(ctx_nodes1) > 0, "Should retrieve context nodes" + + # 2. Shift topic drastically + await test_engine.query("What about pizza toppings? Do pineapples belong?") + + # The history should have reset due to shift drop + assert len(test_engine.topic_tracker.history) == 1 + + # The retrieved context nodes should now be heavily biased toward Pizza + ctx_nodes2 = await test_engine.get_context_nodes(top_k=1) + + assert ctx_nodes2[0][0] != ctx_nodes1[0][0], "Context should have decisively shifted" + + await test_engine.close() diff --git a/tests/test_contradiction.py b/tests/test_contradiction.py new file mode 100644 index 0000000000000000000000000000000000000000..fb400dc9c2614c4a055dacfacbf4505cae6cd5af --- /dev/null +++ b/tests/test_contradiction.py @@ -0,0 +1,180 @@ +""" +Tests for Phase 5.0 Contradiction Detection Module. +""" + +import pytest +from unittest.mock import MagicMock, AsyncMock +from datetime import datetime, timezone + +from mnemocore.core.contradiction import ( + ContradictionRecord, + ContradictionRegistry, + ContradictionDetector, + get_contradiction_detector, +) +from mnemocore.core.provenance import ProvenanceRecord + + +# ------------------------------------------------------------------ # +# Helpers # +# ------------------------------------------------------------------ # + +def _make_node(memory_id: str, content: str = "test content", hdv_data: bytes = None): + import numpy as np + node = MagicMock() + node.id = memory_id + node.content = content + # Create a fake HDV with random binary data + data = np.frombuffer(hdv_data or bytes(2048), dtype=np.uint8) if hdv_data else np.zeros(2048, dtype=np.uint8) + node.hdv = MagicMock() + node.hdv.data = data + node.metadata = {} + node.provenance = None + return node + + +# ------------------------------------------------------------------ # +# ContradictionRecord # +# ------------------------------------------------------------------ # + +class TestContradictionRecord: + def test_auto_group_id(self): + r = ContradictionRecord(memory_a_id="a", memory_b_id="b") + assert r.group_id.startswith("cg_") + + def test_to_dict(self): + r = ContradictionRecord(memory_a_id="mem_a", memory_b_id="mem_b", similarity_score=0.87) + d = r.to_dict() + assert d["memory_a_id"] == "mem_a" + assert d["similarity_score"] == pytest.approx(0.87, abs=0.001) + assert d["resolved"] is False + + +# ------------------------------------------------------------------ # +# ContradictionRegistry # +# ------------------------------------------------------------------ # + +class TestContradictionRegistry: + def test_register_and_list(self): + reg = ContradictionRegistry() + rec = ContradictionRecord(memory_a_id="a", memory_b_id="b") + reg.register(rec) + all_recs = reg.list_all(unresolved_only=False) + assert len(all_recs) == 1 + + def test_unresolved_only(self): + reg = ContradictionRegistry() + r1 = ContradictionRecord(memory_a_id="a", memory_b_id="b") + r2 = ContradictionRecord(memory_a_id="c", memory_b_id="d") + reg.register(r1) + reg.register(r2) + reg.resolve(r1.group_id, note="fixed") + unresolved = reg.list_all(unresolved_only=True) + assert len(unresolved) == 1 + assert unresolved[0].group_id == r2.group_id + + def test_resolve_unknown_returns_false(self): + reg = ContradictionRegistry() + assert reg.resolve("cg_nonexistent") is False + + def test_resolve_sets_note(self): + reg = ContradictionRegistry() + r = ContradictionRecord(memory_a_id="a", memory_b_id="b") + reg.register(r) + reg.resolve(r.group_id, note="duplicate info") + assert reg._records[r.group_id].resolution_note == "duplicate info" + + def test_list_for_memory(self): + reg = ContradictionRegistry() + r = ContradictionRecord(memory_a_id="mem_x", memory_b_id="mem_y") + reg.register(r) + found = reg.list_for_memory("mem_x") + assert len(found) == 1 + not_found = reg.list_for_memory("mem_z") + assert len(not_found) == 0 + + def test_len_counts_unresolved(self): + reg = ContradictionRegistry() + r1 = ContradictionRecord(memory_a_id="a", memory_b_id="b") + r2 = ContradictionRecord(memory_a_id="c", memory_b_id="d") + reg.register(r1) + reg.register(r2) + assert len(reg) == 2 + reg.resolve(r1.group_id) + assert len(reg) == 1 + + +# ------------------------------------------------------------------ # +# ContradictionDetector.hamming_similarity # +# ------------------------------------------------------------------ # + +class TestHammingSimilarity: + def test_identical_nodes_similarity_one(self): + import numpy as np + data = np.random.randint(0, 255, 2048, dtype=np.uint8).tobytes() + a = _make_node("a", hdv_data=data) + b = _make_node("b", hdv_data=data) + detector = ContradictionDetector(engine=None, use_llm=False) + sim = detector._hamming_similarity(a, b) + assert sim == pytest.approx(1.0, abs=1e-5) + + def test_random_nodes_similarity_near_half(self): + """Random binary vectors should have ~50% similarity (Hamming).""" + import numpy as np + rng = np.random.default_rng(42) + a = _make_node("a", hdv_data=rng.integers(0, 255, 2048, dtype=np.uint8).tobytes()) + b = _make_node("b", hdv_data=rng.integers(0, 255, 2048, dtype=np.uint8).tobytes()) + detector = ContradictionDetector(engine=None, use_llm=False) + sim = detector._hamming_similarity(a, b) + # Should be around 0.5 ± 0.1 + assert 0.4 < sim < 0.6 + + +# ------------------------------------------------------------------ # +# check_on_store # +# ------------------------------------------------------------------ # + +class TestCheckOnStore: + @pytest.mark.asyncio + async def test_no_contradiction_when_candidates_empty(self): + detector = ContradictionDetector(engine=None, use_llm=False) + node = _make_node("new_mem") + result = await detector.check_on_store(node, candidates=[]) + assert result is None + + @pytest.mark.asyncio + async def test_high_similarity_without_llm_flags_very_high(self): + """Without LLM, only similarity >= 0.90 counts as contradiction.""" + import numpy as np + identical_data = bytes(2048) + new_node = _make_node("new", hdv_data=identical_data) + existing = _make_node("old", hdv_data=identical_data) + new_node.provenance = ProvenanceRecord.new(origin_type="observation") + existing.provenance = ProvenanceRecord.new(origin_type="observation") + + detector = ContradictionDetector( + engine=None, + use_llm=False, + similarity_threshold=0.80, + ) + result = await detector.check_on_store(new_node, candidates=[existing]) + # Identical nodes have similarity=1.0 → contradiction without LLM + assert result is not None + assert result.memory_a_id == "new" + assert result.memory_b_id == "old" + + @pytest.mark.asyncio + async def test_contradiction_flags_provenance(self): + import numpy as np + identical_data = bytes(2048) + new_node = _make_node("new2", hdv_data=identical_data) + existing = _make_node("old2", hdv_data=identical_data) + new_node.provenance = ProvenanceRecord.new(origin_type="observation") + existing.provenance = ProvenanceRecord.new(origin_type="observation") + + detector = ContradictionDetector(engine=None, use_llm=False, similarity_threshold=0.80) + result = await detector.check_on_store(new_node, candidates=[existing]) + if result: + # Both nodes should be flagged + assert "contradiction_group_id" in new_node.metadata + assert new_node.provenance.is_contradicted() diff --git a/tests/test_daemon_perf.py b/tests/test_daemon_perf.py index 204e6f19861be479626c53c3c316a6fbb23ca1c9..acbe044a4c7e2e18140049f9b97223931b5d6905 100644 --- a/tests/test_daemon_perf.py +++ b/tests/test_daemon_perf.py @@ -24,23 +24,23 @@ except ImportError: pass # Mock dependencies if they are not importable -if "src.core.engine" not in sys.modules: - mock_module("src.core") - mock_module("src.core.engine") - sys.modules["src.core.engine"].HAIMEngine = MagicMock() - mock_module("src.core.node") - sys.modules["src.core.node"].MemoryNode = MagicMock() - mock_module("src.core.qdrant_store") - sys.modules["src.core.qdrant_store"].QdrantStore = MagicMock() - -if "src.core.async_storage" not in sys.modules: - mock_module("src.core.async_storage") - sys.modules["src.core.async_storage"].AsyncRedisStorage = MagicMock() - -if "src.meta.learning_journal" not in sys.modules: - mock_module("src.meta") - mock_module("src.meta.learning_journal") - sys.modules["src.meta.learning_journal"].LearningJournal = MagicMock() +if "mnemocore.core.engine" not in sys.modules: + mock_module("mnemocore.core") + mock_module("mnemocore.core.engine") + sys.modules["mnemocore.core.engine"].HAIMEngine = MagicMock() + mock_module("mnemocore.core.node") + sys.modules["mnemocore.core.node"].MemoryNode = MagicMock() + mock_module("mnemocore.core.qdrant_store") + sys.modules["mnemocore.core.qdrant_store"].QdrantStore = MagicMock() + +if "mnemocore.core.async_storage" not in sys.modules: + mock_module("mnemocore.core.async_storage") + sys.modules["mnemocore.core.async_storage"].AsyncRedisStorage = MagicMock() + +if "mnemocore.meta.learning_journal" not in sys.modules: + mock_module("mnemocore.meta") + mock_module("mnemocore.meta.learning_journal") + sys.modules["mnemocore.meta.learning_journal"].LearningJournal = MagicMock() if "aiohttp" not in sys.modules: mock_module("aiohttp") @@ -60,7 +60,7 @@ async def _async_test_save_evolution_state_non_blocking(): daemon = SubconsciousDaemon() # Use a temp path for the state file to avoid permission issues - with patch("src.subconscious.daemon.EVOLUTION_STATE_PATH", "/tmp/test_evolution_perf.json"): + with patch("mnemocore.subconscious.daemon.EVOLUTION_STATE_PATH", "/tmp/test_evolution_perf.json"): # 2. Patch json.dump to be slow (simulate blocking I/O) # We need to patch it where it is used. daemon.py imports json. diff --git a/tests/test_di_migration.py b/tests/test_di_migration.py index fbc7f0963e466a170a1459488460a065f2abde37..61ecf4e4d77b50cd9eb41520c1c415b349e7be93 100644 --- a/tests/test_di_migration.py +++ b/tests/test_di_migration.py @@ -89,8 +89,8 @@ class TestContainer: # Create a minimal config config = HAIMConfig() - with patch('src.core.container.AsyncRedisStorage') as mock_redis_class, \ - patch('src.core.container.QdrantStore') as mock_qdrant_class: + with patch('mnemocore.core.container.AsyncRedisStorage') as mock_redis_class, \ + patch('mnemocore.core.container.QdrantStore') as mock_qdrant_class: mock_redis_class.return_value = MagicMock() mock_qdrant_class.return_value = MagicMock() @@ -124,8 +124,8 @@ class TestTierManagerDI: config = HAIMConfig() - with patch('src.core.tier_manager.HNSW_AVAILABLE', False), \ - patch('src.core.tier_manager.FAISS_AVAILABLE', False): + with patch('mnemocore.core.tier_manager.HNSW_AVAILABLE', False), \ + patch('mnemocore.core.tier_manager.FAISS_AVAILABLE', False): manager = TierManager(config=config) assert manager.config is config @@ -138,8 +138,8 @@ class TestTierManagerDI: config = HAIMConfig() mock_qdrant = MagicMock() - with patch('src.core.tier_manager.HNSW_AVAILABLE', False), \ - patch('src.core.tier_manager.FAISS_AVAILABLE', False): + with patch('mnemocore.core.tier_manager.HNSW_AVAILABLE', False), \ + patch('mnemocore.core.tier_manager.FAISS_AVAILABLE', False): manager = TierManager(config=config, qdrant_store=mock_qdrant) assert manager.qdrant is mock_qdrant @@ -157,8 +157,8 @@ class TestHAIMEngineDI: config = HAIMConfig() # Patch at tier_manager level since that's where HNSW/FAISS is used - with patch('src.core.tier_manager.HNSW_AVAILABLE', False), \ - patch('src.core.tier_manager.FAISS_AVAILABLE', False): + with patch('mnemocore.core.tier_manager.HNSW_AVAILABLE', False), \ + patch('mnemocore.core.tier_manager.FAISS_AVAILABLE', False): engine = HAIMEngine(config=config) assert engine.config is config @@ -171,8 +171,8 @@ class TestHAIMEngineDI: config = HAIMConfig() - with patch('src.core.tier_manager.HNSW_AVAILABLE', False), \ - patch('src.core.tier_manager.FAISS_AVAILABLE', False): + with patch('mnemocore.core.tier_manager.HNSW_AVAILABLE', False), \ + patch('mnemocore.core.tier_manager.FAISS_AVAILABLE', False): tier_manager = TierManager(config=config) engine = HAIMEngine(config=config, tier_manager=tier_manager) diff --git a/tests/test_dream_loop.py b/tests/test_dream_loop.py index e04a1058bd9f5e8b673c2f883ac2cfe39fc2713c..b949c11c4b7b81da0782e196542f99844f954e30 100644 --- a/tests/test_dream_loop.py +++ b/tests/test_dream_loop.py @@ -119,11 +119,11 @@ def daemon_module(): # Patch sys.modules to inject mocks before import patches = { 'aiohttp': mock_aiohttp, - 'src.subconscious.daemon.aiohttp': mock_aiohttp, - 'src.subconscious.daemon.DREAM_LOOP_TOTAL': mock_dream_loop_total, - 'src.subconscious.daemon.DREAM_LOOP_ITERATION_SECONDS': mock_dream_loop_iteration_seconds, - 'src.subconscious.daemon.DREAM_LOOP_INSIGHTS_GENERATED': mock_dream_loop_insights, - 'src.subconscious.daemon.DREAM_LOOP_ACTIVE': mock_dream_loop_active, + 'mnemocore.subconscious.daemon.aiohttp': mock_aiohttp, + 'mnemocore.subconscious.daemon.DREAM_LOOP_TOTAL': mock_dream_loop_total, + 'mnemocore.subconscious.daemon.DREAM_LOOP_ITERATION_SECONDS': mock_dream_loop_iteration_seconds, + 'mnemocore.subconscious.daemon.DREAM_LOOP_INSIGHTS_GENERATED': mock_dream_loop_insights, + 'mnemocore.subconscious.daemon.DREAM_LOOP_ACTIVE': mock_dream_loop_active, } # Apply patches to sys.modules @@ -134,8 +134,8 @@ def daemon_module(): sys.modules[key] = value # Remove daemon from sys.modules if it exists to force reload - if 'src.subconscious.daemon' in sys.modules: - del sys.modules['src.subconscious.daemon'] + if 'mnemocore.subconscious.daemon' in sys.modules: + del sys.modules['mnemocore.subconscious.daemon'] try: import mnemocore.subconscious.daemon as dm @@ -148,8 +148,8 @@ def daemon_module(): elif key in sys.modules: del sys.modules[key] # Clean up daemon module - if 'src.subconscious.daemon' in sys.modules: - del sys.modules['src.subconscious.daemon'] + if 'mnemocore.subconscious.daemon' in sys.modules: + del sys.modules['mnemocore.subconscious.daemon'] class TestDreamLoopStartsAndStops: diff --git a/tests/test_e2e_flow.py b/tests/test_e2e_flow.py index 9b9d20ce7b04bbfd2b3fea6f4dde70fe4245b301..3a2fba9f962ed932a9de53d41c09e5d34ce797fb 100644 --- a/tests/test_e2e_flow.py +++ b/tests/test_e2e_flow.py @@ -34,6 +34,8 @@ def isolated_engine(tmp_path): (new memories have LTP ~0.55, below the default threshold of 0.7) - HAIM_HOT_MAX_MEMORIES=10000 → prevents eviction during tests """ + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None reset_config() data_dir = tmp_path / "data" data_dir.mkdir() diff --git a/tests/test_emotional_tag.py b/tests/test_emotional_tag.py new file mode 100644 index 0000000000000000000000000000000000000000..f16aca38dfe3fd2b9b8b84929395d0b4ef6d1053 --- /dev/null +++ b/tests/test_emotional_tag.py @@ -0,0 +1,119 @@ +""" +Tests for Phase 5.0 Emotional Tagging Module. +""" + +import pytest +from mnemocore.core.emotional_tag import ( + EmotionalTag, + attach_emotional_tag, + get_emotional_tag, +) +from unittest.mock import MagicMock + + +# ------------------------------------------------------------------ # +# EmotionalTag construction and clamping # +# ------------------------------------------------------------------ # + +class TestEmotionalTag: + def test_default_neutral(self): + t = EmotionalTag() + assert t.valence == 0.0 + assert t.arousal == 0.0 + + def test_clamping_valence_upper(self): + t = EmotionalTag(valence=2.0) + assert t.valence == 1.0 + + def test_clamping_valence_lower(self): + t = EmotionalTag(valence=-5.0) + assert t.valence == -1.0 + + def test_clamping_arousal_lower(self): + t = EmotionalTag(arousal=-0.5) + assert t.arousal == 0.0 + + def test_clamping_arousal_upper(self): + t = EmotionalTag(arousal=99.0) + assert t.arousal == 1.0 + + def test_salience_zero_for_neutral(self): + t = EmotionalTag.neutral() + assert t.salience() == pytest.approx(0.0) + + def test_salience_calculation(self): + t = EmotionalTag(valence=0.8, arousal=0.5) + expected = abs(0.8) * 0.5 + assert t.salience() == pytest.approx(expected) + + def test_salience_negative_valence(self): + t = EmotionalTag(valence=-1.0, arousal=1.0) + assert t.salience() == pytest.approx(1.0) + + def test_is_emotionally_significant(self): + t = EmotionalTag(valence=0.8, arousal=0.9) + assert t.is_emotionally_significant(threshold=0.3) + + def test_not_significant_when_calm(self): + t = EmotionalTag(valence=0.5, arousal=0.1) + # 0.5 * 0.1 = 0.05 < 0.3 + assert not t.is_emotionally_significant(threshold=0.3) + + def test_high_positive_factory(self): + t = EmotionalTag.high_positive() + assert t.valence == 1.0 + assert t.arousal == 1.0 + assert t.salience() == pytest.approx(1.0) + + def test_high_negative_factory(self): + t = EmotionalTag.high_negative() + assert t.valence == -1.0 + assert t.salience() == pytest.approx(1.0) + + def test_to_metadata_dict_keys(self): + t = EmotionalTag(valence=0.6, arousal=0.7) + d = t.to_metadata_dict() + assert "emotional_valence" in d + assert "emotional_arousal" in d + assert "emotional_salience" in d + + def test_from_metadata_roundtrip(self): + t = EmotionalTag(valence=-0.5, arousal=0.8) + d = t.to_metadata_dict() + restored = EmotionalTag.from_metadata(d) + assert restored.valence == pytest.approx(-0.5, abs=0.001) + assert restored.arousal == pytest.approx(0.8, abs=0.001) + + def test_repr(self): + t = EmotionalTag(valence=0.7, arousal=0.4) + r = repr(t) + assert "valence" in r + assert "arousal" in r + + +# ------------------------------------------------------------------ # +# Node helpers # +# ------------------------------------------------------------------ # + +class TestNodeHelpers: + def test_attach_writes_metadata(self): + node = MagicMock() + node.metadata = {} + tag = EmotionalTag(valence=0.9, arousal=0.8) + attach_emotional_tag(node, tag) + assert node.metadata["emotional_valence"] == pytest.approx(0.9) + assert node.metadata["emotional_arousal"] == pytest.approx(0.8) + + def test_get_returns_neutral_when_empty(self): + node = MagicMock() + node.metadata = {} + tag = get_emotional_tag(node) + assert tag.valence == 0.0 + assert tag.arousal == 0.0 + + def test_get_reads_existing_metadata(self): + node = MagicMock() + node.metadata = {"emotional_valence": -0.3, "emotional_arousal": 0.6} + tag = get_emotional_tag(node) + assert tag.valence == pytest.approx(-0.3) + assert tag.arousal == pytest.approx(0.6) diff --git a/tests/test_engine_binary.py b/tests/test_engine_binary.py index 66564b02ebc06e2a3540a54cf2d42526e9f2896e..92f6be3361fe2f1054ca1b9e4bc38d42d763fb70 100644 --- a/tests/test_engine_binary.py +++ b/tests/test_engine_binary.py @@ -19,6 +19,8 @@ from mnemocore.core.node import MemoryNode @pytest.fixture def binary_engine(tmp_path): + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None reset_config() data_dir = tmp_path / "data" data_dir.mkdir() @@ -57,6 +59,7 @@ class TestBinaryEngine: assert isinstance(binary_engine.tier_manager, object) async def test_store_memory_binary(self, binary_engine): + await binary_engine.initialize() mid = await binary_engine.store("Hello World", metadata={"test": True}) # Verify stored in HOT @@ -70,6 +73,7 @@ class TestBinaryEngine: assert os.path.exists(binary_engine.persist_path) async def test_query_memory_binary(self, binary_engine): + await binary_engine.initialize() # Store two distinct memories mid1 = await binary_engine.store("The quick brown fox jumps over the lazy dog") mid2 = await binary_engine.store("Quantum computing uses qubits and superposition") @@ -83,6 +87,7 @@ class TestBinaryEngine: assert score > 0.5 # Should be high similarity async def test_context_vector_binary(self, binary_engine): + await binary_engine.initialize() await binary_engine.store("Context 1") await binary_engine.store("Context 2") @@ -101,6 +106,7 @@ class TestBinaryEngine: class TestRouterBinary: async def test_router_reflex(self, binary_engine): + await binary_engine.initialize() router = CognitiveRouter(binary_engine) await binary_engine.store("What is HAIM?", metadata={"answer": "Holographic memory"}) diff --git a/tests/test_engine_cleanup.py b/tests/test_engine_cleanup.py index d9d2b39bb1c2d368cb6c2fae33ed14c6c24cf2e6..a11ecbe54a580c3e06843d3d9c705bb05c7385cf 100644 --- a/tests/test_engine_cleanup.py +++ b/tests/test_engine_cleanup.py @@ -32,6 +32,7 @@ def test_engine(tmp_path): @pytest.mark.asyncio async def test_cleanup_decay(test_engine): + await test_engine.initialize() # Add dummy synapses # Synapse 1: Weak (below threshold 0.1) syn1 = SynapticConnection("mem_1", "mem_2", initial_strength=0.05) @@ -48,19 +49,20 @@ async def test_cleanup_decay(test_engine): await test_engine.cleanup_decay(threshold=0.1) # Verify results - assert len(test_engine.synapses) == 1 - assert ("mem_3", "mem_4") in test_engine.synapses - assert ("mem_1", "mem_2") not in test_engine.synapses + assert len(test_engine._synapse_index) == 1 + assert test_engine._synapse_index.get("mem_3", "mem_4") is not None + assert test_engine._synapse_index.get("mem_1", "mem_2") is None # Verify persistence assert os.path.exists(test_engine.synapse_path) @pytest.mark.asyncio async def test_cleanup_no_decay(test_engine): + await test_engine.initialize() # All strong syn1 = SynapticConnection("mem_1", "mem_2", initial_strength=0.5) test_engine.synapses[("mem_1", "mem_2")] = syn1 await test_engine.cleanup_decay(threshold=0.1) - assert len(test_engine.synapses) == 1 + assert len(test_engine._synapse_index) == 1 diff --git a/tests/test_episodic_store.py b/tests/test_episodic_store.py new file mode 100644 index 0000000000000000000000000000000000000000..57a51f18f164783de89d67772b511c2779e04096 --- /dev/null +++ b/tests/test_episodic_store.py @@ -0,0 +1,62 @@ +import pytest +from datetime import datetime +from mnemocore.core.episodic_store import EpisodicStoreService + +def test_episodic_store_flow(): + store = EpisodicStoreService() + agent_id = "agent-x" + + # Start episode + ep_id = store.start_episode(agent_id, goal="Find the keys", context="Living room") + assert ep_id is not None + + # Append events + store.append_event(ep_id, kind="action", content="Looked under the sofa", metadata={"location": "sofa"}) + store.append_event(ep_id, kind="observation", content="Found nothing") + + # End episode + store.end_episode(ep_id, outcome="Failed", reward=-1.0) + + # Retrieve recent + recent = store.get_recent(agent_id, limit=2) + assert len(recent) == 1 + + ep = recent[0] + assert ep.id == ep_id + assert ep.agent_id == agent_id + assert ep.goal == "Find the keys" + assert ep.outcome == "Failed" + assert ep.reward == -1.0 + assert len(ep.events) == 2 + assert ep.events[0].kind == "action" + assert ep.events[0].content == "Looked under the sofa" + +def test_episodic_store_context_filtering(): + store = EpisodicStoreService() + agent_id = "agent-x" + + ep1 = store.start_episode(agent_id, goal="Task A", context="ctx1") + store.end_episode(ep1, outcome="Success") + + ep2 = store.start_episode(agent_id, goal="Task B", context="ctx2") + store.end_episode(ep2, outcome="Success") + + recent_ctx1 = store.get_recent(agent_id, context="ctx1") + assert len(recent_ctx1) == 1 + assert recent_ctx1[0].goal == "Task A" + +def test_episodic_eviction(): + store = EpisodicStoreService() + agent_id = "agent-x" + + import time + for i in range(3): + ep_id = store.start_episode(agent_id, goal=f"Goal {i}") + time.sleep(0.02) + store.end_episode(ep_id, outcome="Done") + + recent = store.get_recent(agent_id, limit=2) + assert len(recent) == 2 + # Should contain Goal 2 and Goal 1, Goal 0 evicted visually in get_recent limit + assert recent[0].goal == "Goal 2" + assert recent[1].goal == "Goal 1" diff --git a/tests/test_hnsw_index_scores.py b/tests/test_hnsw_index_scores.py new file mode 100644 index 0000000000000000000000000000000000000000..0a4e399f7dbe0bd251fbf467c369debaa1e2b0fb --- /dev/null +++ b/tests/test_hnsw_index_scores.py @@ -0,0 +1,36 @@ +import numpy as np + +from mnemocore.core.hnsw_index import HNSWIndexManager +import mnemocore.core.hnsw_index as hnsw_module + + +class _FakeIndex: + def __init__(self, d: int, distances, ids): + self.d = d + self._distances = np.asarray(distances, dtype=np.int32) + self._ids = np.asarray(ids, dtype=np.int64) + self.last_query_shape = None + + def search(self, q, k): + self.last_query_shape = q.shape + return self._distances[:, :k], self._ids[:, :k] + + +def test_search_uses_index_dimension_and_clamps_scores(monkeypatch): + manager = HNSWIndexManager.__new__(HNSWIndexManager) + manager.dimension = 16384 + manager._id_map = ["n1", "n2"] + manager._stale_count = 0 + manager._index = _FakeIndex(d=10000, distances=[[12000, 5000]], ids=[[0, 1]]) + + monkeypatch.setattr(hnsw_module, "FAISS_AVAILABLE", True) + + # Query is from a 16384-bit service (2048 bytes), but index is 10000-bit (1250 bytes) + query = np.zeros(2048, dtype=np.uint8) + results = manager.search(query, top_k=2) + + # Query should be adjusted to index byte width + assert manager._index.last_query_shape == (1, 1250) + + assert results[0] == ("n1", 0.0) # 1 - 12000/10000 => clamped to 0.0 + assert results[1] == ("n2", 0.5) # 1 - 5000/10000 diff --git a/tests/test_meta_memory.py b/tests/test_meta_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac64c3dafd0881b3107e835c020a706da951879 --- /dev/null +++ b/tests/test_meta_memory.py @@ -0,0 +1,47 @@ +import pytest +from mnemocore.core.meta_memory import MetaMemoryService + +def test_meta_memory_metrics(): + meta = MetaMemoryService() + + meta.record_metric("inference_time_ms", 120.5, window="1m") + meta.record_metric("inference_time_ms", 130.0, window="1m") + meta.record_metric("token_count", 500, window="1h") + + assert len([m for m in meta.list_metrics() if m.name == "inference_time_ms"]) == 2 + assert len([m for m in meta.list_metrics() if m.name == "token_count"]) == 1 + +import pytest +from datetime import datetime +from mnemocore.core.meta_memory import MetaMemoryService +from mnemocore.core.memory_model import SelfImprovementProposal + +def test_meta_memory_proposals(): + meta = MetaMemoryService() + + proposal = SelfImprovementProposal( + id="prop1", + created_at=datetime.utcnow(), + author="system", + title="Reduce Temp", + description="Agent repeats tools too often", + rationale="Reduce temperature or augment prompt with history", + expected_effect="Less tool looping", + status="pending", + metadata={"confidence": 0.85} + ) + + meta.create_proposal(proposal) + + proposals = meta.list_proposals() + assert len(proposals) == 1 + + p = proposals[0] + assert p.status == "pending" + assert "reduce temperature" in p.rationale.lower() + + # Update status + meta.update_proposal_status(p.id, "approved") + + proposals2 = meta.list_proposals() + assert proposals2[0].status == "approved" diff --git a/tests/test_multi_hop.py b/tests/test_multi_hop.py new file mode 100644 index 0000000000000000000000000000000000000000..04ca0cc75181dd1f8523dbd3aa82844e149fa8d5 --- /dev/null +++ b/tests/test_multi_hop.py @@ -0,0 +1,39 @@ +import pytest +import numpy as np +from mnemocore.core.synapse import SynapticConnection +from mnemocore.core.synapse_index import SynapseIndex + +def test_get_multi_hop_neighbors(): + index = SynapseIndex() + + # Create nodes A, B, C, D + # A <-> B (weight 0.8) + # B <-> C (weight 0.5) + # C <-> D (weight 0.9) + # A <-> C (weight 0.1) + + syn_ab = index.add_or_fire("A", "B") + syn_ab.strength = 0.8 + + syn_bc = index.add_or_fire("B", "C") + syn_bc.strength = 0.5 + + syn_cd = index.add_or_fire("C", "D") + syn_cd.strength = 0.9 + + syn_ac = index.add_or_fire("A", "C") + syn_ac.strength = 0.1 + + # 1 hop from A + hops_1 = index.get_multi_hop_neighbors("A", depth=1) + assert hops_1["B"] == pytest.approx(0.8) + assert hops_1["C"] == pytest.approx(0.1) + assert "D" not in hops_1 + + hops_2 = index.get_multi_hop_neighbors("A", depth=2) + assert hops_2["B"] == pytest.approx(0.8) + assert hops_2["C"] == pytest.approx(0.4) + assert hops_2["D"] == pytest.approx(0.09) + + hops_3 = index.get_multi_hop_neighbors("A", depth=3) + assert hops_3["D"] == pytest.approx(0.36) diff --git a/tests/test_persistence_failure.py b/tests/test_persistence_failure.py index 1ede62314ce537163b01f863eb5727ec2023f929..02155a4249a540a201765cde4dbf3b885b3711df 100644 --- a/tests/test_persistence_failure.py +++ b/tests/test_persistence_failure.py @@ -50,7 +50,8 @@ def test_persistence_failure_logs_error(test_engine, capsys): with patch('builtins.open', side_effect=side_effect): # This should NOT raise an exception - error should be caught and logged - asyncio.run(test_engine.store("Test content")) + asyncio.run(test_engine.initialize()) + asyncio.run(test_engine.store("Test content")) # The test passes if we reach here without an exception # The error is logged to stderr via loguru (verified by manual inspection) diff --git a/tests/test_prediction_store.py b/tests/test_prediction_store.py new file mode 100644 index 0000000000000000000000000000000000000000..042ec7684168938a6ad10378898c1d262d62fa7b --- /dev/null +++ b/tests/test_prediction_store.py @@ -0,0 +1,160 @@ +""" +Tests for Phase 5.0 Prediction Store Module. +""" + +import pytest +import asyncio +from datetime import datetime, timezone, timedelta + +from mnemocore.core.prediction_store import ( + PredictionRecord, + PredictionStore, + STATUS_PENDING, + STATUS_VERIFIED, + STATUS_FALSIFIED, + STATUS_EXPIRED, +) + + +# ------------------------------------------------------------------ # +# PredictionRecord # +# ------------------------------------------------------------------ # + +class TestPredictionRecord: + def test_auto_id(self): + r = PredictionRecord(content="AI will achieve AGI by 2030") + assert r.id.startswith("pred_") + + def test_default_status_pending(self): + r = PredictionRecord() + assert r.status == STATUS_PENDING + + def test_is_expired_no_deadline(self): + r = PredictionRecord() + assert not r.is_expired() + + def test_is_expired_future_deadline(self): + future = (datetime.now(timezone.utc) + timedelta(days=30)).isoformat() + r = PredictionRecord(verification_deadline=future) + assert not r.is_expired() + + def test_is_expired_past_deadline(self): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + r = PredictionRecord(verification_deadline=past, status=STATUS_PENDING) + assert r.is_expired() + + def test_not_expired_if_not_pending(self): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + r = PredictionRecord(verification_deadline=past, status=STATUS_VERIFIED) + assert not r.is_expired() + + def test_to_dict_keys(self): + r = PredictionRecord(content="test") + d = r.to_dict() + for key in ["id", "content", "status", "confidence_at_creation", "outcome"]: + assert key in d + + def test_roundtrip(self): + r = PredictionRecord(content="X will happen", confidence_at_creation=0.75) + restored = PredictionRecord.from_dict(r.to_dict()) + assert restored.content == r.content + assert restored.confidence_at_creation == pytest.approx(0.75) + assert restored.status == STATUS_PENDING + + +# ------------------------------------------------------------------ # +# PredictionStore # +# ------------------------------------------------------------------ # + +class TestPredictionStore: + def test_create_returns_id(self): + store = PredictionStore() + pred_id = store.create("Test prediction", confidence=0.8) + assert pred_id.startswith("pred_") + + def test_get_returns_record(self): + store = PredictionStore() + pred_id = store.create("Something will happen", confidence=0.6) + rec = store.get(pred_id) + assert rec is not None + assert rec.content == "Something will happen" + + def test_get_unknown_returns_none(self): + store = PredictionStore() + assert store.get("pred_nonexistent") is None + + def test_list_all_no_filter(self): + store = PredictionStore() + store.create("A", confidence=0.5) + store.create("B", confidence=0.7) + all_preds = store.list_all() + assert len(all_preds) == 2 + + def test_list_filter_by_status(self): + store = PredictionStore() + store.create("A", confidence=0.5) + pending = store.list_all(status=STATUS_PENDING) + assert len(pending) == 1 + + def test_len(self): + store = PredictionStore() + store.create("X") + store.create("Y") + assert len(store) == 2 + + def test_deadline_days(self): + store = PredictionStore() + pred_id = store.create("future", deadline_days=30) + rec = store.get(pred_id) + assert rec.verification_deadline is not None + deadline = datetime.fromisoformat(rec.verification_deadline) + # Should be ~30 days from now + days_diff = (deadline - datetime.now(timezone.utc)).days + assert 28 <= days_diff <= 31 + + @pytest.mark.asyncio + async def test_verify_success(self): + store = PredictionStore(engine=None) + pred_id = store.create("Test will succeed", confidence=0.9) + result = await store.verify(pred_id, success=True, notes="confirmed") + assert result is not None + assert result.status == STATUS_VERIFIED + assert result.outcome is True + assert result.verification_notes == "confirmed" + + @pytest.mark.asyncio + async def test_verify_failure(self): + store = PredictionStore(engine=None) + pred_id = store.create("This will fail", confidence=0.9) + result = await store.verify(pred_id, success=False) + assert result.status == STATUS_FALSIFIED + assert result.outcome is False + + @pytest.mark.asyncio + async def test_verify_unknown_returns_none(self): + store = PredictionStore() + result = await store.verify("pred_bogus", success=True) + assert result is None + + @pytest.mark.asyncio + async def test_expire_due(self): + store = PredictionStore() + # Past deadline + past = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat() + rec = PredictionRecord(content="overdue", verification_deadline=past) + store._records[rec.id] = rec + expired = await store.expire_due() + assert len(expired) == 1 + assert expired[0].status == STATUS_EXPIRED + + def test_get_due(self): + store = PredictionStore() + past = (datetime.now(timezone.utc) - timedelta(hours=1)).isoformat() + future = (datetime.now(timezone.utc) + timedelta(days=30)).isoformat() + rec_past = PredictionRecord(content="overdue", verification_deadline=past) + rec_future = PredictionRecord(content="future", verification_deadline=future) + store._records[rec_past.id] = rec_past + store._records[rec_future.id] = rec_future + due = store.get_due() + assert rec_past in due + assert rec_future not in due diff --git a/tests/test_preference_learning.py b/tests/test_preference_learning.py new file mode 100644 index 0000000000000000000000000000000000000000..aae94f835840f2b38a8d4bb5067ef8eb161ca67b --- /dev/null +++ b/tests/test_preference_learning.py @@ -0,0 +1,75 @@ +import pytest +import pytest_asyncio +import os +from mnemocore.core.config import get_config, reset_config +from mnemocore.core.engine import HAIMEngine + +@pytest.fixture +def test_engine(tmp_path): + from mnemocore.core.hnsw_index import HNSWIndexManager + HNSWIndexManager._instance = None + reset_config() + + data_dir = tmp_path / "data" + data_dir.mkdir() + + os.environ["HAIM_DATA_DIR"] = str(data_dir) + os.environ["HAIM_ENCODING_MODE"] = "binary" + os.environ["HAIM_DIMENSIONALITY"] = "1024" + os.environ["HAIM_PREFERENCE_ENABLED"] = "True" + os.environ["HAIM_PREFERENCE_LEARNING_RATE"] = "0.2" + os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] = "0.01" + os.environ["HAIM_MEMORY_FILE"] = str(data_dir / "memory.jsonl") + os.environ["HAIM_CODEBOOK_FILE"] = str(data_dir / "codebook.json") + os.environ["HAIM_SYNAPSES_FILE"] = str(data_dir / "synapses.json") + os.environ["HAIM_WARM_MMAP_DIR"] = str(data_dir / "warm") + os.environ["HAIM_COLD_ARCHIVE_DIR"] = str(data_dir / "cold") + + reset_config() + engine = HAIMEngine() + yield engine + + del os.environ["HAIM_DATA_DIR"] + del os.environ["HAIM_ENCODING_MODE"] + del os.environ["HAIM_DIMENSIONALITY"] + del os.environ["HAIM_PREFERENCE_ENABLED"] + del os.environ["HAIM_PREFERENCE_LEARNING_RATE"] + del os.environ["HAIM_TIERS_HOT_LTP_THRESHOLD_MIN"] + for key in ["HAIM_MEMORY_FILE", "HAIM_CODEBOOK_FILE", "HAIM_SYNAPSES_FILE", "HAIM_WARM_MMAP_DIR", "HAIM_COLD_ARCHIVE_DIR"]: + os.environ.pop(key, None) + reset_config() + +@pytest.mark.asyncio +async def test_preference_learning(test_engine): + await test_engine.initialize() + + node_a = await test_engine.store("I want to eat a fresh green salad.") + node_b = await test_engine.store("I want to eat a big greasy burger.") + + # Run a generic query + results_before = await test_engine.query("What should I eat?", top_k=10) + score_a_before = next((s for i, s in results_before if i == node_a), 0.0) + score_b_before = next((s for i, s in results_before if i == node_b), 0.0) + + # User decides to eat salad (a positive outcome mapping to node_a's concept) + await test_engine.log_decision("fresh healthy vegetables and salad", 1.0) + + # Re-run query to see if salad is biased higher + results_after = await test_engine.query("What should I eat?", top_k=10) + score_a_after = next((s for i, s in results_after if i == node_a), 0.0) + score_b_after = next((s for i, s in results_after if i == node_b), 0.0) + + # Score for A should have increased relative to B because of the decided preference + assert score_a_after > score_a_before + + # Negative feedback mapping to burger + await test_engine.log_decision("big greasy burger fast food", -1.0) + await test_engine.log_decision("big greasy burger fast food", -1.0) + + results_final = await test_engine.query("What should I eat?", top_k=10) + score_b_final = next((s for i, s in results_final if i == node_b), 0.0) + + # The inverted target should drive B's score down (or not boost it as much) relative to the first boost + assert score_b_final < score_b_after or score_b_after == score_b_before + + await test_engine.close() diff --git a/tests/test_procedural_store.py b/tests/test_procedural_store.py new file mode 100644 index 0000000000000000000000000000000000000000..a4f14356259a3d4510f1f6d113fb820a54a3cdd5 --- /dev/null +++ b/tests/test_procedural_store.py @@ -0,0 +1,77 @@ +from datetime import datetime +from mnemocore.core.procedural_store import ProceduralStoreService +from mnemocore.core.memory_model import Procedure + +def test_procedural_store_add_and_get(): + store = ProceduralStoreService() + + proc = Procedure( + id="proc-1", + name="extract_information", + description="Extracts names and dates from text", + created_by_agent="system", + created_at=datetime.utcnow(), + updated_at=datetime.utcnow(), + steps=["Read text", "Regex match", "Return JSON"], + trigger_pattern=".*", + success_count=0, + failure_count=0, + reliability=1.0, + tags=[] + ) + + store.store_procedure(proc) + + p2 = store.get_procedure("proc-1") + assert p2 is not None + assert p2.name == "extract_information" + +def test_procedural_store_outcome(): + store = ProceduralStoreService() + proc = Procedure( + id="proc-2", + name="API_Call", + description="GET request", + created_by_agent="system", + created_at=datetime.utcnow(), + updated_at=datetime.utcnow(), + steps=["Requests.get"], + trigger_pattern=".*", + success_count=0, + failure_count=0, + reliability=1.0, + tags=[] + ) + store.store_procedure(proc) + + # Success + store.record_procedure_outcome("proc-2", success=True) + p2 = store.get_procedure("proc-2") + assert p2.success_count == 1 + assert p2.failure_count == 0 + + # Failure + store.record_procedure_outcome("proc-2", success=False) + p3 = store.get_procedure("proc-2") + assert p3.success_count == 1 + assert p3.failure_count == 1 + assert p3.reliability == 0.9 + +def test_procedural_store_find(): + store = ProceduralStoreService() + proc1 = Procedure( + id="p1", name="search_web", description="Find info online", steps=[], + created_by_agent="system", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), trigger_pattern="search", success_count=0, failure_count=0, reliability=1.0, tags=[] + ) + proc2 = Procedure( + id="p2", name="calculate_math", description="Evaluate math expression", steps=[], + created_by_agent="system", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), trigger_pattern="math", success_count=0, failure_count=0, reliability=1.0, tags=[] + ) + + store.store_procedure(proc1) + store.store_procedure(proc2) + + results = store.find_applicable_procedures("math expression") + assert len(results) > 0 + # The basic regex search should catch math + assert any(p.id == "p2" for p in results) diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..f388cf815353626f1081e51476a94d64736cf487 --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,184 @@ +""" +Tests for Phase 5.0 Provenance Tracking Module. +Tests ProvenanceRecord, ProvenanceOrigin, and LineageEvent. +""" + +import pytest +from datetime import datetime, timezone + +from mnemocore.core.provenance import ( + ProvenanceRecord, + ProvenanceOrigin, + LineageEvent, + ORIGIN_TYPES, +) + + +# ------------------------------------------------------------------ # +# LineageEvent # +# ------------------------------------------------------------------ # + +class TestLineageEvent: + def test_basic_creation(self): + evt = LineageEvent(event="created", timestamp="2026-02-21T20:00:00+00:00") + assert evt.event == "created" + assert evt.actor is None + assert evt.source_memories == [] + + def test_to_dict_minimal(self): + evt = LineageEvent(event="archived", timestamp="2026-02-21T20:00:00+00:00") + d = evt.to_dict() + assert d["event"] == "archived" + assert "actor" not in d + assert "source_memories" not in d + + def test_to_dict_full(self): + evt = LineageEvent( + event="consolidated", + timestamp="2026-02-21T20:00:00+00:00", + actor="worker", + source_memories=["mem_a", "mem_b"], + outcome=True, + notes="cluster_7", + ) + d = evt.to_dict() + assert d["actor"] == "worker" + assert d["source_memories"] == ["mem_a", "mem_b"] + assert d["outcome"] is True + + def test_roundtrip(self): + evt = LineageEvent( + event="verified", + timestamp="2026-02-21T20:00:00+00:00", + actor="system", + outcome=True, + ) + restored = LineageEvent.from_dict(evt.to_dict()) + assert restored.event == "verified" + assert restored.outcome is True + + +# ------------------------------------------------------------------ # +# ProvenanceOrigin # +# ------------------------------------------------------------------ # + +class TestProvenanceOrigin: + def test_creation(self): + origin = ProvenanceOrigin(type="observation", agent_id="agent-001") + assert origin.type == "observation" + assert origin.agent_id == "agent-001" + + def test_to_dict_omits_nones(self): + origin = ProvenanceOrigin(type="dream") + d = origin.to_dict() + assert "agent_id" not in d + assert "source_url" not in d + + def test_roundtrip(self): + origin = ProvenanceOrigin( + type="external_sync", + source_url="https://example.com/feed", + session_id="sess_abc", + ) + restored = ProvenanceOrigin.from_dict(origin.to_dict()) + assert restored.type == "external_sync" + assert restored.source_url == "https://example.com/feed" + + +# ------------------------------------------------------------------ # +# ProvenanceRecord # +# ------------------------------------------------------------------ # + +class TestProvenanceRecord: + def test_new_creates_created_event(self): + rec = ProvenanceRecord.new(origin_type="observation", agent_id="agent-42") + assert rec.origin.type == "observation" + assert rec.origin.agent_id == "agent-42" + assert len(rec.lineage) == 1 + assert rec.lineage[0].event == "created" + assert rec.version == 2 # starts at 1, incremented once + + def test_add_event_bumps_version(self): + rec = ProvenanceRecord.new() + v0 = rec.version + rec.add_event("accessed", actor="query_engine") + assert rec.version == v0 + 1 + assert rec.lineage[-1].event == "accessed" + + def test_mark_consolidated(self): + rec = ProvenanceRecord.new() + rec.mark_consolidated(["mem_a", "mem_b"]) + evt = rec.lineage[-1] + assert evt.event == "consolidated" + assert "mem_a" in evt.source_memories + + def test_mark_verified_successful(self): + rec = ProvenanceRecord.new() + rec.mark_verified(success=True) + assert rec.is_verified() + + def test_mark_verified_failed_not_is_verified(self): + rec = ProvenanceRecord.new() + rec.mark_verified(success=False) + assert not rec.is_verified() + + def test_mark_contradicted(self): + rec = ProvenanceRecord.new() + rec.mark_contradicted(contradiction_group_id="cg_001") + assert rec.is_contradicted() + + def test_is_contradicted_false_when_clean(self): + rec = ProvenanceRecord.new() + assert not rec.is_contradicted() + + def test_last_event(self): + rec = ProvenanceRecord.new() + rec.add_event("verified", outcome=True) + rec.add_event("archived") + assert rec.last_event.event == "archived" + + def test_created_at_property(self): + rec = ProvenanceRecord.new() + # Should not raise and should return a parseable ISO string + ts_str = rec.created_at + dt = datetime.fromisoformat(ts_str) + assert dt.tzinfo is not None + + def test_serialization_roundtrip(self): + rec = ProvenanceRecord.new( + origin_type="inference", + agent_id="agent-99", + session_id="s123", + ) + rec.add_event("accessed") + rec.mark_consolidated(["x", "y"]) + + serialized = rec.to_dict() + restored = ProvenanceRecord.from_dict(serialized) + + assert restored.origin.type == "inference" + assert restored.version == rec.version + assert len(restored.lineage) == len(rec.lineage) + assert restored.lineage[0].event == "created" + assert restored.lineage[-1].event == "consolidated" + + def test_unknown_origin_type_defaults_to_observation(self): + rec = ProvenanceRecord.new(origin_type="INVALID_TYPE") + assert rec.origin.type == "observation" + + def test_empty_lineage_last_event_is_none(self): + rec = ProvenanceRecord( + origin=ProvenanceOrigin(type="observation"), + lineage=[], + ) + assert rec.last_event is None + + def test_confidence_source_default(self): + rec = ProvenanceRecord.new() + assert rec.confidence_source == "bayesian_ltp" + + def test_repr(self): + rec = ProvenanceRecord.new(origin_type="dream") + r = repr(rec) + assert "dream" in r + assert "version" in r diff --git a/tests/test_pulse.py b/tests/test_pulse.py new file mode 100644 index 0000000000000000000000000000000000000000..355b7ab1a2e168f391124cdabbec37b735303e0d --- /dev/null +++ b/tests/test_pulse.py @@ -0,0 +1,36 @@ +import pytest +import asyncio +from mnemocore.core.pulse import PulseLoop +from mnemocore.core.config import PulseConfig +from mnemocore.core.container import Container +from mnemocore.core.engine import HAIMEngine + +@pytest.fixture +def mock_container(): + c = Container(config=None) # type: ignore + return c + +@pytest.fixture +def mock_engine(): + return HAIMEngine() + +def test_pulse_initialization(mock_container): + config = PulseConfig(enabled=True, interval_seconds=1) + pulse = PulseLoop(config=config, container=mock_container) + + assert getattr(pulse.config, "enabled", False) is True + assert pulse._running is False + +@pytest.mark.asyncio +async def test_pulse_start_stop(mock_container): + config = PulseConfig(enabled=True, interval_seconds=1) + pulse = PulseLoop(config=config, container=mock_container) + + task = asyncio.create_task(pulse.start()) + await asyncio.sleep(0.1) + + assert pulse._running is True + + pulse.stop() + await task + assert pulse._running is False diff --git a/tests/test_semantic_store.py b/tests/test_semantic_store.py new file mode 100644 index 0000000000000000000000000000000000000000..cdb63df2fd36cf75f8ea5dae98113d214f00a739 --- /dev/null +++ b/tests/test_semantic_store.py @@ -0,0 +1,52 @@ +import pytest +from datetime import datetime +from unittest.mock import MagicMock +from mnemocore.core.semantic_store import SemanticStoreService +from mnemocore.core.memory_model import SemanticConcept +from mnemocore.core.binary_hdv import BinaryHDV + +@pytest.fixture +def mock_qdrant(): + return MagicMock() + +def test_semantic_store_upsert_and_get(mock_qdrant): + store = SemanticStoreService(qdrant_store=mock_qdrant) + + mock_hdv = MagicMock(spec=BinaryHDV) + + concept = SemanticConcept( + id="Dog", + label="Dog", + description="A domesticated carnivorous mammal", + tags=["animal"], + prototype_hdv=mock_hdv, + support_episode_ids=[], + reliability=1.0, + last_updated_at=datetime.utcnow(), + metadata={"legs": 4, "sound": "bark"} + ) + + store.upsert_concept(concept) + + retrieved = store.get_concept("Dog") + assert retrieved is not None + assert retrieved.label == "Dog" + assert retrieved.description == "A domesticated carnivorous mammal" + assert retrieved.metadata["legs"] == 4 + + # Upsert existing + concept.metadata["friendly"] = True + store.upsert_concept(concept) + + concept2 = store.get_concept("Dog") + assert concept2.metadata["legs"] == 4 + assert concept2.metadata["friendly"] is True + +def test_semantic_store_find_nearby(mock_qdrant): + store = SemanticStoreService(qdrant_store=mock_qdrant) + + # We just test the empty return when Qdrant is mocked and we haven't wired + # the advanced embedding layer in the test + results = store.find_nearby_concepts("Puppy") + assert isinstance(results, list) + assert len(results) == 0 diff --git a/tests/test_stability.py b/tests/test_stability.py index 5964679f2b8892bfa5e6946a47df2fc088e67cf4..758ce6d131dc45ece7a17c62fef27e73f895e4ba 100644 --- a/tests/test_stability.py +++ b/tests/test_stability.py @@ -30,9 +30,9 @@ def mock_deps(): mock_config.security = mock_security mock_config.dimensionality = 1024 - with patch("src.api.main.HAIMEngine", return_value=mock_engine), \ - patch("src.api.main.build_container", return_value=mock_container), \ - patch("src.api.main.get_config", return_value=mock_config): + with patch("mnemocore.api.main.HAIMEngine", return_value=mock_engine), \ + patch("mnemocore.api.main.build_container", return_value=mock_container), \ + patch("mnemocore.api.main.get_config", return_value=mock_config): yield mock_engine, mock_redis def test_engine_lifecycle(mock_deps): @@ -91,9 +91,9 @@ def test_security_middleware_fallback(mock_deps): mock_conf_no_sec.security = mock_security_no_key mock_conf_no_sec.dimensionality = 1024 - with patch("src.api.main.get_config", return_value=mock_conf_no_sec), \ - patch("src.api.main.HAIMEngine", return_value=mock_engine2), \ - patch("src.api.main.build_container", return_value=mock_container2), \ + with patch("mnemocore.api.main.get_config", return_value=mock_conf_no_sec), \ + patch("mnemocore.api.main.HAIMEngine", return_value=mock_engine2), \ + patch("mnemocore.api.main.build_container", return_value=mock_container2), \ patch.dict(os.environ, {"HAIM_API_KEY": "env-secret-key"}, clear=False): # Re-import to get fresh app with new patches applied diff --git a/tests/test_temporal_decay.py b/tests/test_temporal_decay.py new file mode 100644 index 0000000000000000000000000000000000000000..d74a85952c15928cb0afbeef7282de3e7bd85d20 --- /dev/null +++ b/tests/test_temporal_decay.py @@ -0,0 +1,174 @@ +""" +Tests for Phase 5.0 Temporal Decay Module. +Tests AdaptiveDecayEngine: retention, stability, review candidates, eviction. +""" + +import math +import pytest +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock + +from mnemocore.core.temporal_decay import ( + AdaptiveDecayEngine, + get_adaptive_decay_engine, + REVIEW_THRESHOLD, + EVICTION_THRESHOLD, + S_BASE, + K_GROWTH, +) + + +# ------------------------------------------------------------------ # +# Helpers # +# ------------------------------------------------------------------ # + +def _make_node(access_count: int = 1, days_since_access: float = 0.0, stability: float = 1.0): + node = MagicMock() + node.id = "test-node-0001" + node.access_count = access_count + now = datetime.now(timezone.utc) + node.last_accessed = now - timedelta(days=days_since_access) + node.created_at = now - timedelta(days=days_since_access + 1) + node.stability = stability + node.review_candidate = False + node.epistemic_value = 0.5 + return node + + +# ------------------------------------------------------------------ # +# Stability # +# ------------------------------------------------------------------ # + +class TestStability: + def test_stability_grows_with_access_count(self): + engine = AdaptiveDecayEngine() + node1 = _make_node(access_count=1) + node10 = _make_node(access_count=10) + assert engine.stability(node10) > engine.stability(node1) + + def test_stability_formula(self): + engine = AdaptiveDecayEngine(s_base=1.0, k_growth=0.5) + node = _make_node(access_count=5) + expected = 1.0 * (1.0 + 0.5 * math.log1p(5)) + assert engine.stability(node) == pytest.approx(expected, rel=1e-5) + + def test_stability_never_below_s_base(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=1) + assert engine.stability(node) >= S_BASE + + +# ------------------------------------------------------------------ # +# Retention # +# ------------------------------------------------------------------ # + +class TestRetention: + def test_fresh_memory_high_retention(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=5, days_since_access=0.0) + r = engine.retention(node) + assert r > 0.99 + + def test_old_memory_low_retention(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=1, days_since_access=30.0) + r = engine.retention(node) + assert r < 0.5 + + def test_high_access_count_slows_decay(self): + engine = AdaptiveDecayEngine() + node_low = _make_node(access_count=1, days_since_access=5) + node_high = _make_node(access_count=50, days_since_access=5) + # node_high has higher stability → higher retention after same time + assert engine.retention(node_high) > engine.retention(node_low) + + def test_retention_between_0_and_1(self): + engine = AdaptiveDecayEngine() + for days in [0, 1, 10, 100]: + node = _make_node(access_count=1, days_since_access=days) + r = engine.retention(node) + assert 0.0 < r <= 1.0 + + +# ------------------------------------------------------------------ # +# Review candidates # +# ------------------------------------------------------------------ # + +class TestReviewCandidates: + def test_node_flagged_when_retention_low(self): + engine = AdaptiveDecayEngine(review_threshold=0.99) + node = _make_node(access_count=1, days_since_access=0.0) + flagged = engine.update_review_candidate(node) + # retention near 1.0 but threshold is 0.99 → might flag + # The point is the flag matches the result + assert node.review_candidate == flagged + + def test_recently_accessed_node_not_flagged(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=5, days_since_access=0.0) + flagged = engine.update_review_candidate(node) + # Fresh node with good access_count should not be a candidate + assert not flagged + assert not node.review_candidate + + def test_scan_returns_candidates(self): + engine = AdaptiveDecayEngine() + fresh = _make_node(access_count=10, days_since_access=0) + old = _make_node(access_count=1, days_since_access=100) + candidates = engine.scan_review_candidates([fresh, old]) + assert old in candidates + assert fresh not in candidates + + +# ------------------------------------------------------------------ # +# Eviction # +# ------------------------------------------------------------------ # + +class TestEviction: + def test_very_old_low_access_should_evict(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=1, days_since_access=200) + assert engine.should_evict(node) + + def test_recent_accessed_should_not_evict(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=5, days_since_access=0) + assert not engine.should_evict(node) + + def test_eviction_candidates_batch(self): + engine = AdaptiveDecayEngine() + keepers = [_make_node(access_count=10, days_since_access=0) for _ in range(3)] + evicts = [_make_node(access_count=1, days_since_access=300) for _ in range(2)] + result = engine.eviction_candidates(keepers + evicts) + assert len(result) >= 2 + + +# ------------------------------------------------------------------ # +# Singleton # +# ------------------------------------------------------------------ # + +class TestSingleton: + def test_same_object(self): + a = get_adaptive_decay_engine() + b = get_adaptive_decay_engine() + assert a is b + + +# ------------------------------------------------------------------ # +# update_after_access # +# ------------------------------------------------------------------ # + +class TestUpdateAfterAccess: + def test_stability_written_back(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=3) + expected = engine.stability(node) + engine.update_after_access(node) + assert node.stability == pytest.approx(expected, rel=1e-5) + + def test_review_candidate_cleared(self): + engine = AdaptiveDecayEngine() + node = _make_node(access_count=3) + node.review_candidate = True + engine.update_after_access(node) + assert not node.review_candidate diff --git a/tests/test_working_memory.py b/tests/test_working_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..3825904ea25565aaaeaf47aa6204f3ef7f2d5df1 --- /dev/null +++ b/tests/test_working_memory.py @@ -0,0 +1,88 @@ +import pytest +from datetime import datetime, timedelta +from mnemocore.core.working_memory import WorkingMemoryService +from mnemocore.core.memory_model import WorkingMemoryItem + +def test_working_memory_push_and_get(): + wm = WorkingMemoryService(max_items_per_agent=5) + + item = WorkingMemoryItem( + id="wm_123", + agent_id="agent1", + created_at=datetime.utcnow(), + ttl_seconds=3600, + content="Testing working memory", + importance=0.8, + tags=["test"], + kind="thought" + ) + + wm.push_item("agent1", item) + state = wm.get_state("agent1") + + assert state is not None + assert len(state.items) == 1 + assert state.items[0].id == "wm_123" + +def test_working_memory_eviction(): + wm = WorkingMemoryService(max_items_per_agent=2) + + for i in range(3): + item = WorkingMemoryItem( + id=f"wm_{i}", + agent_id="agent1", + created_at=datetime.utcnow() + timedelta(milliseconds=i*50), + ttl_seconds=3600, + content=f"Test {i}", + importance=0.5, + kind="thought", + tags=[] + ) + wm.push_item("agent1", item) + + state = wm.get_state("agent1") + assert state is not None + assert len(state.items) == 2 + # The oldest one (wm_0) should be evicted. + assert state.items[0].id == "wm_1" + assert state.items[1].id == "wm_2" + +def test_working_memory_clear(): + wm = WorkingMemoryService() + + item = WorkingMemoryItem( + id="wm_123", + agent_id="agent1", + created_at=datetime.utcnow(), + ttl_seconds=3600, + content="Testing working memory", + importance=0.8, + kind="thought", + tags=[] + ) + + wm.push_item("agent1", item) + wm.clear("agent1") + + state = wm.get_state("agent1") + assert state is None or len(state.items) == 0 + +def test_working_memory_prune(): + wm = WorkingMemoryService() + + # Push an item that is immediately expired + item = WorkingMemoryItem( + id="wm_expired", + agent_id="agent1", + created_at=datetime.utcnow(), + ttl_seconds=-1, # Expired + content="Expired memory", + importance=0.1, + kind="thought", + tags=[] + ) + wm.push_item("agent1", item) + + wm.prune_all() + state = wm.get_state("agent1") + assert state is None or len(state.items) == 0 diff --git a/tests/test_xor_attention.py b/tests/test_xor_attention.py index 57ac2c4510483de8d1cb3bf5a03c4df36021a07b..2973a07671bce423ef7b2d232efc92d01ae96845 100644 --- a/tests/test_xor_attention.py +++ b/tests/test_xor_attention.py @@ -10,6 +10,15 @@ import numpy as np from mnemocore.core.attention import XORIsolationMask, IsolationConfig from mnemocore.core.binary_hdv import BinaryHDV +from mnemocore.core.hnsw_index import HNSWIndexManager + +@pytest.fixture(autouse=True) +def reset_hnsw_singleton(): + """Reset the HNSWIndexManager singleton to prevent cross-test pollution.""" + HNSWIndexManager._instance = None + yield + HNSWIndexManager._instance = None + class TestXORIsolationMask: @@ -175,6 +184,8 @@ class TestXORIsolationMaskIntegration: cold_archive_dir=os.path.join(tmpdir, "cold"), ), ) + import mnemocore.core.config as config_module + config_module._CONFIG = config engine = HAIMEngine(config=config) await engine.initialize() @@ -226,6 +237,8 @@ class TestXORIsolationMaskIntegration: cold_archive_dir=os.path.join(tmpdir, "cold"), ), ) + import mnemocore.core.config as config_module + config_module._CONFIG = config engine = HAIMEngine(config=config) await engine.initialize() @@ -274,6 +287,8 @@ class TestXORIsolationMaskIntegration: cold_archive_dir=os.path.join(tmpdir, "cold"), ), ) + import mnemocore.core.config as config_module + config_module._CONFIG = config engine = HAIMEngine(config=config) await engine.initialize()