diff --git a/.playwright-mcp/page-2026-06-10T14-22-38-882Z.yml b/.playwright-mcp/page-2026-06-10T14-22-38-882Z.yml new file mode 100644 index 0000000000000000000000000000000000000000..91abd965b34f0a4523d4516cc05ab6b8834d74ef --- /dev/null +++ b/.playwright-mcp/page-2026-06-10T14-22-38-882Z.yml @@ -0,0 +1,3 @@ +- generic [ref=e5]: + - img [ref=e9] + - paragraph [ref=e20]: Laden... \ No newline at end of file diff --git a/.playwright-mcp/page-2026-06-10T14-27-49-510Z.yml b/.playwright-mcp/page-2026-06-10T14-27-49-510Z.yml new file mode 100644 index 0000000000000000000000000000000000000000..161808e9533f5801b0184c96170c8a8811bd0b51 --- /dev/null +++ b/.playwright-mcp/page-2026-06-10T14-27-49-510Z.yml @@ -0,0 +1,60 @@ +- generic [ref=e21]: + - main [ref=e22]: + - generic [ref=e23]: + - heading "πŸ”₯ HearthNet β€” Community AI Mesh" [level=1] [ref=e28] + - generic [ref=e29]: + - generic [ref=e32]: ● ONLINE + - paragraph [ref=e37]: + - text: "Node:" + - code [ref=e38]: unknown + - generic [ref=e39]: + - generic [ref=e40]: + - generic [ref=e41]: + - button [ref=e42] [cursor=pointer]: Ask + - button [ref=e43] [cursor=pointer]: Chat + - button [ref=e44] [cursor=pointer]: Marketplace + - button [ref=e45] [cursor=pointer]: Files + - button [ref=e46] [cursor=pointer]: Emergency + - button [ref=e47] [cursor=pointer]: Settings + - tablist [ref=e48]: + - tab "Ask" [ref=e49] [cursor=pointer] + - tab "Chat" [active] [selected] [ref=e50] [cursor=pointer] + - tab "Marketplace" [ref=e51] [cursor=pointer] + - tab "Files" [ref=e52] [cursor=pointer] + - tab "Emergency" [ref=e53] [cursor=pointer] + - tab "Settings" [ref=e54] [cursor=pointer] + - tabpanel [ref=e106]: + - generic [ref=e108]: + - heading "Direct Messages" [level=3] [ref=e113] + - generic [ref=e114]: + - generic [ref=e117]: + - generic [ref=e118]: Recipient Node ID + - textbox "Recipient Node ID" [ref=e120]: + - /placeholder: ed25519:... + - button "Load History" [ref=e121] [cursor=pointer] + - generic [ref=e123]: + - generic: + - generic: + - img + - text: Messages + - log "chatbot conversation" [ref=e124]: + - complementary [ref=e125] + - generic [ref=e126]: + - generic [ref=e129]: + - generic [ref=e130]: Message + - textbox "Message" [ref=e132]: + - /placeholder: "" + - button "Send" [ref=e133] [cursor=pointer] + - contentinfo "Gradio footer navigation" [ref=e97]: + - button "Über API verwenden Logo" [ref=e98] [cursor=pointer]: + - text: Über API verwenden + - img "Logo" [ref=e99] + - generic [ref=e100]: Β· + - link "Mit Gradio erstellt Logo" [ref=e101] [cursor=pointer]: + - /url: https://gradio.app + - text: Mit Gradio erstellt + - img "Logo" [ref=e102] + - generic [ref=e103]: Β· + - button "Einstellungen Einstellungen" [ref=e104] [cursor=pointer]: + - text: Einstellungen + - img "Einstellungen" [ref=e105] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-06-10T14-28-44-543Z.yml b/.playwright-mcp/page-2026-06-10T14-28-44-543Z.yml new file mode 100644 index 0000000000000000000000000000000000000000..1a51572c5224134067455e448160231a37059d32 --- /dev/null +++ b/.playwright-mcp/page-2026-06-10T14-28-44-543Z.yml @@ -0,0 +1,67 @@ +- generic [ref=e21]: + - main [ref=e22]: + - generic [ref=e23]: + - heading "πŸ”₯ HearthNet β€” Community AI Mesh" [level=1] [ref=e28] + - generic [ref=e29]: + - generic [ref=e32]: ● ONLINE + - paragraph [ref=e37]: + - text: "Node:" + - code [ref=e38]: unknown + - generic [ref=e39]: + - generic [ref=e40]: + - generic [ref=e41]: + - button [ref=e42] [cursor=pointer]: Ask + - button [ref=e43] [cursor=pointer]: Chat + - button [ref=e44] [cursor=pointer]: Marketplace + - button [ref=e45] [cursor=pointer]: Files + - button [ref=e46] [cursor=pointer]: Emergency + - button [ref=e47] [cursor=pointer]: Settings + - tablist [ref=e48]: + - tab "Ask" [ref=e49] [cursor=pointer] + - tab "Chat" [ref=e50] [cursor=pointer] + - tab "Marketplace" [active] [selected] [ref=e51] [cursor=pointer] + - tab "Files" [ref=e52] [cursor=pointer] + - tab "Emergency" [ref=e53] [cursor=pointer] + - tab "Settings" [ref=e54] [cursor=pointer] + - tabpanel [ref=e134]: + - generic [ref=e136]: + - heading "Community Marketplace" [level=3] [ref=e141] + - button "πŸ”„ Refresh" [ref=e142] [cursor=pointer] + - generic [ref=e143]: + - generic [ref=e144]: + - generic: + - generic: + - img + - text: Active Posts + - generic "Empty value" [ref=e146]: + - img [ref=e148] + - heading "Post Something" [level=4] [ref=e154] + - generic [ref=e156]: + - generic [ref=e158]: + - generic [ref=e159]: Title + - textbox "Title" [ref=e161]: + - /placeholder: "" + - generic [ref=e163]: + - generic [ref=e164]: Category + - generic [ref=e167]: + - listbox "Category" [ref=e168]: info + - generic: + - img + - generic [ref=e171]: + - generic [ref=e172]: Description + - textbox "Description" [ref=e174]: + - /placeholder: "" + - button "Post" [ref=e175] [cursor=pointer] + - contentinfo "Gradio footer navigation" [ref=e97]: + - button "Über API verwenden Logo" [ref=e98] [cursor=pointer]: + - text: Über API verwenden + - img "Logo" [ref=e99] + - generic [ref=e100]: Β· + - link "Mit Gradio erstellt Logo" [ref=e101] [cursor=pointer]: + - /url: https://gradio.app + - text: Mit Gradio erstellt + - img "Logo" [ref=e102] + - generic [ref=e103]: Β· + - button "Einstellungen Einstellungen" [ref=e104] [cursor=pointer]: + - text: Einstellungen + - img "Einstellungen" [ref=e105] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-06-10T14-28-52-588Z.yml b/.playwright-mcp/page-2026-06-10T14-28-52-588Z.yml new file mode 100644 index 0000000000000000000000000000000000000000..dbc01e1551c5550f9f5f1b3d8084332f6f140902 --- /dev/null +++ b/.playwright-mcp/page-2026-06-10T14-28-52-588Z.yml @@ -0,0 +1,53 @@ +- generic [ref=e21]: + - main [ref=e22]: + - generic [ref=e23]: + - heading "πŸ”₯ HearthNet β€” Community AI Mesh" [level=1] [ref=e28] + - generic [ref=e29]: + - generic [ref=e32]: ● ONLINE + - paragraph [ref=e37]: + - text: "Node:" + - code [ref=e38]: unknown + - generic [ref=e39]: + - generic [ref=e40]: + - generic [ref=e41]: + - button [ref=e42] [cursor=pointer]: Ask + - button [ref=e43] [cursor=pointer]: Chat + - button [ref=e44] [cursor=pointer]: Marketplace + - button [ref=e45] [cursor=pointer]: Files + - button [ref=e46] [cursor=pointer]: Emergency + - button [ref=e47] [cursor=pointer]: Settings + - tablist [ref=e48]: + - tab "Ask" [ref=e49] [cursor=pointer] + - tab "Chat" [ref=e50] [cursor=pointer] + - tab "Marketplace" [ref=e51] [cursor=pointer] + - tab "Files" [ref=e52] [cursor=pointer] + - tab "Emergency" [active] [selected] [ref=e53] [cursor=pointer] + - tab "Settings" [ref=e54] [cursor=pointer] + - tabpanel [ref=e176]: + - generic [ref=e178]: + - heading "🚨 Emergency Mode" [level=3] [ref=e183] + - generic [ref=e184]: + - generic [ref=e185]: + - generic: + - generic: + - img + - text: Current Mode + - generic "Empty value" [ref=e187]: + - img [ref=e189] + - button "Check Status" [ref=e191] [cursor=pointer] + - heading "Local Resources" [level=4] [ref=e196] + - paragraph [ref=e201]: In offline mode, all capabilities route to local nodes only. + - button "Run Connectivity Probe" [ref=e203] [cursor=pointer] + - contentinfo "Gradio footer navigation" [ref=e97]: + - button "Über API verwenden Logo" [ref=e98] [cursor=pointer]: + - text: Über API verwenden + - img "Logo" [ref=e99] + - generic [ref=e100]: Β· + - link "Mit Gradio erstellt Logo" [ref=e101] [cursor=pointer]: + - /url: https://gradio.app + - text: Mit Gradio erstellt + - img "Logo" [ref=e102] + - generic [ref=e103]: Β· + - button "Einstellungen Einstellungen" [ref=e104] [cursor=pointer]: + - text: Einstellungen + - img "Einstellungen" [ref=e105] \ No newline at end of file diff --git a/.playwright-mcp/page-2026-06-10T14-29-00-723Z.yml b/.playwright-mcp/page-2026-06-10T14-29-00-723Z.yml new file mode 100644 index 0000000000000000000000000000000000000000..99c3dfcd6b1cf3b8d7b41ab60a111c493750652e --- /dev/null +++ b/.playwright-mcp/page-2026-06-10T14-29-00-723Z.yml @@ -0,0 +1,110 @@ +- generic [ref=e21]: + - main [ref=e22]: + - generic [ref=e23]: + - heading "πŸ”₯ HearthNet β€” Community AI Mesh" [level=1] [ref=e28] + - generic [ref=e29]: + - generic [ref=e32]: ● ONLINE + - paragraph [ref=e37]: + - text: "Node:" + - code [ref=e38]: unknown + - generic [ref=e39]: + - generic [ref=e40]: + - generic [ref=e41]: + - button [ref=e42] [cursor=pointer]: Ask + - button [ref=e43] [cursor=pointer]: Chat + - button [ref=e44] [cursor=pointer]: Marketplace + - button [ref=e45] [cursor=pointer]: Files + - button [ref=e46] [cursor=pointer]: Emergency + - button [ref=e47] [cursor=pointer]: Settings + - tablist [ref=e48]: + - tab "Ask" [ref=e49] [cursor=pointer] + - tab "Chat" [ref=e50] [cursor=pointer] + - tab "Marketplace" [ref=e51] [cursor=pointer] + - tab "Files" [ref=e52] [cursor=pointer] + - tab "Emergency" [ref=e53] [cursor=pointer] + - tab "Settings" [active] [selected] [ref=e54] [cursor=pointer] + - tabpanel [ref=e204]: + - generic [ref=e206]: + - heading "Settings" [level=3] [ref=e211] + - heading "Node Identity" [level=4] [ref=e216] + - paragraph [ref=e221]: + - text: "Node ID:" + - code [ref=e222]: not initialized + - paragraph [ref=e227]: + - text: "Profile:" + - code [ref=e228]: hearth + - heading "Community" [level=4] [ref=e233] + - paragraph [ref=e238]: + - text: "Community:" + - code [ref=e239]: none + - heading "Phase Labels" [level=4] [ref=e244] + - table [ref=e249]: + - rowgroup [ref=e250]: + - row "Module Status" [ref=e251]: + - columnheader "Module" [ref=e252] + - columnheader "Status" [ref=e253] + - rowgroup [ref=e254]: + - row "M01 Identity βœ… Implemented" [ref=e255]: + - cell "M01 Identity" [ref=e256] + - cell "βœ… Implemented" [ref=e257] + - row "M02 Discovery βœ… Implemented (mDNS/UDP)" [ref=e258]: + - cell "M02 Discovery" [ref=e259] + - cell "βœ… Implemented (mDNS/UDP)" [ref=e260] + - row "M03 Bus βœ… Implemented" [ref=e261]: + - cell "M03 Bus" [ref=e262] + - cell "βœ… Implemented" [ref=e263] + - row "M04 LLM βœ… Implemented (Ollama/llama.cpp/HF)" [ref=e264]: + - cell "M04 LLM" [ref=e265] + - cell "βœ… Implemented (Ollama/llama.cpp/HF)" [ref=e266] + - row "M05 RAG βœ… Implemented" [ref=e267]: + - cell "M05 RAG" [ref=e268] + - cell "βœ… Implemented" [ref=e269] + - row "M06 Marketplace βœ… Implemented (event-sourced)" [ref=e270]: + - cell "M06 Marketplace" [ref=e271] + - cell "βœ… Implemented (event-sourced)" [ref=e272] + - row "M07 Blobs βœ… Implemented" [ref=e273]: + - cell "M07 Blobs" [ref=e274] + - cell "βœ… Implemented" [ref=e275] + - row "M08 UI βœ… This UI" [ref=e276]: + - cell "M08 UI" [ref=e277] + - cell "βœ… This UI" [ref=e278] + - row "M09 Emergency βœ… Implemented" [ref=e279]: + - cell "M09 Emergency" [ref=e280] + - cell "βœ… Implemented" [ref=e281] + - row "M10 Chat βœ… Implemented" [ref=e282]: + - cell "M10 Chat" [ref=e283] + - cell "βœ… Implemented" [ref=e284] + - row "M11 Embedding βœ… Implemented" [ref=e285]: + - cell "M11 Embedding" [ref=e286] + - cell "βœ… Implemented" [ref=e287] + - row "M12 CLI βœ… Implemented" [ref=e288]: + - cell "M12 CLI" [ref=e289] + - cell "βœ… Implemented" [ref=e290] + - row "M13 Onboarding βœ… Implemented" [ref=e291]: + - cell "M13 Onboarding" [ref=e292] + - cell "βœ… Implemented" [ref=e293] + - row "X01 Transport βœ… Implemented (FastAPI)" [ref=e294]: + - cell "X01 Transport" [ref=e295] + - cell "βœ… Implemented (FastAPI)" [ref=e296] + - row "X02 Events βœ… Implemented (SQLite)" [ref=e297]: + - cell "X02 Events" [ref=e298] + - cell "βœ… Implemented (SQLite)" [ref=e299] + - row "X03 Observability βœ… Implemented" [ref=e300]: + - cell "X03 Observability" [ref=e301] + - cell "βœ… Implemented" [ref=e302] + - row "X04 Config βœ… Implemented" [ref=e303]: + - cell "X04 Config" [ref=e304] + - cell "βœ… Implemented" [ref=e305] + - contentinfo "Gradio footer navigation" [ref=e97]: + - button "Über API verwenden Logo" [ref=e98] [cursor=pointer]: + - text: Über API verwenden + - img "Logo" [ref=e99] + - generic [ref=e100]: Β· + - link "Mit Gradio erstellt Logo" [ref=e101] [cursor=pointer]: + - /url: https://gradio.app + - text: Mit Gradio erstellt + - img "Logo" [ref=e102] + - generic [ref=e103]: Β· + - button "Einstellungen Einstellungen" [ref=e104] [cursor=pointer]: + - text: Einstellungen + - img "Einstellungen" [ref=e105] \ No newline at end of file diff --git a/README.md b/README.md index 25d0e1ad40a9f3c84de6a105e54020a077eb41ed..b7d1242e314aa6e3ea19ee4884d2a5d9a312e407 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ ---- +ο»Ώ--- title: HearthNet -emoji: 🐒 +emoji: πŸ”₯ colorFrom: purple colorTo: pink sdk: gradio -sdk_version: 6.16.0 -python_version: '3.12' +sdk_version: 6.17.3 +python_version: "3.13" app_file: app.py pinned: true short_description: Community-Owned AI That Works Even When The Internet Doesn't @@ -16,352 +16,249 @@ short_description: Community-Owned AI That Works Even When The Internet Doesn't ### Community-Owned AI That Works Even When The Internet Doesn't

- HearthNet Banner -

- -

- -🌐 Local-First AI β€’ 🀝 Community-Powered β€’ πŸ›‘οΈ Resilient by Design β€’ ⚑ Offline-Capable - + Local-First AI  β€’  Community-Powered  β€’  Resilient by Design  β€’  Offline-Capable

--- -## 🚨 The Problem - -Today's AI depends on centralized cloud infrastructure. +## What is HearthNet? -When the internet goes down, so does access to: +HearthNet transforms the computers already in your community into a **resilient, fully local AI network**. -* πŸ€– AI assistants -* πŸ“š Knowledge bases -* πŸ’¬ Communication platforms -* πŸ›’ Marketplaces -* πŸ—ΊοΈ Navigation and local information - -Whether caused by outages, disasters, censorship, infrastructure failures, or simply poor connectivity, modern communities become digitally isolated almost instantly. - -What if AI could continue working anyway? +- No centralized cloud required +- No single point of failure +- Works during internet outages, censorship, or infrastructure failures +- Real local AI inference via Ollama, llama.cpp, or Hugging Face Transformers --- -# πŸ”₯ Introducing HearthNet - -**HearthNet** transforms the computers already around you into a resilient local AI network. +## Screenshots -Your gaming PC. +### Ask (LLM chat with local AI) -Your neighbor's old laptop. +![Ask tab](docs/screenshots/ui-ask-tab.png) -A Raspberry Pi in a community center. +### Local Chat -An unused workstation in a local business. +![Chat tab](docs/screenshots/ui-chat-tab.png) -Together, they become a shared AI cooperative. +### Community Marketplace -No centralized server required. +![Marketplace tab](docs/screenshots/ui-marketplace-tab.png) -No cloud dependency required. - -No single point of failure. - ---- +### Emergency / Connectivity Status -## 🎬 Imagine This Scenario +![Emergency tab](docs/screenshots/ui-emergency-tab.png) -A storm damages the internet connection for an entire neighborhood. +### Settings & Identity -Most online services become unavailable. - -With HearthNet: - -βœ… AI assistants still answer questions - -βœ… Local document search continues working - -βœ… Emergency information remains available - -βœ… Community chat stays online - -βœ… Local marketplaces continue operating - -βœ… Knowledge remains accessible - -The internet disappears. - -The community does not. +![Settings tab](docs/screenshots/ui-settings-tab.png) --- -# ⚑ Key Features - -## πŸ€– Distributed AI Inference - -Every node advertises its capabilities. - -A gaming PC might offer: +## Quick Start -* Large Language Models -* Vision Models -* Embeddings +```bash +# Clone +git clone https://github.com/HearthNet/hearthnet +cd hearthnet -A Raspberry Pi might offer: +# Install +pip install -e ".[dev]" -* Message relaying -* Local storage -* Discovery services - -Requests are automatically routed to the most suitable node. - ---- - -## πŸ“š Local Knowledge (RAG) - -HearthNet can host community knowledge libraries including: - -* First aid manuals -* Emergency procedures -* Community resources -* Local guides -* Educational content - -Even completely offline. - ---- - -## 🌐 Automatic Discovery - -No manual configuration. - -No IP addresses. - -No complex setup. - -Open HearthNet and nearby nodes automatically discover each other. - ---- +# Run Gradio UI +python app.py +# open http://127.0.0.1:7860 +``` -## πŸ›‘οΈ Internet-Outage Resilience +### With a local LLM (Ollama) -When the internet is available: +```bash +# Install Ollama: https://ollama.com +ollama pull llama3 -```text -Community ↔ Internet ↔ Cloud Services +# Start HearthNet β€” it will auto-detect Ollama +python app.py ``` -When the internet disappears: +### CLI -```text -Community ↔ Community ↔ Community +```bash +python -m hearthnet.cli --help +python -m hearthnet.cli node info +python -m hearthnet.cli ask "What is HearthNet?" ``` -HearthNet automatically switches to local-first operation. - ---- - -## πŸ’¬ Local Community Communication - -Communities can communicate directly through: - -* Chat -* Announcements -* Local messaging -* Shared information boards - -No external servers required. - --- -## πŸ›’ Community Marketplace +## Architecture -Buy. - -Sell. - -Trade. - -Share resources. - -Even during connectivity disruptions. - ---- - -# πŸ—οΈ Architecture - -```text -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Frontend β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Capability Bus β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β–Ό β–Ό β–Ό - -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ LLM β”‚ β”‚ RAG β”‚ β”‚ Chat β”‚ -β”‚ Node β”‚ β”‚ Node β”‚ β”‚ Node β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - - β–² β–² β–² - -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚Laptop Aβ”‚ β”‚Laptop Bβ”‚ β”‚Pi Zero β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` ++----------------------------------------------------------+ +| Gradio UI (app.py) . FastAPI Transport (X01) | ++-----------------------------+----------------------------+ + | Capability Bus (M03) + +-------------------+-------------------+ + v v v + +--------------+ +-------------+ +-----------------------+ + | LLM (M04) | | RAG (M05) | | Group Chat (M25) | + | Ollama | | Chroma | | ThreadService | + | llama.cpp | | Embed | | ThreadViewStore | + | HF Transfm | | Ingest | +-----------------------+ + +--------------+ +-------------+ + v v + +--------------+ +------------------------------------------+ + | Identity | | Phase 2 Services | + | Ed25519 | | OCR (M17) . Translation (M18) | + | X3DH | | STT/TTS (M19) . Vision (M20) | + | Ratchet | | Tool Calls (M21) . Mobile (M22) | + | Tokens | | E2E Encrypt (M23) . Rerank (M24) | + +--------------+ +------------------------------------------+ + | + +--------------------------+-------------------+ + v v v ++----------+ +-------------------+ +--------------------+ +| DHT/P2P | | Federation (M14) | | Observability (X03)| +| Kademlia| | Relay Tier (M15) | | Metrics . Tracing | +| WebSocket | federated metrics| | JSON logging | ++----------+ +-------------------+ +--------------------+ ``` --- -# πŸš€ Hackathon Goals - -Our hackathon MVP demonstrates: - -### Phase 1 - -* [x] Node discovery -* [x] Capability registration -* [x] Local AI inference -* [x] Knowledge retrieval -* [x] Offline operation - -### Phase 2 - -* [ ] Local marketplace -* [ ] Community messaging -* [ ] Health monitoring -* [ ] Distributed storage - -### Phase 3 - -* [ ] Long-distance mesh networking -* [ ] Federated communities -* [ ] Distributed inference -* [ ] Federated learning - ---- - -# 🧠 Why This Matters - -AI is rapidly becoming essential infrastructure. - -Today, that infrastructure is mostly controlled by a handful of organizations. - -HearthNet explores a different future: - -A future where communities own and operate AI together. - -A future where knowledge remains available during outages. - -A future where local resilience is built directly into our digital systems. - ---- - -# 🌍 Potential Use Cases - -## πŸš‘ Emergency Response - -Access critical information when external services are unavailable. - ---- - -## 🏫 Schools - -Run local educational AI systems without constant cloud access. +## Module Reference + +### Phase 1 β€” Core Infrastructure + +| Module | Description | Status | +|--------|-------------|--------| +| M01 | Node identity (Ed25519, manifests, canonical JSON) | done | +| M02 | Peer discovery (mDNS, UDP broadcast, PeerRegistry) | done | +| M03 | Capability bus (schema validation, routing, tracing) | done | +| M04 | LLM service (Ollama, llama.cpp, HF Transformers, OpenAI fallback) | done | +| M05 | RAG / knowledge (chunker, ChromaDB, IngestPipeline) | done | +| M06 | Marketplace (event-sourced, local-first) | done | +| M07 | File blobs (BLAKE3 hash, chunking, FileService) | done | +| M08 | Gradio UI (6 tabs: Ask, Chat, Marketplace, Files, Emergency, Settings) | done | +| M09 | Emergency mode (async connectivity probe loop) | done | +| M10 | Chat (event-backed 1:1 messaging) | done | +| M11 | Embeddings (embed.text, SimpleHashBackend) | done | +| M12 | CLI (click, ask / node info / marketplace) | done | +| M13 | Onboarding (invite QR, hnvite:// deep links) | done | +| X01 | Transport (FastAPI server, 12 REST endpoints) | done | +| X02 | Events (SQLite, Lamport clocks, ReplayEngine) | done | +| X03 | Observability (structured JSON logging, metrics, tracing) | done | +| X04 | Config (typed frozen Config, TOML, env overlay) | done | + +### Phase 2 β€” Advanced Features + +| Module | Description | Status | +|--------|-------------|--------| +| M14 | Federation (bilateral cross-community trust, manifest signing) | done | +| M15 | Relay tier (NAT traversal, keepalive, push token registry) | done | +| M16 | Capability tokens (Ed25519 JWS-style hntoken://v1/ format) | done | +| M17 | OCR (Tesseract + TrOCR backends, graceful degradation) | done | +| M18 | Translation (NLLB backend, LRU cache, 4000-char limit) | done | +| M19 | STT/TTS (Whisper local STT, Edge TTS synthesis) | done | +| M20 | Vision (Florence-2 image describe, generate placeholder) | done | +| M21 | Tool calls (LLM mid-generation bus dispatch, ToolExecutor) | done | +| M22 | Mobile native (Flutter contract, hnapp:// invites, push authority) | done | +| M23 | E2E encryption (X3DH key agreement, Double Ratchet, envelope) | done | +| M24 | Reranking (BGE + CrossEncoder backends, 100-doc limit) | done | +| M25 | Group chat (ThreadService, ThreadViewStore, event-sourced) | done | +| X05 | DHT (Kademlia node, 256-bucket routing table, bootstrap) | done | +| X06 | WebSocket upgrade (bidirectional pubsub, WsClient) | done | +| X07 | Federated metrics (NodeMetricsTick, MetricsAggregator, OTLP) | done | + +### Phase 3 β€” Research / Experimental + +All Phase 3 modules are feature-flag gated (config.research.*). +They are experimental and not enabled by default. + +| Module | Description | Status | +|--------|-------------|--------| +| M26 | Distributed inference (ShardDescriptor, PipelineOrchestrator) | experimental | +| M27 | MoE routing (ExpertRegistry, MoeRouter) | experimental | +| M28 | Federated learning (FedLearnCoordinator, RoundManifest) | experimental | +| M29 | LoRa beacons (32-byte frames, long-range low-bandwidth signaling) | experimental | +| M30 | Evidence graph / EBKH (ClaimStore, attestations, disputes) | experimental | +| M31 | Civil defense NRW (AuditChain, role certs, structured alerts) | experimental | --- -## 🌾 Rural Communities +## Local-First AI Backends -Provide AI services in areas with limited connectivity. +HearthNet uses **real local models** β€” no mocks, no fake responses. ---- +Priority order: -## πŸŽͺ Events & Festivals +1. **Ollama** (preferred β€” zero-config, 70+ models) +2. **llama.cpp** HTTP server +3. **Hugging Face Transformers** (local model files) +4. **OpenAI API** β€” opt-in online fallback only, never the default -Create temporary local AI networks for attendees. +If no backend is reachable the service returns `{"status": "unavailable"}` and the UI shows a clear degraded state message. --- -## 🏘️ Neighborhoods +## Security -Community knowledge sharing and local services. +- Ed25519 signatures on all node manifests and capability tokens +- X3DH key agreement + Double Ratchet for end-to-end encrypted chat +- BLAKE3 content-addressed file blobs +- All CLI HTTP requests restricted to localhost only +- Emergency probes use TLS verification (verify=True) +- Bandit HIGH findings: 0 --- -# πŸ› οΈ Technology Stack - -### AI - -* llama.cpp -* Ollama -* Hugging Face Models -* Sentence Transformers - -### Backend +## Quality Gates -* Python -* FastAPI -* WebSockets +```bash +# Lint +ruff check hearthnet/ tests/ -### Knowledge Layer +# Type check +mypy hearthnet/ -* ChromaDB -* FAISS -* Local Embeddings +# Security scan +bandit -r hearthnet/ -ll -### Infrastructure +# Unit + integration tests (62 total) +python -m pytest tests/ -q -* Docker -* Docker Compose - -### Frontend - -* Gradio / Web UI +# E2E browser tests (Playwright) +python -m pytest tests/test_e2e_playwright.py -v +``` --- -# πŸ‘₯ Team - -Built during the Hugging Face Hackathon ❀️ - -We believe that the future of AI should be: - -βœ… Open +## Test Coverage -βœ… Local - -βœ… Resilient - -βœ… Community-Owned +| Suite | Tests | Notes | +|-------|-------|-------| +| Phase 1 (M01-M13, X01-X04) | 13 | Core bus, routing, emergency, snapshot | +| Phase 2 (M14-M25, X05-X07) | 23 | Crypto, tokens, federation, OCR, chat, DHT | +| Phase 3 experimental | 15 | Distributed inference, MoE, fedlearn, LoRa, evidence, civdef | +| E2E Playwright browser | 11 | All 6 tabs, API health, mobile viewport | +| **Total** | **62** | All passing | --- -# 🀝 Join Us - -We're actively looking for: +## Contributing -* AI Engineers -* Distributed Systems Developers -* Frontend Developers -* DevOps Engineers -* Designers -* Community Builders - -Ideas, feedback, contributions, and discussions are always welcome. - ---- +See [tasks.md](tasks.md) for current status, known gaps, and next steps. -# 🌟 Vision +Architecture decisions are documented in the `docs/` folder: -> "The cloud owns AI today. -> -> HearthNet is our bet that communities will own it tomorrow." +- [docs/00-OVERVIEW.md](docs/00-OVERVIEW.md) β€” system overview +- [docs/CAPABILITY_CONTRACT.md](docs/CAPABILITY_CONTRACT.md) β€” capability API contract +- [docs/GLOSSARY.md](docs/GLOSSARY.md) β€” terminology +- [docs/roadmap.md](docs/roadmap.md) β€” roadmap +- [docs/p2_p3/](docs/p2_p3/) β€” Phase 2 and 3 specs ---

- Built with ❀️, β˜•, open source, and a slightly unreasonable belief that neighborhoods should have their own AI. + Built with open source models and the belief that communities should own their AI.

diff --git a/docs/impl_ref.md b/docs/impl_ref.md new file mode 100644 index 0000000000000000000000000000000000000000..3f7308fe2cd6bb8526720bea4c5682a6bd2db5c9 --- /dev/null +++ b/docs/impl_ref.md @@ -0,0 +1,1634 @@ +# HearthNet β€” Implementation Reference + +**Purpose:** complete inventory of every file, class, method, and function across the spec set. +Use this as a coding checklist. For *why* and behaviour β†’ follow the spec link on each entry. +For *what to call it* and *what it returns* β†’ this document is normative. + +**Spec set:** see [`00-OVERVIEW.md`](00-OVERVIEW.md), [`GLOSSARY.md`](GLOSSARY.md), [`CAPABILITY_CONTRACT.md`](CAPABILITY_CONTRACT.md). + +--- + +## 0. Conventions (read first) + +### 0.1 Type aliases β€” `hearthnet/types.py` + +*Re-exported by every module that uses them. Never invent synonyms.* β€” [00-OVERVIEW Β§4.1](00-OVERVIEW.md) + +| Alias | Underlying | Example | +|-------|------------|---------| +| `NodeID` | `str` | `"ed25519:7H4G-Y9KL-2P3M-X8QR"` (short) or full base64-url | +| `CommunityID` | `str` | full base64-url | +| `CapabilityName` | `str` | `"llm.chat"` | +| `Version` | `tuple[int, int]` | `(1, 0)` | +| `Lamport` | `int` | monotonic per community | +| `CID` | `str` | `"blake3:"` | +| `EventID` | `str` | ULID | +| `TraceID` | `str` | ULID | +| `SchemaHash` | `str` | `"blake3:"` | +| `WallClock` | `str` | `"2026-05-26T08:14:22Z"` | +| `Signature` | `str` | `"ed25519:"` | +| `Topic` | `str` | `"marketplace.post.created"` | +| `ErrorCode` | `Literal[...]` | see [CONTRACT Β§9](CAPABILITY_CONTRACT.md) | +| `TrustLevel` | `Literal["unknown","member","trusted","anchor"]` | | +| `Profile` | `Literal["anchor","hearth","spark","bridge"]` | | +| `Stability` | `Literal["experimental","beta","stable"]` | | +| `Mode` | `Literal["online","degraded","offline"]` | emergency state | +| `Category` | `Literal["offer","request","info","emergency"]` | marketplace post | +| `EventType` | `Literal[...]` | 19 values; see [X02 Β§3.1](cross-cutting/X02-events.md) | + +### 0.2 Constants β€” `hearthnet/constants.py` + +Hardcoded; never configurable. Complete list in [GLOSSARY.md Β§Defaults](GLOSSARY.md). + +`MANIFEST_TTL_SECONDS=30`, `MANIFEST_REPUBLISH_INTERVAL_SECONDS=20`, `DISCOVERY_UDP_INTERVAL_SECONDS` (5 active / 30 stable), `EMERGENCY_PROBE_INTERVAL_ONLINE=10`, `EMERGENCY_PROBE_INTERVAL_OFFLINE=2`, `EMERGENCY_PROBE_TIMEOUT_SECONDS=2`, `EMERGENCY_TRANSITION_DEBOUNCE_SECONDS=30`, `CONNECTION_IDLE_SECONDS=60`, `RECONNECT_BACKOFF_CAP_SECONDS=30`, `STREAM_WINDOW_FRAMES=16`, `STREAM_ACK_INTERVAL_FRAMES=8`, `STREAM_ACK_TIMEOUT_SECONDS=5`, `RPC_DEFAULT_TIMEOUT_SECONDS=30`, `LLM_GENERATION_DEFAULT_TIMEOUT_SECONDS=120`, `CHUNK_SIZE_BYTES=262144`, `BLOB_GC_DISK_THRESHOLD=0.80`, `RAG_CHUNK_TOKENS=1000`, `RAG_CHUNK_OVERLAP_TOKENS=200`, `RAG_DEFAULT_K=5`, `RAG_MAX_K=20`, `HEALTH_WINDOW_CALLS=100`, `HEALTH_QUARANTINE_THRESHOLD=0.5`, `HEALTH_QUARANTINE_SECONDS=60`, `RATE_LIMIT_SOFT_RPS_PER_CAP=10`, `RATE_LIMIT_HARD_RPS_PER_CAP=100`, `RATE_LIMIT_SOFT_RPS_TOTAL=100`, `RATE_LIMIT_HARD_RPS_TOTAL=1000`, `EVENT_LOG_RETENTION_DAYS=30`, `SNAPSHOT_LAG_LAMPORT=1000`, `TRACE_RING_BUFFER=10000`, `LOG_RETENTION_DAYS=14`. + +### 0.3 Naming rules + +- Functions: `snake_case`, verb-first +- Classes: `PascalCase`, noun +- Constants: `SCREAMING_SNAKE` +- Async I/O: `async def`; no `async_` prefix on names +- Protocols: `PascalCase` ending in capability noun (`LlmBackend`, `Service`) +- Private: leading underscore (`_compute_canonical_json`) + +### 0.4 Universal error code β†’ wire code mapping + +| Domain exception | Wire `ErrorCode` | HTTP | +|------------------|------------------|------| +| `IdentityError("invalid_signature")` | `invalid_signature` | 401 | +| `IdentityError("expired")` | `expired` | 410 | +| `BusError("schema_mismatch")` | `schema_mismatch` | 400 | +| `BusError("not_found")` | `not_found` | 404 | +| `BusError("capacity_exceeded")` | `capacity_exceeded` | 429 | +| `BusError("quarantined" / "partition")` | `partition` | 503 | +| `BusError("timeout")` | `timeout` | 408 | +| `EventLogError("invalid_signature")` | `invalid_signature` | β€” (internal) | +| `BlobError("not_found")` | `not_found` | 404 | +| `BlobError("hash_mismatch")` | `bad_request` | 400 | +| `BlobError("disk_full")` | `capacity_exceeded` | 429 | +| `OnboardingError(*)` | β€” (local UI only) | β€” | +| `ConfigError(*)` | β€” (startup) | β€” | + +--- + +## 1. X04 β€” Configuration + +**Spec:** [`cross-cutting/X04-config.md`](cross-cutting/X04-config.md) Β· **Path:** `hearthnet/config.py` + `hearthnet/constants.py` + +### `hearthnet/config.py` + +#### Dataclasses (all `@dataclass(frozen=True)`) + +`IdentityConfig` β€” Β§3: +- `keys_dir: Path` +- `auto_generate: bool = True` + +`CommunityConfig` β€” Β§3: +- `community_id: Optional[str] = None` +- `state_dir: Path = Path()` + +`TransportConfig` β€” Β§3: +- `host: str = "0.0.0.0"` +- `port: int = 7080` +- `tls_cert: Optional[Path] = None` +- `tls_key: Optional[Path] = None` + +`DiscoveryConfig` β€” Β§3: +- `mdns_enabled: bool = True` +- `udp_enabled: bool = True` +- `udp_multicast_group: str = "239.255.42.42"` +- `udp_port: int = 42424` +- `relay_urls: list[str] = []` + +`BusConfig` β€” Β§3: +- `prefer_local: bool = True` +- `local_load_threshold: float = 0.80` + +`LlmBackendConfig` β€” Β§3: +- `name: str` β€” one of `"llama_cpp" | "ollama" | "lmstudio" | "vllm" | "hf_api" | "anthropic_api" | "nemotron" | "openbmb"` +- `url: Optional[str] = None` +- `model: Optional[str] = None` +- `api_key_env: Optional[str] = None` + +`LlmConfig` β€” Β§3: +- `backends: list[LlmBackendConfig] = []` + +`EmbeddingConfig` β€” Β§3: +- `model: str = "BAAI/bge-small-en-v1.5"` +- `device: str = "auto"` + +`RagConfig` β€” Β§3: +- `enabled: bool = True` +- `corpora_dir: Path = Path()` + +`FileConfig` β€” Β§3: +- `blobs_dir: Path = Path()` +- `gc_threshold: float = 0.80` + +`MarketConfig` β€” Β§3: +- `enabled: bool = True` +- `default_ttl_seconds: int = 604800` +- `max_ttl_seconds: int = 2592000` + +`ChatConfig` β€” Β§3: +- `enabled: bool = True` +- `store_and_forward: bool = True` + +`EmergencyConfig` β€” Β§3: +- `probe_targets: list[str] = ["1.1.1.1","8.8.8.8","cloudflare.com","quad9.net"]` + +`UiConfig` β€” Β§3: +- `host: str = "127.0.0.1"` +- `port: int = 7860` +- `launch_browser: bool = True` + +`ObservabilityConfig` β€” Β§3 (+ trackio addition): +- `log_level: str = "info"` +- `log_dir: Path = Path()` +- `metrics_enabled: bool = True` +- `otlp_endpoint: Optional[str] = None` +- `trackio_project: Optional[str] = None` β€” local trackio project name; enables trackio exporter when set +- `trackio_space: Optional[str] = None` β€” HF Space URL to mirror trackio runs to; optional + +`Config` β€” Β§3: +- holds one of each of the above as named attributes + +#### Functions + +- `load(path: Path | None = None) -> Config` β€” Β§4. Read TOML, apply defaults, resolve paths, validate. Raises `ConfigError`. +- `default_config() -> Config` β€” Β§4. All-defaults Config. +- `save(config: Config, path: Path | None = None) -> None` β€” Β§4. Atomic TOML write. +- `resolve_paths(config: Config) -> Config` β€” Β§4. Resolve empty `Path()` to XDG locations. Idempotent. +- `validate(config: Config) -> None` β€” Β§4. Cross-field checks; raises `ConfigError`. + +#### Exception + +`ConfigError(Exception)` β€” Β§4: +- `__init__(code: str, **details)` +- `code: str` +- `details: dict` + +### `hearthnet/constants.py` + +Module-level constants from [GLOSSARY.md Β§Defaults](GLOSSARY.md). No classes; just `NAME = value` lines. + +--- + +## 2. X03 β€” Observability + +**Spec:** [`cross-cutting/X03-observability.md`](cross-cutting/X03-observability.md) Β· **Path:** `hearthnet/observability/` + +### `hearthnet/observability/logging.py` β€” Β§3 + +#### Functions + +- `configure(config: ObservabilityConfig) -> None` β€” install handlers + rotation; idempotent. +- `get_logger(name: str) -> logging.Logger` β€” JSON-formatted logger. + +#### Class + +`JsonFormatter(logging.Formatter)` β€” Β§3.1: +- `format(record: LogRecord) -> str` β€” emit `{"ts","level","logger","msg",**extras}`. + +`RateLimitedLogger` β€” Β§3.2 (internal wrapper): +- `__init__(logger: Logger, per_key_seconds: float = 1.0)` +- `info(msg: str, key: str, **extras) -> None` +- `warning(msg: str, key: str, **extras) -> None` + +### `hearthnet/observability/metrics.py` β€” Β§4 + +#### Functions + +- `configure(config: ObservabilityConfig) -> None` β€” set up registries, start `/metrics` endpoint. +- `counter(name: str, doc: str, labels: list[str] = []) -> Counter` +- `histogram(name: str, doc: str, labels: list[str] = [], buckets: list[float] | None = None) -> Histogram` +- `gauge(name: str, doc: str, labels: list[str] = []) -> Gauge` +- `disabled() -> bool` β€” true when metrics are off. + +#### Standard metric set β€” Β§4.2 + +Pre-registered at startup with these exact names: +`hearthnet_requests_total{capability,result}`, `hearthnet_request_duration_ms{capability,quantile}`, `hearthnet_active_streams{capability}`, `hearthnet_nodes_online{community}`, `hearthnet_event_log_size{community}`, `hearthnet_event_log_lamport_head{community}`, `hearthnet_emergency_mode{state}`, `hearthnet_blob_storage_bytes`, `hearthnet_llm_tokens_generated_total{model,backend}`, `hearthnet_llm_concurrent{model}`, `hearthnet_capability_health_success_rate{capability,node}`, `hearthnet_rate_limited_total{capability,reason}`, `hearthnet_signature_failures_total{reason}`, `hearthnet_quarantines_total`. + +#### Trackio integration (new) β€” Β§4.4 + +`TrackioExporter` β€” optional. Activated when `config.observability.trackio_project` is set. +- `__init__(project: str, space: str | None = None)` β€” opens a trackio run. +- `record_call(capability: str, model: str | None, latency_ms: float, tokens_in: int | None, tokens_out: int | None, result: str) -> None` β€” logs one inference call as a step in the run. +- `record_topology_snapshot(snapshot: TopologySnapshot) -> None` β€” periodic mesh health log. +- `close() -> None` + +Use trackio when you want a Gradio-native dashboard for run/inference history (alternative or complement to Prometheus). Bridged into `TraceHook.on_call_end` when active. + +### `hearthnet/observability/tracing.py` β€” Β§5 + +#### Dataclasses + +`Trace` β€” Β§5.1: +- `trace_id: str` (ULID) +- `capability: str` +- `started_at: float` +- `spans: list[Span]` + +`Span` β€” Β§5.1: +- `name: str` +- `started_at: float` +- `ended_at: float | None` +- `extras: dict` + +#### Functions + +- `new_trace(capability: str) -> Trace` β€” open a new trace, attach to current task. +- `current_trace() -> Trace | None` +- `attach(trace: Trace) -> None` +- `detach() -> None` β€” close the current trace, push to ring buffer. +- `span(name: str, **extras) -> AbstractAsyncContextManager[Span]` β€” open a sub-span. +- `get_recent(n: int = 100) -> list[Trace]` β€” read from ring buffer (size `TRACE_RING_BUFFER`). + +### `hearthnet/observability/doctor.py` β€” Β§6 + +#### Dataclass + +`CheckResult` β€” Β§6.1: +- `name: str` +- `ok: bool` +- `detail: str` +- `fix: str | None` + +#### Functions + +- `register(name: str, check: Callable[[Config, CapabilityBus], CheckResult]) -> None` +- `run_all(config: Config, bus: CapabilityBus) -> list[CheckResult]` +- `run_one(name: str, config: Config, bus: CapabilityBus) -> CheckResult` + +#### Standard checks (registered at startup) β€” Β§6.2 + +`keys_present`, `keys_loadable`, `community_present`, `event_log_writable`, `mdns_socket`, `udp_multicast`, `transport_port`, `at_least_one_capability`, `disk_space`, `clock_sanity`, `llm_backend_reachable`, `recent_error_rate`. + +--- + +## 3. X02 β€” Events + +**Spec:** [`cross-cutting/X02-events.md`](cross-cutting/X02-events.md) Β· **Path:** `hearthnet/events/` + +### `hearthnet/events/types.py` β€” Β§3.1 + +`EventType` β€” Literal of 19 strings, exactly: +`community.created`, `community.member.invited`, `community.member.joined`, `community.member.revoked`, `community.member.promoted`, `community.member.demoted`, `community.policy.updated`, `node.manifest.updated`, `market.post.created`, `market.post.updated`, `market.post.expired`, `chat.message.sent`, `chat.message.delivered`, `chat.message.read`, `file.cid.advertised`, `file.cid.unpinned`, `rag.document.ingested`, `federation.peer.added`, `federation.peer.removed`. + +`Event` *(frozen dataclass)*: +- `schema_version: int` +- `event_id: str` +- `lamport: int` +- `wall_clock: str` +- `community_id: str` +- `author: str` +- `event_type: EventType` +- `data: dict` +- `signature: str` + +### `hearthnet/events/lamport.py` β€” Β§3.2 + +`LamportClock`: +- `__init__(conn: sqlite3.Connection, community_id: str)` β€” load current value. +- `current: int` *(property)* +- `tick_for_send() -> int` β€” increment + persist; returns new value. +- `observe(received_lamport: int) -> None` β€” `max(current, received) + 1`. + +### `hearthnet/events/log.py` β€” Β§3.3 + +`EventLog`: +- `__init__(db_path: Path, community_id: str)` β€” open/create SQLite (WAL); apply schema. +- `append_local(event_type: EventType, data: dict, author_kp: KeyPair) -> Event` β€” mint, sign, persist, fan out. +- `append_received(event: Event) -> bool` β€” verify, persist if new. Returns True if new. +- `head() -> int` β€” highest Lamport. +- `get(event_id: str) -> Event | None` +- `replay(*, since_lamport: int = 0, event_types: list[EventType] | None = None, limit: int | None = None) -> Iterator[Event]` +- `heads_by_type() -> dict[EventType, int]` +- `subscribe(event_types: list[EventType] | None = None) -> AsyncIterator[Event]` + +`EventLogError(Exception)`: +- `code in {"invalid_signature","out_of_order","unknown_author","revoked_author","schema_unknown","db_corrupt"}` + +### `hearthnet/events/replay.py` β€” Β§3.4 + +`MaterialisedView` *(Protocol)*: +- `reset() -> None` +- `apply(event: Event) -> None` +- `snapshot_state() -> dict` +- `restore_state(state: dict) -> None` + +`ReplayEngine`: +- `__init__(log: EventLog)` +- `register(name: str, view: MaterialisedView, event_types: list[EventType]) -> None` +- `rebuild(view_name: str, from_lamport: int = 0) -> None` +- `rebuild_all() -> None` +- `on_event(event: Event) -> None` β€” wired from `EventLog`. + +### `hearthnet/events/snapshot.py` β€” Β§3.5 + +`Snapshot` *(frozen dataclass)*: +- `schema_version: int` +- `community_id: str` +- `lamport: int` +- `wall_clock: str` +- `state: dict` +- `covers_events_up_to: int` +- `signature: str` + +`SnapshotStore`: +- `__init__(dir_path: Path, community_id: str)` +- `latest() -> Snapshot | None` +- `write(snap: Snapshot) -> None` β€” atomic. +- `list() -> list[int]` +- `prune(keep_last_n: int = 7) -> None` + +Free functions: +- `build_snapshot(log: EventLog, engine: ReplayEngine, signing_kp: KeyPair, at_lamport: int | None = None) -> Snapshot` +- `restore_from_snapshot(snap: Snapshot, engine: ReplayEngine, log: EventLog) -> None` + +### `hearthnet/events/sync.py` β€” Β§3.6 + +`HeadsReport` *(frozen dataclass)*: +- `community_id: str` +- `heads_by_type: dict[EventType, int]` +- `head: int` + +`SyncResult` *(frozen dataclass)*: +- `sent_count: int` +- `received_count: int` +- `duration_ms: int` + +`SyncClient`: +- `__init__(log: EventLog, transport_client: HttpClient)` +- `sync_with(peer_endpoint: Endpoint) -> SyncResult` *(async)* +- `run_round(peer_registry: PeerRegistry) -> list[SyncResult]` *(async)* β€” sync against all known peers. + +`SyncServer`: +- `__init__(log: EventLog)` +- `serve_heads() -> HeadsReport` *(async)* +- `serve_events(events: list[Event]) -> dict` *(async)* + +--- + +## 4. X01 β€” Transport + +**Spec:** [`cross-cutting/X01-transport.md`](cross-cutting/X01-transport.md) Β· **Path:** `hearthnet/transport/` + +### `hearthnet/transport/server.py` β€” Β§3 + +`HttpServer`: +- `__init__(config: TransportConfig, kp: KeyPair, bus: CapabilityBus, event_sync: SyncServer, community_manifest_provider: Callable[[], CommunityManifest])` +- `app() -> FastAPI` β€” for tests. +- `run() -> None` *(async)* β€” block, serve. +- `shutdown() -> None` *(async)* + +#### Mounted endpoints β€” Β§3.2 + +`POST /bus/v1/call`, `GET /manifest`, `GET /community/manifest`, `GET /sync/v1/heads`, `POST /sync/v1/events`, `GET /pubsub/v1/subscribe`, `GET /health`, `GET /ready`, `GET /metrics`, `GET /trace/recent`. + +`PubSubServer` β€” Β§8: +- `publish(topic: str, payload: dict) -> None` *(async)* +- `subscribe(topic: str, *, last_seq: int = 0, timeout_seconds: float = 30) -> dict` *(async)* β€” long-poll. + +### `hearthnet/transport/client.py` β€” Β§5 + +`HttpClient`: +- `__init__(kp: KeyPair, node_id: str, community_id: str, pinned_certs: PinnedCerts, timeout_default_seconds: float = RPC_DEFAULT_TIMEOUT_SECONDS)` +- `call(peer: Endpoint, capability: str, version: str, body: dict, *, trace_id: str | None = None, timeout_seconds: float | None = None) -> dict` *(async)* β€” signed RPC. +- `stream(peer: Endpoint, capability: str, version: str, body: dict, *, trace_id: str | None = None, cancel: asyncio.Event | None = None) -> AsyncIterator[Frame]` β€” signed stream. +- `close() -> None` *(async)* + +`CallError(Exception)`: +- `code: ErrorCode` +- `message: str` +- `retry_after_ms: int | None` +- `alt_capabilities: list[str]` +- `alt_nodes: list[str]` + +### `hearthnet/transport/streams.py` β€” Β§6 + +`Frame` *(frozen dataclass)*: +- `event: str` β€” `"token" | "chunk" | "progress" | "ack" | "done" | "error" | "manifest" | "ready" | "tool_call_delta"` +- `data: dict` +- `seq: int` + +`SseWriter`: +- `__init__(response: StreamingResponse)` +- `emit(event: str, data: dict) -> None` *(async)* +- `emit_token(token: dict) -> None` *(async)* +- `emit_progress(current: int, total: int, stage: str) -> None` *(async)* +- `emit_error(code: ErrorCode, **kwargs) -> None` *(async)* +- `emit_done(**meta) -> None` *(async)* +- `emit_ack(upto: int) -> None` *(async)* +- `cancelled: bool` *(property)* + +`SseReader`: +- `__aiter__() -> AsyncIterator[Frame]` +- `cancel() -> None` *(async)* + +### `hearthnet/transport/backpressure.py` β€” Β§6.3 + +`FlowControl`: +- `__init__(window: int = STREAM_WINDOW_FRAMES, ack_interval: int = STREAM_ACK_INTERVAL_FRAMES)` +- `window_used: int` *(property)* +- `send() -> None` *(async)* β€” await if window full. +- `ack(upto: int) -> None` +- `needs_ack: bool` *(property)* + +### `hearthnet/transport/tls.py` β€” Β§4 + +`PinnedCerts`: +- `__init__(db_path: Path)` +- `record(node_id: str, fingerprint: bytes) -> None` +- `expected(node_id: str) -> bytes | None` +- `verify(node_id: str, presented: bytes) -> bool` + +### `hearthnet/transport/__init__.py` β€” Β§7 + +`RateCheck` *(frozen dataclass)*: +- `allowed: bool` +- `soft_exceeded: bool` +- `retry_after_ms: int` + +`RateLimiter`: +- `__init__(config: TransportConfig)` +- `check(peer_node_id: str, capability: str) -> RateCheck` + +--- + +## 5. M01 β€” Identity & Manifests + +**Spec:** [`modules/M01-identity.md`](modules/M01-identity.md) Β· **Path:** `hearthnet/identity/` + +### `hearthnet/identity/keys.py` β€” Β§3.1 + +#### Class + +`KeyPair` *(frozen dataclass)*: +- `signing_key: nacl.signing.SigningKey` +- `verify_key: nacl.signing.VerifyKey` +- `node_id_full: str` +- `node_id_short: str` +- `sign(payload: dict) -> dict` β€” returns `payload` + `signature` field. +- `sign_bytes(data: bytes) -> Signature` + +#### Functions + +- `generate() -> KeyPair` +- `load(keys_dir: Path) -> KeyPair` β€” raises `IdentityError("keys_missing"|"keys_invalid"|"keys_permissions")`. +- `load_or_generate(keys_dir: Path) -> KeyPair` +- `save(kp: KeyPair, keys_dir: Path) -> None` β€” 0600 perms. +- `short_node_id(verify_key_bytes: bytes) -> str` β€” `"ed25519:XXXX-XXXX-XXXX-XXXX"`. +- `full_node_id(verify_key_bytes: bytes) -> str` β€” `"ed25519:"`. +- `parse_node_id(node_id: str) -> bytes` β€” accepts only full form. +- `verify_key_from_full(node_id_full: str) -> VerifyKey` +- `canonical_json(obj: Any) -> bytes` β€” sorted, no whitespace, no trailing zeros, UTF-8. +- `sign_payload(payload: dict, kp: KeyPair) -> dict` +- `verify_payload(payload: dict, vk: VerifyKey) -> bool` +- `verify_payload_with_node_id(payload: dict, expected_node_id_full: str) -> bool` +- `generate_self_signed_cert(kp: KeyPair, host: str = "0.0.0.0") -> tuple[bytes, bytes]` β€” `(cert_pem, key_pem)`, 10-year validity. + +#### Exception + +`IdentityError(Exception)`: +- `code in {"keys_missing","keys_invalid","keys_permissions","bad_node_id","sign_failed","verify_failed","bad_manifest","expired","invalid_signature"}` + +### `hearthnet/identity/manifest.py` β€” Β§3.2 + +#### Dataclasses + +`Endpoint` *(frozen)*: `transport: str`, `host: str`, `port: int`. + +`HardwareSpec` *(frozen)*: `gpu: str | None`, `vram_gb: float`, `ram_gb: float`, `cpu_cores: int`, `disk_free_gb: float`. + +`CapabilitySpec` *(frozen)* β€” subset of `CapabilityDescriptor` for manifest embedding: +- `name: str`, `version: str`, `stability: str`, `schema_hash: str`, `params: dict`, `max_concurrent: int`. + +`NodeManifest` *(frozen)*: +- `version: int`, `contract_version: str`, `node_id: str`, `display_name: str`, `community_id: str`, `profile: str`, `endpoints: list[Endpoint]`, `hardware: HardwareSpec`, `capabilities: list[CapabilitySpec]`, `uptime_seconds: int`, `load: dict`, `issued_at: str`, `expires_at: str`, `signature: str`. +- `as_dict() -> dict` +- `is_expired(now: datetime | None = None) -> bool` + +`CommunityPolicy` *(frozen)*: +- `min_signatures_to_invite: int` +- `min_signatures_to_demote: int` +- `min_signatures_to_revoke: int` +- `capability_token_ttl_seconds: int` +- `federation_enabled: bool` +- `default_member_can_invite: bool` + +`CommunityMember` *(frozen)*: `node_id: str`, `level: TrustLevel`, `added_at: str`, `added_by: str`. + +`RevokedEntry` *(frozen)*: `node_id: str`, `revoked_at: str`. + +`CommunityManifest` *(frozen)*: +- `version: int`, `community_id: str`, `name: str`, `root_key: str`, `created_at: str`, `lamport_at_creation: int`, `policy: CommunityPolicy`, `members: list[CommunityMember]`, `revoked: list[RevokedEntry]`, `head_lamport: int`, `signature: str`. +- `is_member(node_id: str) -> bool` +- `level_of(node_id: str) -> TrustLevel | None` +- `is_revoked(node_id: str) -> bool` + +#### Functions + +- `build_node_manifest(kp: KeyPair, community_id: str, display_name: str, profile: str, endpoints: list[Endpoint], hardware: HardwareSpec, capabilities: list[CapabilitySpec], uptime_seconds: int, load: dict) -> NodeManifest` +- `parse_node_manifest(blob: bytes | dict) -> NodeManifest` +- `verify_node_manifest(manifest: NodeManifest, *, now: datetime | None = None) -> None` +- `build_community_manifest(root_kp: KeyPair, name: str, policy: CommunityPolicy) -> CommunityManifest` +- `regenerate_community_manifest_from_state(materialised_state: dict, signing_kp: KeyPair) -> CommunityManifest` +- `parse_community_manifest(blob: bytes | dict) -> CommunityManifest` +- `verify_community_manifest(cm: CommunityManifest) -> None` +- `load_or_regenerate(state_dir: Path, signing_kp: KeyPair | None = None) -> CommunityManifest` β€” convenience used by `node.py`. + +### `hearthnet/identity/tokens.py` β€” Β§3.3 *(Phase 2; stub in MVP)* + +`CapabilityToken` *(frozen)*: `issuer: str`, `subject: str`, `capability: str`, `issued_at: str`, `expires_at: str`, `nonce: str`, `signature: str`. + +Functions (stubs): +- `issue_token(issuer_kp: KeyPair, subject_node_id: str, capability: str, ttl_seconds: int = 86400) -> CapabilityToken` +- `verify_token(token: CapabilityToken, expected_issuer: str) -> None` + +--- + +## 6. M02 β€” Discovery + +**Spec:** [`modules/M02-discovery.md`](modules/M02-discovery.md) Β· **Path:** `hearthnet/discovery/` + +### `hearthnet/discovery/peers.py` β€” Β§3.1 + +`PeerRecord` *(dataclass)*: +- `node_id: str` (short), `node_id_full: str`, `display_name: str`, `community_id: str`, `profile: Profile`, `endpoints: list[Endpoint]`, `manifest: NodeManifest | None`, `last_seen: float`, `rtt_ms: float | None`, `source: str` (`"mdns"|"udp"|"relay"`). + +`PeerEvent` *(frozen)*: `kind: str` (`"added"|"removed"|"updated"`), `peer: PeerRecord`. + +`PeerRegistry`: +- `__init__(our_node_id_full: str, community_id: str)` +- `upsert(record: PeerRecord) -> bool` β€” True if new. +- `remove(node_id_full: str) -> bool` +- `get(node_id_full: str) -> PeerRecord | None` +- `all() -> list[PeerRecord]` +- `for_community(community_id: str) -> list[PeerRecord]` +- `prune_stale(max_age_seconds: int = 90) -> int` +- `subscribe() -> AsyncIterator[PeerEvent]` +- `set_pruning_aggressive(enabled: bool) -> None` β€” toggled by M09; uses 30s when on, 90s when off. + +### `hearthnet/discovery/mdns.py` β€” Β§3.2 + +`MdnsAnnouncer`: +- `__init__(kp: KeyPair, node_id_short: str, display_name: str, community_id_short: str, profile: Profile, port: int, capabilities_names: list[str], manifest_url: str)` +- `start() -> None` *(async)* +- `stop() -> None` *(async)* +- `update(*, capabilities_names: list[str] | None = None) -> None` + +`MdnsBrowser`: +- `__init__(registry: PeerRegistry, our_community_id: str)` +- `start() -> None` *(async)* +- `stop() -> None` *(async)* + +### `hearthnet/discovery/udp.py` β€” Β§3.4 + +`UdpAnnouncer`: +- `__init__(kp: KeyPair, registry: PeerRegistry, node_id_short: str, community_id_short: str, port: int, capabilities_names: list[str], multicast_group: str = "239.255.42.42", multicast_port: int = 42424)` +- `run() -> None` *(async)* + +`UdpListener`: +- `__init__(registry: PeerRegistry, our_community_id: str)` +- `run() -> None` *(async)* + +### `hearthnet/discovery/relay.py` *(Phase 2 stub)* + +`InternetRelayClient` β€” not implemented in MVP. Reserved. + +#### Exception + +`DiscoveryError(Exception)`: +- `code in {"socket_in_use","mdns_unavailable","manifest_fetch_failed","manifest_invalid"}` + +--- + +## 7. M03 β€” Capability Bus + +**Spec:** [`modules/M03-bus.md`](modules/M03-bus.md) Β· **Path:** `hearthnet/bus/` + +### `hearthnet/bus/capability.py` β€” Β§3.1 + +`CapabilityDescriptor` *(frozen dataclass)*: +- `name: CapabilityName`, `version: Version`, `stability: Stability`, `request_schema: dict`, `response_schema: dict | None`, `stream_schema: dict | None`, `params: dict`, `max_concurrent: int`, `trust_required: str` (`"member"|"trusted"|"anchor"|"self"`), `timeout_seconds: int`, `idempotent: bool`. +- `version_str -> str` *(property)* +- `schema_hash() -> str` β€” BLAKE3 of canonical-JSON of `{name, version, request_schema, response_schema, stream_schema}`. + +`CapabilityEntry` *(dataclass)*: +- `node_id: str`, `descriptor: CapabilityDescriptor`, `is_local: bool`, `handler: Callable | None`, `endpoint: Endpoint | None`, `in_flight: int`, `last_seen: float`, `p50_latency_ms: float`, `p99_latency_ms: float`, `success_rate: float`, `quarantined_until: float`, `sticky_sessions: set[str]`. + +`RouteRequest` *(frozen dataclass)*: +- `capability: CapabilityName`, `version_req: Version`, `body: dict`, `caller: str`, `trace_id: str`, `session_id: str | None`, `deadline_ms: int`, `stream: bool`. + +`ParamsPredicate` β€” type alias: `Callable[[dict, dict], bool]`. + +### `hearthnet/bus/registry.py` β€” Β§3.2 + +`Diff` *(frozen dataclass)*: `added`, `removed`, `updated` β€” each `list[CapabilityEntry]`. + +`RegistryEvent` *(frozen dataclass)*: `kind: str` (`"added"|"removed"|"updated"`), `entry: CapabilityEntry`. + +`Registry`: +- `__init__(our_node_id: str)` +- `register_local(descriptor: CapabilityDescriptor, handler: Callable, params_compatible: ParamsPredicate | None = None) -> None` +- `deregister_local(name: CapabilityName, version: Version) -> None` +- `update_from_peer_manifest(peer: PeerRecord, manifest: NodeManifest) -> Diff` +- `remove_peer(node_id: str) -> int` +- `find(name: CapabilityName, version_req: Version, params_filter: Callable[[dict], bool] | None = None) -> list[CapabilityEntry]` +- `entry(node_id: str, name: CapabilityName, version: Version) -> CapabilityEntry | None` +- `all_local() -> list[CapabilityEntry]` +- `all() -> list[CapabilityEntry]` +- `subscribe() -> AsyncIterator[RegistryEvent]` + +### `hearthnet/bus/health.py` β€” Β§3.3 + +`HealthTracker`: +- `__init__(window: int = HEALTH_WINDOW_CALLS)` +- `record(entry: CapabilityEntry, *, success: bool, latency_ms: float) -> None` +- `is_quarantined(entry: CapabilityEntry) -> bool` +- `reset(entry: CapabilityEntry) -> None` + +### `hearthnet/bus/schema.py` β€” Β§3.4 + +`SchemaValidator`: +- `__init__()` +- `validate_request(descriptor: CapabilityDescriptor, body: dict) -> None` +- `validate_response(descriptor: CapabilityDescriptor, body: dict) -> None` +- `validate_stream_frame(descriptor: CapabilityDescriptor, frame: dict) -> None` + +Free function: +- `compute_schema_hash(descriptor_partial: dict) -> str` β€” `"blake3:"`. See [CONTRACT Β§11](CAPABILITY_CONTRACT.md). + +### `hearthnet/bus/router.py` β€” Β§3.5 + +`Router`: +- `__init__(registry: Registry, config: BusConfig, our_node_id: str)` +- `route(req: RouteRequest) -> CapabilityEntry | None` β€” scoring algorithm; see Β§5.4. +- `route_sticky(req: RouteRequest) -> CapabilityEntry | None` +- `release_session(session_id: str) -> None` + +### `hearthnet/bus/trace.py` β€” Β§3.6 + +`CallTraceEvent` *(frozen dataclass)*: +- `ts: str`, `trace_id: str`, `capability: CapabilityName`, `version: str`, `from_node: str`, `to_node: str`, `is_local: bool`, `result: str`, `ms: float`, `tokens_in: int | None`, `tokens_out: int | None`, `bytes_in: int`, `bytes_out: int`. + +`TraceHook`: +- `__init__()` +- `on_call_start(req: RouteRequest, entry: CapabilityEntry) -> None` +- `on_call_end(req: RouteRequest, entry: CapabilityEntry, *, result: str, latency_ms: float, bytes_in: int, bytes_out: int, tokens_in: int | None = None, tokens_out: int | None = None) -> None` + +### `hearthnet/bus/__init__.py` β€” Β§3.7 + +`TopologySnapshot` *(frozen dataclass)*: +- `our_node_id: str`, `peers: list[PeerRecord]`, `capabilities_local: list[CapabilityEntry]`, `capabilities_remote: list[CapabilityEntry]`, `in_flight_total: int`. + +`CapabilityBus`: +- `__init__(node_id_full: str, community_id: str, config: BusConfig, transport_client: HttpClient, community_manifest_provider: Callable[[], CommunityManifest])` +- attributes: `registry`, `health`, `schema`, `router`, `trace` +- `register_service(service: Service) -> None` +- `register_capability(descriptor: CapabilityDescriptor, handler: Callable, params_compatible: ParamsPredicate | None = None) -> None` +- `handle_call(req: RouteRequest) -> dict | AsyncIterator[dict]` *(async)* +- `call(capability: CapabilityName, version_req: Version, body: dict, *, session_id: str | None = None, timeout_seconds: float | None = None) -> dict` *(async)* +- `stream(capability: CapabilityName, version_req: Version, body: dict, *, session_id: str | None = None) -> AsyncIterator[Frame]` +- `on_peer_added(peer: PeerRecord) -> None` +- `on_peer_updated(peer: PeerRecord) -> None` +- `on_peer_removed(node_id: str) -> None` +- `topology_snapshot() -> TopologySnapshot` +- `recent_traces(n: int = 50) -> list[CallTraceEvent]` +- `stats() -> dict` + +`BusError(Exception)`: +- `code in {"schema_invalid","namespace_violation","schema_mismatch","not_found","capacity_exceeded","quarantined","partition","timeout","internal_error"}` + +### `hearthnet/services/base.py` β€” M03 Β§4 + +`Service` *(Protocol)*: +- `name: str` +- `version: str` +- `capabilities() -> list[tuple[CapabilityDescriptor, Callable, ParamsPredicate]]` +- `start() -> None` *(async)* +- `stop() -> None` *(async)* +- `health() -> dict` + +--- + +## 8. M11 β€” Embedding Service + +**Spec:** [`modules/M11-embedding.md`](modules/M11-embedding.md) Β· **Path:** `hearthnet/services/embedding/` + +### `hearthnet/services/embedding/backends.py` β€” Β§3.1 + +`EmbeddingBackend` *(Protocol)*: +- attrs: `name: str`, `model: str`, `dim: int`, `max_input: int` +- `embed(texts: list[str], *, normalize: bool = True) -> list[list[float]]` *(async)* +- `warm() -> None` *(async)* +- `close() -> None` *(async)* +- `health() -> dict` + +`SentenceTransformerBackend`: +- `__init__(model: str, device: str = "auto")` β€” `device` ∈ `{"auto","cpu","cuda"}`. +- all `EmbeddingBackend` methods. + +### `hearthnet/services/embedding/service.py` β€” Β§3.2 + +`EmbeddingService` *(implements `Service`)*: +- `name = "embedding"`, `version = "1.0"` +- `__init__(config: EmbeddingConfig)` +- `capabilities() -> [...]` β€” registers `embed.text@1.0`. +- `start()`, `stop()`, `health()` *(async)* +- `handle_embed_text(req: RouteRequest) -> dict` *(async)* β€” implements [CONTRACT Β§4.3](CAPABILITY_CONTRACT.md). + +#### Capability params predicate + +```python +def params_compatible(offered: dict, requested: dict) -> bool: + return requested.get("model") == offered.get("model") +``` + +--- + +## 9. M04 β€” LLM Service + +**Spec:** [`modules/M04-llm.md`](modules/M04-llm.md) Β· **Path:** `hearthnet/services/llm/` + +### `hearthnet/services/llm/backends/base.py` β€” Β§3.1 + +`Token` *(frozen dataclass)*: `text: str`, `logprob: float | None`, `stop: bool`. + +`ChatResult` *(frozen dataclass)*: `text: str`, `tokens_in: int`, `tokens_out: int`, `stop_reason: str`, `ms: int`. + +`BackendModel` *(frozen dataclass)*: `name: str`, `quant: str`, `ctx_max: int`, `modalities: list[str]`, `requires_internet: bool`. + +`LlmBackend` *(Protocol)*: +- attrs: `name: str`, `models: list[BackendModel]` +- `warm(model: str) -> None` *(async)* +- `close() -> None` *(async)* +- `chat(*, model: str, messages: list[dict], max_tokens: int = 1024, temperature: float = 0.7, top_p: float = 0.95, stop: list[str] | None = None, seed: int | None = None, stream: bool = True) -> AsyncIterator[Token]` +- `complete(*, model: str, prompt: str, max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.95, stop: list[str] | None = None, seed: int | None = None, stream: bool = True) -> AsyncIterator[Token]` +- `count_tokens(model: str, text: str) -> int` +- `max_concurrent(model: str) -> int` +- `health() -> dict` + +### Concrete backends β€” Β§3.2 + +Each implements `LlmBackend`. Same method set; only constructor varies. + +| File | Class | Constructor signature | +|------|-------|-----------------------| +| `backends/llama_cpp.py` | `LlamaCppBackend` | `__init__(model_path: Path, model_meta: BackendModel, gpu_layers: int = -1)` | +| `backends/ollama.py` | `OllamaBackend` | `__init__(base_url: str = "http://localhost:11434", models: list[str] | None = None)` | +| `backends/lmstudio.py` | `LmStudioBackend` | `__init__(base_url: str, default_model: str)` β€” OpenAI-compatible HTTP | +| `backends/hf_api.py` | `HfApiBackend` | `__init__(model: str, token_env: str = "HF_TOKEN")` β€” `requires_internet=True` | +| `backends/anthropic_api.py` | `AnthropicApiBackend` | `__init__(model: str = "claude-sonnet-4-6", token_env: str = "ANTHROPIC_API_KEY")` β€” `requires_internet=True` | +| `backends/nemotron.py` *(new)* | `NemotronBackend` | `__init__(base_url: str = "https://integrate.api.nvidia.com/v1", model: str = "nvidia/llama-3.1-nemotron-70b-instruct", token_env: str = "NVIDIA_API_KEY", local: bool = False)` β€” OpenAI-compatible; `requires_internet=True` unless `local=True` (locally-hosted NIM endpoint) | +| `backends/openbmb.py` *(new)* | `OpenBmbBackend` | `__init__(base_url: str = "http://localhost:8000", model: str = "openbmb/MiniCPM4-8B", token_env: str | None = None)` β€” OpenAI-compatible HTTP (vLLM / llama.cpp serve / SGLang); `requires_internet=False`. Designed around Christof's MiniCPM workbench | + +All backends declare their `models: list[BackendModel]` so the service can enumerate `(backend, model)` pairs at registration time. + +### `hearthnet/services/llm/tokenizers.py` β€” Β§3.3 + +- `count_tokens_approx(model_family: str, text: str) -> int` +- `model_family(model_name: str) -> str` β€” e.g. `"qwen2.5-7b-instruct"` β†’ `"qwen"`, `"nemotron-70b"` β†’ `"nemotron"`, `"MiniCPM4-8B"` β†’ `"minicpm"`. + +### `hearthnet/services/llm/service.py` β€” Β§3.4 + +`LlmService` *(implements `Service`)*: +- `name = "llm"`, `version = "1.0"` +- `__init__(config: LlmConfig)` +- `_build_backends(config: LlmConfig) -> list[LlmBackend]` +- `capabilities() -> [...]` β€” emits one descriptor per `(backend, model)` Γ— `{llm.chat, llm.complete}`. +- `start(), stop(), health()` *(async)* +- `handle_chat(req: RouteRequest) -> AsyncIterator[dict]` *(async)* β€” implements [CONTRACT Β§4.1](CAPABILITY_CONTRACT.md). +- `handle_complete(req: RouteRequest) -> AsyncIterator[dict]` *(async)* β€” implements [CONTRACT Β§4.2](CAPABILITY_CONTRACT.md). + +#### Capability params predicate β€” Β§3.6 + +```python +def params_compatible(offered: dict, requested: dict) -> bool: + if requested.get("model") != offered.get("model"): + return False + if "ctx" in requested and requested["ctx"] > offered["ctx"]: + return False + return True +``` + +--- + +## 10. M05 β€” RAG Service + +**Spec:** [`modules/M05-rag.md`](modules/M05-rag.md) Β· **Path:** `hearthnet/services/rag/` + +### `hearthnet/services/rag/chunker.py` β€” Β§3.1 + +`Chunk` *(frozen dataclass)*: `text: str`, `metadata: dict`. + +Functions: +- `chunk_text(text: str, *, tokens_per_chunk: int = RAG_CHUNK_TOKENS, overlap_tokens: int = RAG_CHUNK_OVERLAP_TOKENS, metadata: dict | None = None) -> list[Chunk]` +- `chunk_pdf(pdf_bytes: bytes, *, doc_metadata: dict) -> list[Chunk]` + +### `hearthnet/services/rag/store.py` β€” Β§3.2 + +`ScoredChunk` *(frozen dataclass)*: `chunk: Chunk`, `score: float`. + +`CorpusStore`: +- `__init__(corpora_dir: Path, corpus: str, embedding_dim: int)` +- `add_chunks(chunks: list[Chunk], embeddings: list[list[float]]) -> None` +- `has_document(doc_cid: str) -> bool` +- `query(embedding: list[float], *, k: int, filter: dict | None = None) -> list[ScoredChunk]` +- `count() -> int` +- `size_bytes() -> int` +- `language_majority() -> str | None` + +Free functions: +- `list_corpora(corpora_dir: Path) -> list[str]` +- `corpus_info(corpora_dir: Path, corpus: str) -> dict` + +### `hearthnet/services/rag/ingest.py` β€” Β§3.3 + +`IngestResult` *(frozen dataclass)*: `doc_cid: str`, `chunks_indexed: int`, `tokens_indexed: int`, `ingest_event_id: str`, `ms: int`. + +`IngestPipeline`: +- `__init__(bus: CapabilityBus, blob_store: BlobStore, corpora_dir: Path, event_log: EventLog)` +- `ingest_document(doc_cid: str, corpus: str, title: str, language: str, metadata: dict, author_kp: KeyPair) -> IngestResult` *(async)* + +### `hearthnet/services/rag/service.py` β€” Β§3.4 + +`RagService` *(implements `Service`)*: +- `name = "rag"`, `version = "1.0"` +- `__init__(config: RagConfig, bus: CapabilityBus, blob_store: BlobStore, event_log: EventLog, community_manifest_provider: Callable[[], CommunityManifest])` +- `capabilities() -> [...]` β€” `rag.query@1.0` per corpus, `rag.ingest@1.0` once, `rag.list_corpora@1.0` once. +- `start(), stop(), health()` *(async)* +- `handle_query(req: RouteRequest) -> dict` *(async)* β€” [CONTRACT Β§4.4](CAPABILITY_CONTRACT.md). +- `handle_ingest(req: RouteRequest) -> dict` *(async)* β€” [CONTRACT Β§4.5](CAPABILITY_CONTRACT.md). +- `handle_list_corpora(req: RouteRequest) -> dict` *(async)* β€” [CONTRACT Β§4.6](CAPABILITY_CONTRACT.md). + +#### Capability params predicate β€” Β§3.5 + +```python +def query_params_compatible(offered: dict, requested: dict) -> bool: + return requested.get("corpus") == offered.get("corpus") +``` + +--- + +## 11. M07 β€” File & Blobs + +**Spec:** [`modules/M07-file-blobs.md`](modules/M07-file-blobs.md) Β· **Paths:** `hearthnet/blobs/` + `hearthnet/services/file/` + +### `hearthnet/blobs/chunker.py` β€” Β§3.1 + +`ChunkRef` *(frozen)*: `index: int`, `cid: str`, `size_bytes: int`. + +`BlobManifest` *(frozen)*: `cid: str`, `size_bytes: int`, `chunk_size_bytes: int`, `chunks: list[ChunkRef]`, `mime_type: str | None`, `filename: str | None`. + +Functions: +- `hash_bytes(data: bytes) -> str` β€” `"blake3:"`. +- `chunk_blob(data: bytes, *, chunk_size: int = CHUNK_SIZE_BYTES) -> tuple[BlobManifest, list[bytes]]` +- `manifest_cid(manifest: BlobManifest) -> str` +- `reassemble(chunks: list[bytes]) -> bytes` +- `verify_chunk(data: bytes, expected_cid: str) -> None` β€” raises `BlobError("hash_mismatch")`. + +### `hearthnet/blobs/store.py` β€” Β§3.2 + +`BlobStore`: +- `__init__(dir_path: Path, gc_threshold: float = BLOB_GC_DISK_THRESHOLD)` +- `has(cid: str) -> bool` +- `read_chunk(cid: str) -> bytes` +- `write_chunk(cid: str, data: bytes) -> None` +- `delete_chunk(cid: str) -> bool` +- `has_blob(manifest_cid: str) -> bool` +- `read_manifest(manifest_cid: str) -> BlobManifest` +- `write_blob(manifest: BlobManifest, chunks: list[bytes]) -> None` +- `read_blob_bytes(manifest_cid: str) -> bytes` +- `read_blob_stream(manifest_cid: str) -> AsyncIterator[tuple[ChunkRef, bytes]]` *(async)* +- `list_cids(prefix: str | None = None) -> list[str]` +- `total_bytes() -> int` +- `pin(cid: str) -> None` +- `unpin(cid: str) -> None` +- `is_pinned(cid: str) -> bool` +- `gc(target_fraction: float = 0.7) -> int` β€” bytes freed. + +`BlobError(Exception)`: +- `code in {"not_found","hash_mismatch","io_error","disk_full","manifest_invalid"}` + +### `hearthnet/blobs/transfer.py` β€” Β§3.3 + +`TransferManager`: +- `__init__(store: BlobStore, bus: CapabilityBus, concurrency: int = 4)` +- `fetch_blob(manifest_cid: str, *, sources: list[str] | None = None) -> BlobManifest` *(async)* +- `advertise(cids: list[str]) -> None` *(async)* + +### `hearthnet/services/file/service.py` β€” Β§4.1 + +`FileService` *(implements `Service`)*: +- `name = "file"`, `version = "1.0"` +- `__init__(config: FileConfig, store: BlobStore, event_log: EventLog)` +- `capabilities() -> [...]` β€” `file.read`, `file.list`, `file.advertise`, `file.put` (all `@1.0`). +- `start(), stop(), health()` *(async)* +- `handle_read(req: RouteRequest) -> AsyncIterator[dict] | dict` *(async)* β€” [CONTRACT Β§4.7](CAPABILITY_CONTRACT.md). +- `handle_list(req: RouteRequest) -> dict` *(async)* β€” [CONTRACT Β§4.8](CAPABILITY_CONTRACT.md). +- `handle_advertise(req: RouteRequest) -> dict` *(async)* β€” [CONTRACT Β§4.9](CAPABILITY_CONTRACT.md). +- `handle_put(req: RouteRequest) -> AsyncIterator[dict]` *(async)* β€” [CONTRACT Β§4.10](CAPABILITY_CONTRACT.md). + +All four `file.*` use default `lambda offered, requested: True` as params predicate. + +--- + +## 12. M06 β€” Marketplace Service + +**Spec:** [`modules/M06-marketplace.md`](modules/M06-marketplace.md) Β· **Path:** `hearthnet/services/marketplace/` + +### `hearthnet/services/marketplace/post.py` β€” Β§3.1 + +`Location` *(frozen dataclass)*: `lat: float`, `lng: float`, `label: str`. + +`Post` *(frozen dataclass)*: +- `event_id: str`, `lamport: int`, `author: str`, `category: Category`, `title: str`, `body: str`, `location: Location | None`, `tags: list[str]`, `created_at: str`, `expires_at: str`, `expired_via_event_id: str | None`, `expiry_reason: str | None`. +- `is_expired(now: datetime | None = None) -> bool` + +### `hearthnet/services/marketplace/views.py` β€” Β§3.2 + +`MarketplaceView` *(implements `MaterialisedView` from X02)*: +- `__init__()` +- `reset() -> None` +- `apply(event: Event) -> None` +- `snapshot_state() -> dict` +- `restore_state(state: dict) -> None` +- `list(*, category: Category | None = None, tags: list[str] | None = None, since_lamport: int = 0, limit: int = 50) -> list[Post]` +- `get(event_id: str) -> Post | None` +- `max_lamport() -> int` +- `all_active() -> list[Post]` + +### `hearthnet/services/marketplace/service.py` β€” Β§3.3 + +`MarketplaceService` *(implements `Service`)*: +- `name = "marketplace"`, `version = "1.0"` +- `__init__(config: MarketConfig, bus: CapabilityBus, event_log: EventLog, replay_engine: ReplayEngine, author_kp: KeyPair, community_manifest_provider: Callable[[], CommunityManifest])` +- `capabilities() -> [...]` β€” `market.list`, `market.post`, `market.expire`, `market.search` (all `@1.0`). +- `start(), stop(), health()` *(async)* β€” start replays events and installs auto-expiry sweeper. +- `handle_list(req) -> dict` *(async)* β€” [CONTRACT Β§4.11](CAPABILITY_CONTRACT.md). +- `handle_post(req) -> dict` *(async)* β€” [CONTRACT Β§4.12](CAPABILITY_CONTRACT.md). +- `handle_expire(req) -> dict` *(async)* β€” [CONTRACT Β§4.13](CAPABILITY_CONTRACT.md). +- `handle_search(req) -> dict` *(async)* β€” [CONTRACT Β§4.14](CAPABILITY_CONTRACT.md). +- `_auto_expire_sweep() -> None` *(async)* β€” internal background task. + +All four use default `lambda offered, requested: True` predicate. + +--- + +## 13. M10 β€” Chat Service + +**Spec:** [`modules/M10-chat.md`](modules/M10-chat.md) Β· **Path:** `hearthnet/services/chat/` + +### `hearthnet/services/chat/views.py` β€” Β§3.1 + +`ChatMessage` *(frozen dataclass)*: +- `event_id: str`, `lamport: int`, `sender: str`, `recipient: str`, `body: str`, `attachments: list[dict]`, `created_at: str`, `delivered_at: str | None`, `read_at: str | None`. + +`ChatView` *(implements `MaterialisedView`)*: +- `__init__(our_node_id_full: str)` +- `reset(), apply(event), snapshot_state(), restore_state(state)` +- `history_with(peer: str | None = None, *, since_lamport: int = 0, limit: int = 200) -> list[ChatMessage]` +- `peers() -> list[str]` +- `unread_count(peer: str) -> int` + +### `hearthnet/services/chat/delivery.py` β€” Β§3.2 + +`DeliveryManager`: +- `__init__(bus: CapabilityBus, event_log: EventLog, author_kp: KeyPair, peer_registry: PeerRegistry, config: ChatConfig)` +- `deliver(message_event: Event) -> str` *(async)* β€” returns `"direct"|"forwarded"|"queued"`. +- `on_local_message_arrived(message_event: Event) -> None` *(async)* +- `on_pubsub_message(payload: dict) -> None` *(async)* + +### `hearthnet/services/chat/service.py` β€” Β§3.3 + +`ChatService` *(implements `Service`)*: +- `name = "chat"`, `version = "1.0"` +- `__init__(config: ChatConfig, bus: CapabilityBus, event_log: EventLog, replay_engine: ReplayEngine, peer_registry: PeerRegistry, author_kp: KeyPair, our_node_id_full: str)` +- `capabilities() -> [...]` β€” `chat.send@1.0` (member trust), `chat.history@1.0` (self trust). +- `start(), stop(), health()` *(async)* +- `handle_send(req) -> dict` *(async)* β€” [CONTRACT Β§4.15](CAPABILITY_CONTRACT.md). +- `handle_history(req) -> dict` *(async)* β€” [CONTRACT Β§4.16](CAPABILITY_CONTRACT.md). Enforces `caller == our_node_id_full`. + +--- + +## 14. M09 β€” Emergency Mode Detector + +**Spec:** [`modules/M09-emergency.md`](modules/M09-emergency.md) Β· **Path:** `hearthnet/emergency/` + +### `hearthnet/emergency/state.py` β€” Β§3.1 + +`EmergencyState` *(frozen dataclass)*: +- `mode: Mode`, `since: WallClock`, `last_probe: WallClock`, `probe_results: dict[str, bool]`. + +`StateBus`: +- `__init__()` +- `current() -> EmergencyState` +- `subscribe() -> AsyncIterator[EmergencyState]` +- `_emit(state: EmergencyState) -> None` *(internal)* + +### `hearthnet/emergency/detector.py` β€” Β§3.2 + +`Detector`: +- `__init__(config: EmergencyConfig, bus: CapabilityBus, state_bus: StateBus)` +- `run() -> None` *(async)* +- `shutdown() -> None` *(async)* +- `_probe_dns(host: str) -> bool` *(async, internal)* +- `_probe_http(url: str) -> bool` *(async, internal)* + +State-transition effects (Β§5.2): +- entering offline β†’ deregister local capabilities whose descriptor `params.requires_internet == True` +- entering online β†’ re-register those backends +- offline ↔ online β†’ flip `peer_registry.set_pruning_aggressive(...)` (M02) + +--- + +## 15. M08 β€” UI + +**Spec:** [`modules/M08-ui.md`](modules/M08-ui.md) Β· **Path:** `hearthnet/ui/` + +### `hearthnet/ui/app.py` β€” Β§3.1 + +`UiApp`: +- `__init__(bus: CapabilityBus, state_bus: StateBus, config: UiConfig, node_id_short: str, community_name: str)` +- `build() -> gr.Blocks` +- `launch_async() -> None` *(async)* +- `shutdown() -> None` *(async)* + +Free function: +- `build_ui(bus: CapabilityBus, state_bus: StateBus, config: UiConfig, **meta) -> UiApp` + +### `hearthnet/ui/topology.py` β€” Β§3.2 + +`TopologyComponent`: +- `__init__(bus: CapabilityBus)` +- `render() -> gr.HTML` +- `push_trace(event: CallTraceEvent) -> None` +- `push_topology(snapshot: TopologySnapshot) -> None` + +### `hearthnet/ui/theme.py` β€” Β§7 + +- `hearthnet_theme: gr.Theme` *(module-level constant)* +- `emergency_theme: gr.Theme` *(module-level constant)* +- CSS variables documented in spec Β§7 + +### `hearthnet/ui/tabs/` + +Each file exports a builder function returning a `gr.Tab` or `gr.Blocks` fragment. + +| File | Function | Spec | +|------|----------|------| +| `tabs/ask.py` | `build_ask_tab(bus: CapabilityBus) -> gr.Tab` | Β§5.1 | +| `tabs/chat.py` | `build_chat_tab(bus: CapabilityBus, our_node_id_full: str) -> gr.Tab` | Β§5.3 | +| `tabs/marketplace.py` | `build_marketplace_tab(bus: CapabilityBus) -> gr.Tab` | Β§5.4 | +| `tabs/files.py` | `build_files_tab(bus: CapabilityBus) -> gr.Tab` | Β§5.5 | +| `tabs/emergency.py` | `build_emergency_tab(bus: CapabilityBus, state_bus: StateBus) -> gr.Tab` | Β§5.6 | +| `tabs/settings.py` | `build_settings_tab(bus: CapabilityBus, config: Config) -> gr.Tab` | Β§5.2 | + +### `hearthnet/ui/mobile/` β€” Β§6 + +Static assets served at `/mobile/*` by [X01](cross-cutting/X01-transport.md): +- `index.html` β€” single-page app +- `app.js` β€” same bus API; uses signed requests via WebCrypto +- `style.css` + +--- + +## 16. M13 β€” Onboarding + +**Spec:** [`modules/M13-onboarding.md`](modules/M13-onboarding.md) Β· **Path:** `hearthnet/ui/onboarding.py` + +### `hearthnet/ui/onboarding.py` β€” Β§3.1 + +`InviteBlob` *(frozen dataclass)*: +- `schema_version: int`, `community_id: str`, `community_name: str`, `inviter_node_id: str`, `invitee_node_id: str`, `initial_level: str`, `bootstrap_endpoints: list[Endpoint]`, `expires_at: str`, `signature: str`. + +#### Functions + +- `encode_invite(blob: InviteBlob) -> str` β€” `"hearthnet://v1/"`. +- `decode_invite(text: str) -> InviteBlob` +- `invite_to_qr_png(blob: InviteBlob, *, box_size: int = 8) -> bytes` +- `create_community(name: str, policy: CommunityPolicy, kp: KeyPair, state_dir: Path, event_log: EventLog) -> CommunityManifest` +- `make_invite(invitee_node_id_full: str, inviter_kp: KeyPair, community_manifest: CommunityManifest, bootstrap_endpoints: list[Endpoint], initial_level: str = "member", ttl_seconds: int = 86400) -> InviteBlob` +- `redeem_invite(blob: InviteBlob, our_kp: KeyPair, transport_client: HttpClient, event_log: EventLog) -> CommunityManifest` *(async)* +- `build_onboarding(config: Config, kp_provider: Callable[[], KeyPair]) -> gr.Blocks` + +Exception: +`OnboardingError(Exception)`: +- `code in {"invite_invalid","invite_expired","invitee_mismatch","bootstrap_unreachable","community_manifest_invalid","sync_failed","already_member"}` + +--- + +## 17. M12 β€” CLI & Orchestrator + +**Spec:** [`modules/M12-cli.md`](modules/M12-cli.md) Β· **Paths:** `hearthnet/cli.py` + `hearthnet/node.py` + +### `hearthnet/cli.py` β€” Β§3 + +Click group + subcommands. Each is a top-level function decorated with `@main.command()`. + +| Command | Function | Spec | +|---------|----------|------| +| (root) | `main(ctx, config)` | Β§4 | +| `init` | `init(name: str, profile: str, non_interactive: bool)` | Β§3.1 | +| `run` | `run(config: str, no_ui: bool, debug: bool)` | Β§3.2 | +| `status` | `status(json_output: bool)` | Β§3.3 | +| `caps` | `caps(remote_only: bool, local_only: bool, name: str)` | Β§3.4 | +| `call` | `call(name_at_version: str, body: str, stream: bool)` | Β§3.5 | +| `log` | `log(follow: bool, level: str, component: str)` | Β§3.6 | +| `trace` | `trace_recent(n: int, capability: str)` | Β§3.7 | +| `doctor` | `doctor(check: str)` | Β§3.8 | +| `export` | `export(out: str)` | Β§3.9 | +| `erase` | `erase(keep_keys: bool, yes: bool)` | Β§3.10 | +| `rag list` | `rag_list()` | Β§3.11 | +| `rag ingest` | `rag_ingest(path: str, corpus: str)` | Β§3.11 | +| `rag reindex` | `rag_reindex(corpus: str, embedding_model: str)` | Β§3.11 | +| `invite create` | `invite_create(node_id: str, level: str, ttl: int)` | Β§3.12 | +| `invite redeem` | `invite_redeem(text_or_path: str)` | Β§3.12 | +| `version` | `version_cmd()` | Β§3.13 | + +Exit codes β€” Β§6: `0` success, `1` generic error, `2` user abort / bad usage, `3` no running node, `4` auth, `5` capacity. + +### `hearthnet/node.py` β€” Β§5 + +Single function β€” the canonical wiring: + +```python +async def start(config: Config) -> None: + """The 15-step composition. Do not deviate.""" +``` + +Sequence (each numbered in spec Β§5): +1. observability configure +2. identity load_or_generate +3. community check / onboarding redirect +4. event log + snapshot store + replay engine + community manifest +5. blob store +6. pinned-certs + transport client + bus +7. peer registry + mdns/udp announcer + listener +8. instantiate services (Embedding, Llm, Rag, File, Marketplace, Chat) and register with bus +9. state bus + Detector +10. http server +11. UI app +12. wire peer events β†’ bus +13. ManifestPublisher +14. SyncClient periodic loop +15. asyncio.gather(...) β€” block until shutdown + +Auxiliary class declared inline in this module: + +`ManifestPublisher`: +- `__init__(kp: KeyPair, community_manifest_provider: Callable, bus: CapabilityBus, peer_registry: PeerRegistry, interval_seconds: int = MANIFEST_REPUBLISH_INTERVAL_SECONDS)` +- `run() -> None` *(async)* +- Publishes the freshly-built node manifest to mDNS + UDP every `interval_seconds`. Triggered also on `bus.registry` change events (capability added/removed). + +`PeriodicTask` *(helper)*: +- `__init__(fn: Callable[[], Awaitable], interval_seconds: int)` +- `run() -> None` *(async)* + +### `hearthnet/__main__.py` + +Single line: `from hearthnet.cli import main; main()` + +--- + +## 18. Cross-module symbol index (alphabetical) + +For "where is `X` declared?" + +| Symbol | Module | File | +|--------|--------|------| +| `AnthropicApiBackend` | M04 | `services/llm/backends/anthropic_api.py` | +| `BackendModel` | M04 | `services/llm/backends/base.py` | +| `BlobError` | M07 | `blobs/store.py` | +| `BlobManifest` | M07 | `blobs/chunker.py` | +| `BlobStore` | M07 | `blobs/store.py` | +| `BusConfig` | X04 | `config.py` | +| `BusError` | M03 | `bus/__init__.py` | +| `CallError` | X01 | `transport/client.py` | +| `CallTraceEvent` | M03 | `bus/trace.py` | +| `CapabilityBus` | M03 | `bus/__init__.py` | +| `CapabilityDescriptor` | M03 | `bus/capability.py` | +| `CapabilityEntry` | M03 | `bus/capability.py` | +| `CapabilitySpec` | M01 | `identity/manifest.py` | +| `CapabilityToken` | M01 | `identity/tokens.py` | +| `Category` | M06 | `services/marketplace/post.py` (Literal alias) | +| `ChatConfig` | X04 | `config.py` | +| `ChatMessage` | M10 | `services/chat/views.py` | +| `ChatService` | M10 | `services/chat/service.py` | +| `ChatView` | M10 | `services/chat/views.py` | +| `CheckResult` | X03 | `observability/doctor.py` | +| `Chunk` | M05 | `services/rag/chunker.py` | +| `ChunkRef` | M07 | `blobs/chunker.py` | +| `CommunityConfig` | X04 | `config.py` | +| `CommunityManifest` | M01 | `identity/manifest.py` | +| `CommunityMember` | M01 | `identity/manifest.py` | +| `CommunityPolicy` | M01 | `identity/manifest.py` | +| `Config` | X04 | `config.py` | +| `ConfigError` | X04 | `config.py` | +| `CorpusStore` | M05 | `services/rag/store.py` | +| `DeliveryManager` | M10 | `services/chat/delivery.py` | +| `Detector` | M09 | `emergency/detector.py` | +| `Diff` | M03 | `bus/registry.py` | +| `DiscoveryConfig` | X04 | `config.py` | +| `DiscoveryError` | M02 | `discovery/__init__.py` | +| `EmbeddingBackend` | M11 | `services/embedding/backends.py` | +| `EmbeddingConfig` | X04 | `config.py` | +| `EmbeddingService` | M11 | `services/embedding/service.py` | +| `EmergencyConfig` | X04 | `config.py` | +| `EmergencyState` | M09 | `emergency/state.py` | +| `Endpoint` | M01 | `identity/manifest.py` | +| `Event` | X02 | `events/types.py` | +| `EventLog` | X02 | `events/log.py` | +| `EventLogError` | X02 | `events/log.py` | +| `EventType` | X02 | `events/types.py` | +| `FileConfig` | X04 | `config.py` | +| `FileService` | M07 | `services/file/service.py` | +| `FlowControl` | X01 | `transport/backpressure.py` | +| `Frame` | X01 | `transport/streams.py` | +| `HardwareSpec` | M01 | `identity/manifest.py` | +| `HeadsReport` | X02 | `events/sync.py` | +| `HealthTracker` | M03 | `bus/health.py` | +| `HfApiBackend` | M04 | `services/llm/backends/hf_api.py` | +| `HttpClient` | X01 | `transport/client.py` | +| `HttpServer` | X01 | `transport/server.py` | +| `IdentityConfig` | X04 | `config.py` | +| `IdentityError` | M01 | `identity/keys.py` | +| `IngestPipeline` | M05 | `services/rag/ingest.py` | +| `IngestResult` | M05 | `services/rag/ingest.py` | +| `InviteBlob` | M13 | `ui/onboarding.py` | +| `JsonFormatter` | X03 | `observability/logging.py` | +| `KeyPair` | M01 | `identity/keys.py` | +| `LamportClock` | X02 | `events/lamport.py` | +| `LlamaCppBackend` | M04 | `services/llm/backends/llama_cpp.py` | +| `LlmBackend` | M04 | `services/llm/backends/base.py` | +| `LlmBackendConfig` | X04 | `config.py` | +| `LlmConfig` | X04 | `config.py` | +| `LlmService` | M04 | `services/llm/service.py` | +| `LmStudioBackend` | M04 | `services/llm/backends/lmstudio.py` | +| `Location` | M06 | `services/marketplace/post.py` | +| `ManifestPublisher` | M12 | `node.py` | +| `MarketConfig` | X04 | `config.py` | +| `MarketplaceService` | M06 | `services/marketplace/service.py` | +| `MarketplaceView` | M06 | `services/marketplace/views.py` | +| `MaterialisedView` | X02 | `events/replay.py` (Protocol) | +| `MdnsAnnouncer` | M02 | `discovery/mdns.py` | +| `MdnsBrowser` | M02 | `discovery/mdns.py` | +| `Mode` | M09 | `emergency/state.py` (Literal alias) | +| `NemotronBackend` *(new)* | M04 | `services/llm/backends/nemotron.py` | +| `NodeManifest` | M01 | `identity/manifest.py` | +| `ObservabilityConfig` | X04 | `config.py` | +| `OllamaBackend` | M04 | `services/llm/backends/ollama.py` | +| `OnboardingError` | M13 | `ui/onboarding.py` | +| `OpenBmbBackend` *(new)* | M04 | `services/llm/backends/openbmb.py` | +| `ParamsPredicate` | M03 | `bus/capability.py` (type alias) | +| `PeerEvent` | M02 | `discovery/peers.py` | +| `PeerRecord` | M02 | `discovery/peers.py` | +| `PeerRegistry` | M02 | `discovery/peers.py` | +| `PeriodicTask` | M12 | `node.py` | +| `PinnedCerts` | X01 | `transport/tls.py` | +| `Post` | M06 | `services/marketplace/post.py` | +| `Profile` | (types) | `hearthnet/types.py` (Literal alias) | +| `PubSubServer` | X01 | `transport/server.py` | +| `RagConfig` | X04 | `config.py` | +| `RagService` | M05 | `services/rag/service.py` | +| `RateCheck` | X01 | `transport/__init__.py` | +| `RateLimiter` | X01 | `transport/__init__.py` | +| `RateLimitedLogger` | X03 | `observability/logging.py` | +| `Registry` | M03 | `bus/registry.py` | +| `RegistryEvent` | M03 | `bus/registry.py` | +| `ReplayEngine` | X02 | `events/replay.py` | +| `RevokedEntry` | M01 | `identity/manifest.py` | +| `RouteRequest` | M03 | `bus/capability.py` | +| `Router` | M03 | `bus/router.py` | +| `SchemaValidator` | M03 | `bus/schema.py` | +| `ScoredChunk` | M05 | `services/rag/store.py` | +| `SentenceTransformerBackend` | M11 | `services/embedding/backends.py` | +| `Service` | M03 | `services/base.py` (Protocol) | +| `Snapshot` | X02 | `events/snapshot.py` | +| `SnapshotStore` | X02 | `events/snapshot.py` | +| `Span` | X03 | `observability/tracing.py` | +| `SseReader` | X01 | `transport/streams.py` | +| `SseWriter` | X01 | `transport/streams.py` | +| `StateBus` | M09 | `emergency/state.py` | +| `SyncClient` | X02 | `events/sync.py` | +| `SyncResult` | X02 | `events/sync.py` | +| `SyncServer` | X02 | `events/sync.py` | +| `Token` | M04 | `services/llm/backends/base.py` | +| `TopologyComponent` | M08 | `ui/topology.py` | +| `TopologySnapshot` | M03 | `bus/__init__.py` | +| `Trace` | X03 | `observability/tracing.py` | +| `TraceHook` | M03 | `bus/trace.py` | +| `TrackioExporter` *(new)* | X03 | `observability/metrics.py` | +| `TransferManager` | M07 | `blobs/transfer.py` | +| `TransportConfig` | X04 | `config.py` | +| `UdpAnnouncer` | M02 | `discovery/udp.py` | +| `UdpListener` | M02 | `discovery/udp.py` | +| `UiApp` | M08 | `ui/app.py` | +| `UiConfig` | X04 | `config.py` | + +--- + +## 19. Capability β†’ handler index + +For each capability in [CONTRACT Β§3.2](CAPABILITY_CONTRACT.md), where the handler lives: + +| Capability | Service | Handler | Trust | +|------------|---------|---------|-------| +| `llm.chat@1.0` | M04 `LlmService` | `handle_chat` | member | +| `llm.complete@1.0` | M04 `LlmService` | `handle_complete` | member | +| `embed.text@1.0` | M11 `EmbeddingService` | `handle_embed_text` | member | +| `rag.query@1.0` | M05 `RagService` | `handle_query` | member | +| `rag.ingest@1.0` | M05 `RagService` | `handle_ingest` | trusted | +| `rag.list_corpora@1.0` | M05 `RagService` | `handle_list_corpora` | member | +| `file.read@1.0` | M07 `FileService` | `handle_read` | member | +| `file.list@1.0` | M07 `FileService` | `handle_list` | member | +| `file.advertise@1.0` | M07 `FileService` | `handle_advertise` | member | +| `file.put@1.0` | M07 `FileService` | `handle_put` | trusted | +| `market.list@1.0` | M06 `MarketplaceService` | `handle_list` | member | +| `market.post@1.0` | M06 `MarketplaceService` | `handle_post` | member | +| `market.expire@1.0` | M06 `MarketplaceService` | `handle_expire` | member | +| `market.search@1.0` | M06 `MarketplaceService` | `handle_search` | member | +| `chat.send@1.0` | M10 `ChatService` | `handle_send` | member | +| `chat.history@1.0` | M10 `ChatService` | `handle_history` | self | +| `community.invite@1.0` | M13 (handler via bus from `make_invite`) | n/a | member with `can_invite` | +| `community.revoke@1.0` | M13 / M01 helper | n/a | 3 trusted signatures | + +--- + +## 20. Event-type β†’ producer/consumer index + +For each [CONTRACT Β§7.2](CAPABILITY_CONTRACT.md) event type: + +| Event type | Produced by | View(s) consuming | +|------------|-------------|--------------------| +| `community.created` | M13 `create_community` | M01 community manifest builder | +| `community.member.invited` | M13 `make_invite` | M01 | +| `community.member.joined` | M13 `redeem_invite` | M01 | +| `community.member.revoked` | M01 helper / `community.revoke` handler | M01 | +| `community.member.promoted` / `.demoted` | M01 helpers | M01 | +| `community.policy.updated` | M01 (root key only) | M01 | +| `node.manifest.updated` | M12 `ManifestPublisher` | optional audit views | +| `market.post.created` | M06 `handle_post` | M06 `MarketplaceView` | +| `market.post.updated` | M06 (author only) | M06 `MarketplaceView` | +| `market.post.expired` | M06 (author or sweeper) | M06 `MarketplaceView` | +| `chat.message.sent` | M10 `handle_send` | M10 `ChatView` | +| `chat.message.delivered` | M10 `DeliveryManager` | M10 `ChatView` | +| `chat.message.read` | M10 (UI) | M10 `ChatView` | +| `file.cid.advertised` | M07 `TransferManager.advertise` | local source index in `FileService` | +| `file.cid.unpinned` | M07 `BlobStore.unpin` | local source index | +| `rag.document.ingested` | M05 `IngestPipeline` | M05 (replicas may pre-fetch) | +| `federation.peer.added` / `.removed` | reserved (Phase 2) | β€” | + +--- + +## 21. Standard params for each capability descriptor + +Used by [CONTRACT Β§6.1](CAPABILITY_CONTRACT.md) node manifest embedding and by the bus's params-compatibility check. + +| Capability | `params` keys | +|------------|---------------| +| `llm.chat` | `model`, `quant`, `ctx`, `backend`, `modalities`, optionally `requires_internet` | +| `llm.complete` | same as `llm.chat` | +| `embed.text` | `model` | +| `rag.query` | `corpus`, `embedding_model`, `k_max` | +| `rag.ingest` | `corpora_available` (list) | +| `rag.list_corpora` | `{}` | +| `file.read` | `{}` | +| `file.list` | `{}` | +| `file.advertise` | `{}` | +| `file.put` | `{}` | +| `market.*` | `{}` | +| `chat.send` | `{}` | +| `chat.history` | `{}` | + +--- + +## 22. Implementation checklist (one row per implementable unit) + +Tick these off as you build. Order: dependency-correct. + +### X04 Config (~6 dataclasses, ~5 functions, ~1 exception) +- [ ] `IdentityConfig`, `CommunityConfig`, `TransportConfig`, `DiscoveryConfig`, `BusConfig` +- [ ] `LlmBackendConfig`, `LlmConfig` +- [ ] `EmbeddingConfig`, `RagConfig`, `FileConfig`, `MarketConfig`, `ChatConfig` +- [ ] `EmergencyConfig`, `UiConfig`, `ObservabilityConfig` *(incl. trackio_project/trackio_space)* +- [ ] `Config` (aggregate) +- [ ] `load`, `default_config`, `save`, `resolve_paths`, `validate` +- [ ] `ConfigError` +- [ ] `constants.py` with all 31 named constants + +### X03 Observability (~6 classes, ~14 functions) +- [ ] `configure`, `get_logger`, `JsonFormatter`, `RateLimitedLogger` +- [ ] `configure` (metrics), `counter`, `histogram`, `gauge`, `disabled` +- [ ] All 14 standard metrics pre-registered +- [ ] `TrackioExporter` *(new, optional)* +- [ ] `Trace`, `Span`, `new_trace`, `current_trace`, `attach`, `detach`, `span`, `get_recent` +- [ ] `CheckResult`, `register`, `run_all`, `run_one` +- [ ] 12 standard checks registered + +### X02 Events (~7 classes, ~3 functions) +- [ ] `EventType`, `Event` +- [ ] `LamportClock` +- [ ] `EventLog`, `EventLogError` +- [ ] `MaterialisedView` (Protocol), `ReplayEngine` +- [ ] `Snapshot`, `SnapshotStore`, `build_snapshot`, `restore_from_snapshot` +- [ ] `HeadsReport`, `SyncResult`, `SyncClient`, `SyncServer` + +### X01 Transport (~9 classes, ~1 exception) +- [ ] `HttpServer` + 10 endpoints +- [ ] `HttpClient`, `CallError` +- [ ] `Frame`, `SseWriter`, `SseReader` +- [ ] `FlowControl` +- [ ] `PinnedCerts` +- [ ] `RateCheck`, `RateLimiter` +- [ ] `PubSubServer` + +### M01 Identity (~12 classes, ~16 functions, ~1 exception) +- [ ] `KeyPair`, all keys.py functions, `IdentityError` +- [ ] `Endpoint`, `HardwareSpec`, `CapabilitySpec`, `NodeManifest` +- [ ] `CommunityPolicy`, `CommunityMember`, `RevokedEntry`, `CommunityManifest` +- [ ] All builder/parser/verifier functions +- [ ] `CapabilityToken` stub + +### M02 Discovery (~6 classes) +- [ ] `PeerRecord`, `PeerEvent`, `PeerRegistry` +- [ ] `MdnsAnnouncer`, `MdnsBrowser` +- [ ] `UdpAnnouncer`, `UdpListener` +- [ ] `DiscoveryError` + +### M03 Capability Bus (~10 classes, ~1 function, ~1 exception) β€” CRITICAL +- [ ] `CapabilityDescriptor`, `CapabilityEntry`, `RouteRequest` +- [ ] `Diff`, `RegistryEvent`, `Registry` +- [ ] `HealthTracker` +- [ ] `SchemaValidator`, `compute_schema_hash` +- [ ] `Router` (with scoring algorithm from M03 Β§5.4) +- [ ] `CallTraceEvent`, `TraceHook` +- [ ] `TopologySnapshot`, `CapabilityBus` (facade), `BusError` +- [ ] `Service` Protocol in `services/base.py` + +### M11 Embedding (~3 classes) +- [ ] `EmbeddingBackend` Protocol +- [ ] `SentenceTransformerBackend` +- [ ] `EmbeddingService` + `handle_embed_text` + params predicate + +### M04 LLM (~6 backends + 3 base classes + 2 base functions + 1 service) +- [ ] `Token`, `ChatResult`, `BackendModel`, `LlmBackend` Protocol +- [ ] `LlamaCppBackend` +- [ ] `OllamaBackend` +- [ ] `LmStudioBackend` +- [ ] `HfApiBackend` +- [ ] `AnthropicApiBackend` +- [ ] **`NemotronBackend`** *(new β€” NVIDIA NIM / locally-hosted)* +- [ ] **`OpenBmbBackend`** *(new β€” MiniCPM via vLLM/llama.cpp serve/SGLang)* +- [ ] `count_tokens_approx`, `model_family` +- [ ] `LlmService` + `handle_chat` + `handle_complete` + params predicate + +### M05 RAG (~5 classes, ~4 functions) +- [ ] `Chunk`, `chunk_text`, `chunk_pdf` +- [ ] `ScoredChunk`, `CorpusStore`, `list_corpora`, `corpus_info` +- [ ] `IngestResult`, `IngestPipeline` +- [ ] `RagService` + 3 handlers + params predicate + +### M07 File & Blobs (~4 classes, ~5 functions, ~1 exception) +- [ ] `ChunkRef`, `BlobManifest` +- [ ] All chunker.py functions +- [ ] `BlobStore`, `BlobError` +- [ ] `TransferManager` +- [ ] `FileService` + 4 handlers + +### M06 Marketplace (~3 classes) +- [ ] `Location`, `Post` +- [ ] `MarketplaceView` +- [ ] `MarketplaceService` + 4 handlers + sweeper + +### M10 Chat (~3 classes) +- [ ] `ChatMessage`, `ChatView` +- [ ] `DeliveryManager` +- [ ] `ChatService` + 2 handlers + +### M09 Emergency (~3 classes) +- [ ] `EmergencyState`, `StateBus` +- [ ] `Detector` (state machine + probe loop) + +### M08 UI (~2 classes + ~6 tab builders + theme + mobile assets) +- [ ] `UiApp`, `build_ui` +- [ ] `TopologyComponent` +- [ ] `hearthnet_theme`, `emergency_theme` +- [ ] 6 tab builders +- [ ] Mobile static assets + +### M13 Onboarding (~1 class, ~7 functions, ~1 exception) +- [ ] `InviteBlob` +- [ ] All onboarding functions +- [ ] `build_onboarding` +- [ ] `OnboardingError` + +### M12 CLI & Orchestrator (~17 commands + ~2 helper classes + 1 function) +- [ ] All 17 Click subcommands +- [ ] `ManifestPublisher`, `PeriodicTask` +- [ ] `node.start()` β€” the 15-step composition + +--- + +## 23. Notes on the trackio integration + +[Trackio](https://github.com/huggingface/trackio) is HuggingFace's local-first experiment tracker built on Gradio. Optional in HearthNet; enable by setting `config.observability.trackio_project`. + +Integration points: + +1. **Activated by config.** `TrackioExporter` is constructed only if `trackio_project` is set. Otherwise the class is unused; HearthNet runs Prometheus-only. + +2. **Bridged from TraceHook.** `M03 Β§3.6` `TraceHook.on_call_end` checks for an active exporter and forwards. No service code calls trackio directly. + +3. **Optional HF Spaces sync.** If `trackio_space` is set, runs mirror to the named Space β€” handy for sharing demo telemetry. Off by default; the demo on Christof's machine logs locally. + +4. **What gets logged.** Each LLM call is one step with: `latency_ms`, `tokens_in`, `tokens_out`, `model`, `backend`, `result`. Topology snapshots logged every 60s with mesh size, online state, capability counts. Marketplace post counts and chat throughput as gauges. + +5. **Why this fits HearthNet.** Trackio is local-first (matches HearthNet's ethos), Gradio-native (matches the existing UI stack), and gives Christof a dashboard he already knows how to extend without adding Prometheus + Grafana. + +--- + +## 24. Notes on the Nemotron and OpenBMB backends + +Both register exactly like existing backends: as `LlmBackend` implementations producing `BackendModel` entries that the service enumerates as `(backend, model)` capability instances. + +### `NemotronBackend` + +NVIDIA's Nemotron family (Llama-3.1-Nemotron-70B, Nemotron-mini, Nemotron-4-340B-instruct). Two modes: + +- **Cloud (default):** `https://integrate.api.nvidia.com/v1`, OpenAI-compatible. `requires_internet=True`. Free tier exists; bring an `NVIDIA_API_KEY`. M09 will deregister this backend automatically when offline. +- **Local (`local=True`):** point at a self-hosted NIM endpoint or vLLM-served Nemotron model. `requires_internet=False`. + +Models declared by `models: list[BackendModel]` at construction time. Use these typical entries: + +```python +BackendModel("nvidia/llama-3.1-nemotron-70b-instruct", quant="api", ctx_max=128000, modalities=["text"], requires_internet=True) +BackendModel("nvidia/nemotron-mini-4b-instruct", quant="api", ctx_max=4096, modalities=["text"], requires_internet=True) +``` + +### `OpenBmbBackend` + +OpenBMB's MiniCPM family β€” Christof's primary local-AI-workbench target. Typically served via vLLM, SGLang, or llama.cpp's HTTP server on `http://localhost:8000` (or wherever the workbench binds). OpenAI-compatible HTTP. `requires_internet=False` (always local). + +Models declared: + +```python +BackendModel("openbmb/MiniCPM4-8B", quant="fp16", ctx_max=32768, modalities=["text"], requires_internet=False) +BackendModel("openbmb/MiniCPM-V-2_6", quant="fp16", ctx_max=8192, modalities=["text","vision"], requires_internet=False) +``` + +Vision-capable MiniCPM-V variant is reserved for Phase 2 when [CONTRACT Β§12 open question 1](CAPABILITY_CONTRACT.md) lifts; vision messages stay text-only in MVP. + +### Config example + +```toml +[[llm.backends]] +name = "openbmb" +url = "http://localhost:8000" +model = "openbmb/MiniCPM4-8B" + +[[llm.backends]] +name = "nemotron" +url = "https://integrate.api.nvidia.com/v1" +model = "nvidia/llama-3.1-nemotron-70b-instruct" +api_key_env = "NVIDIA_API_KEY" + +[[llm.backends]] +name = "lmstudio" +url = "http://192.168.188.25:1234" +model = "qwen2.5-7b-instruct" +``` + +Three backends, four models if MiniCPM-V is later added β†’ eight capability entries on the bus (two each for `llm.chat` and `llm.complete` Γ— four models). The router picks among them at call time. + +--- + +## 25. Coherence guarantees enforced by this reference + +If you implement strictly against this document, the following hold automatically: + +- **No symbol name appears in two different modules** β€” see Β§18. +- **Every capability has exactly one handler** β€” see Β§19. +- **Every event type has at least one producer** β€” see Β§20. +- **Every constant is defined in `constants.py` and nowhere else** β€” see Β§0.2. +- **The 15-step orchestration produces a runnable node** β€” see Β§17 `node.py`. +- **Cross-references resolve** β€” every `M0N`/`X0N` link points at an existing spec; every spec section number used here exists. + +If you find a contradiction between this document and a spec, the spec wins by default β€” but file the discrepancy. The most common drift will be in field names of capability `params` (caught by the params predicate at registration time). + +--- + +## 26. What is intentionally NOT in this document + +- **Test code** β€” see the `tests/` section in each spec. Implement after the production code compiles. +- **Service-internal helpers** that are pure implementation detail (private functions inside one file with leading underscore). Add as you need them. +- **Phase 2/3 modules** β€” `federation.*`, `ocr.*`, `tts.*`, `stt.*`, `trans.*`, `img.*`, `chat.thread.*`, `chat.forward.*` are mentioned in specs but have no MVP symbols. +- **Vendor-specific tuning** β€” llama.cpp `n_threads`, vLLM tensor parallel, Nemotron prompt prefixes. Decide per backend at integration time. + +--- + +*End of HearthNet Implementation Reference.* +*Spec set version: v1.0 Β· this document touched: 2026-06-09.* \ No newline at end of file diff --git a/docs/screenshots/ui-ask-tab.png b/docs/screenshots/ui-ask-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..edaa28a6122920078d5eb420a0a2fdd8f84f4a82 Binary files /dev/null and b/docs/screenshots/ui-ask-tab.png differ diff --git a/docs/screenshots/ui-chat-tab.png b/docs/screenshots/ui-chat-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..a708ccabcc9a7c84275b128e27124289e8c5d462 Binary files /dev/null and b/docs/screenshots/ui-chat-tab.png differ diff --git a/docs/screenshots/ui-emergency-tab.png b/docs/screenshots/ui-emergency-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..7e6005fc75786fc8c3edb070716e949751a53c42 Binary files /dev/null and b/docs/screenshots/ui-emergency-tab.png differ diff --git a/docs/screenshots/ui-marketplace-tab.png b/docs/screenshots/ui-marketplace-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..2aeb9a947c65d5d531e8106c8a0ff8b55d748dd4 Binary files /dev/null and b/docs/screenshots/ui-marketplace-tab.png differ diff --git a/docs/screenshots/ui-settings-tab.png b/docs/screenshots/ui-settings-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..2e035b464f0bc190637453f881a1f4cbf190e875 Binary files /dev/null and b/docs/screenshots/ui-settings-tab.png differ diff --git a/hearthnet/civdef/__init__.py b/hearthnet/civdef/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f5ad746b2d6228aa392fa49d2212d19ec04c77db --- /dev/null +++ b/hearthnet/civdef/__init__.py @@ -0,0 +1,6 @@ +"""M31 β€” Civil Defense package (experimental, Phase 3).""" +from __future__ import annotations + +from hearthnet.civdef.service import Alert, AuditChain, CivilDefenseService, RoleCertificate + +__all__ = ["Alert", "AuditChain", "CivilDefenseService", "RoleCertificate"] diff --git a/hearthnet/civdef/service.py b/hearthnet/civdef/service.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2561e1ccc5c84e26dbead5b3f2e84978c7e1ca --- /dev/null +++ b/hearthnet/civdef/service.py @@ -0,0 +1,209 @@ +"""M31 β€” Civil Defense (NRW BevΓΆlkerungsschutz pilot, experimental Phase 3). + +Bridges HearthNet with THW/DRK/Feuerwehr/KatS role structures. +Produces tamper-evident audit trails for incident coordination. +Gated by config.research.civil_defense = True. +""" +from __future__ import annotations + +import hashlib +import json +import time +import uuid +from dataclasses import dataclass, field +from typing import Any + +# NRW role taxonomy +NRW_ROLES = { + "thw_helferin": "THW Helferin/Helfer", + "thw_gruppenfuehrer": "THW GruppenfΓΌhrer", + "drk_ersthelfer": "DRK Ersthelfer", + "drk_sanitaeter": "DRK SanitΓ€ter", + "feuerwehr_angehoeriger": "Feuerwehr-AngehΓΆriger", + "feuerwehr_fuehrungskraft": "Feuerwehr-FΓΌhrungskraft", + "kats_koordinator": "KatS-Koordinator", + "kats_leiterin": "KatS-Leiterin", + "bevoelkerungsschutz_beauftragte": "BevΓΆlkerungsschutzbeauftragte(r)", +} + + +@dataclass(frozen=True) +class AlertSeverity: + INFORMATION = "information" + WARNING = "warning" + ALERT = "alert" + EMERGENCY = "emergency" + + +@dataclass(frozen=True) +class RoleCertificate: + """A role certificate issued by an authority for a community member.""" + cert_id: str + role_key: str # key from NRW_ROLES + role_label: str + holder_node_id: str + issuer_node_id: str + community_id: str + region: str = "NRW" + issued_at: float = field(default_factory=time.time) + expires_at: float | None = None + issuer_signature: bytes = b"" + + def is_expired(self, now: float | None = None) -> bool: + if self.expires_at is None: + return False + return (now or time.time()) > self.expires_at + + def role_name(self) -> str: + return NRW_ROLES.get(self.role_key, self.role_label) + + +@dataclass(frozen=True) +class Alert: + """A civil-defense alert with full provenance.""" + alert_id: str + severity: str # AlertSeverity constant + title: str + body: str + area_description: str # e.g. "Issum, Kreis Kleve, NRW" + issuer_node_id: str + issuer_role_cert_id: str | None + community_id: str + event_log_id: str | None = None # optional backlink to event log entry + issued_at: float = field(default_factory=time.time) + expires_at: float | None = None + issuer_signature: bytes = b"" + + +class AuditChain: + """Tamper-evident append-only audit log for civil-defense operations. + + Each entry is a JSON-serialised dict with a backlink hash for chain integrity. + For production use, entries should be stored in the event log (X02) with + Ed25519 signatures to satisfy legal audit retention requirements. + """ + + def __init__(self) -> None: + self._entries: list[dict] = [] + self._head_hash: str = "0" * 64 + + def _hash_entry(self, entry: dict) -> str: + serialised = json.dumps(entry, sort_keys=True, ensure_ascii=True) + return hashlib.sha256(serialised.encode()).hexdigest() + + def append(self, entry_type: str, actor_node_id: str, payload: dict[str, Any]) -> str: + entry = { + "entry_id": str(uuid.uuid4()), + "entry_type": entry_type, + "actor": actor_node_id, + "payload": payload, + "timestamp": time.time(), + "prev_hash": self._head_hash, + } + entry_hash = self._hash_entry(entry) + entry["hash"] = entry_hash + self._entries.append(entry) + self._head_hash = entry_hash + return entry_hash + + def verify_integrity(self) -> bool: + """Walk the chain and verify all backlinks.""" + prev = "0" * 64 + for entry in self._entries: + if entry.get("prev_hash") != prev: + return False + expected_hash = entry["hash"] + entry_copy = {k: v for k, v in entry.items() if k != "hash"} + if self._hash_entry(entry_copy) != expected_hash: + return False + prev = expected_hash + return True + + def export(self) -> list[dict]: + return list(self._entries) + + def length(self) -> int: + return len(self._entries) + + +class CivilDefenseService: + """Civil-defense pilot service for NRW. + + Registers capabilities: + civdef.alert.issue@1.0 β€” publish a signed alert + civdef.alert.list@1.0 β€” list active alerts + civdef.cert.verify@1.0 β€” verify a role certificate + civdef.audit.export@1.0 β€” export tamper-evident audit chain + + Only active when config.research.civil_defense = True. + """ + + def __init__(self, keypair=None, bus=None) -> None: + self._keypair = keypair + self._bus = bus + self._alerts: dict[str, Alert] = {} + self._certs: dict[str, RoleCertificate] = {} + self._audit = AuditChain() + + def issue_alert( + self, + severity: str, + title: str, + body: str, + area: str, + role_cert_id: str | None = None, + community_id: str = "", + expires_in_hours: float | None = 24.0, + ) -> Alert: + node_id = getattr(self._keypair, "node_id_short", "unknown") + alert = Alert( + alert_id=str(uuid.uuid4()), + severity=severity, + title=title, + body=body, + area_description=area, + issuer_node_id=node_id, + issuer_role_cert_id=role_cert_id, + community_id=community_id, + expires_at=time.time() + expires_in_hours * 3600 if expires_in_hours else None, + ) + self._alerts[alert.alert_id] = alert + self._audit.append("alert.issued", node_id, { + "alert_id": alert.alert_id, + "severity": alert.severity, + "title": alert.title, + }) + return alert + + def list_active_alerts(self, now: float | None = None) -> list[Alert]: + now = now or time.time() + return [ + a for a in self._alerts.values() + if a.expires_at is None or a.expires_at > now + ] + + def register_cert(self, cert: RoleCertificate) -> None: + self._certs[cert.cert_id] = cert + self._audit.append("cert.registered", cert.issuer_node_id, { + "cert_id": cert.cert_id, "role": cert.role_key, "holder": cert.holder_node_id + }) + + def verify_cert(self, cert_id: str) -> dict: + cert = self._certs.get(cert_id) + if cert is None: + return {"valid": False, "reason": "cert_not_found"} + if cert.is_expired(): + return {"valid": False, "reason": "cert_expired", "cert_id": cert_id} + return { + "valid": True, + "role": cert.role_name(), + "holder": cert.holder_node_id, + "expires_at": cert.expires_at, + } + + def export_audit(self) -> dict: + return { + "entries": self._audit.export(), + "chain_valid": self._audit.verify_integrity(), + "length": self._audit.length(), + } diff --git a/hearthnet/config.py b/hearthnet/config.py index 18a96bb3853cf1104769b8e701d34ea435faef8e..d4bf52e0b03f36497133d4601ea0a645bd5ffba6 100644 --- a/hearthnet/config.py +++ b/hearthnet/config.py @@ -140,6 +140,19 @@ class ObservabilityConfig: otlp_endpoint: str | None = None +@dataclass(frozen=True) +class ResearchConfig: + """Phase 3 experimental feature flags. All default False.""" + enable: bool = False + distributed_inference: bool = False + moe_routing: bool = False + federated_learning: bool = False + lora_beacons: bool = False + evidence_graph: bool = False + civil_defense: bool = False + conformance_suite: bool = False + + @dataclass(frozen=True) class Config: identity: IdentityConfig = field(default_factory=IdentityConfig) @@ -156,6 +169,7 @@ class Config: emergency: EmergencyConfig = field(default_factory=EmergencyConfig) ui: UiConfig = field(default_factory=UiConfig) observability: ObservabilityConfig = field(default_factory=ObservabilityConfig) + research: ResearchConfig = field(default_factory=ResearchConfig) # ── ConfigError ─────────────────────────────────────────────────────────────── diff --git a/hearthnet/constants.py b/hearthnet/constants.py index d50df12a85e5c81332cf57d84fb0e806f2ffd2a8..d8221fb4eb3ee473daf79a5bf5eecbae108fb09d 100644 --- a/hearthnet/constants.py +++ b/hearthnet/constants.py @@ -58,6 +58,10 @@ INVITE_DEFAULT_TTL_SECONDS: int = 86400 # 24 h # ── RAG / Embedding ────────────────────────────────────────────────────────── RAG_DEFAULT_CHUNK_SIZE_TOKENS: int = 512 + +# ── Rerank ──────────────────────────────────────────────────────────────────── +RERANK_MAX_DOCS: int = 100 +RERANK_LOAD_TIMEOUT_SECONDS: int = 60 EMBED_MAX_TEXTS: int = 256 EMBED_MAX_CHARS: int = 8192 RAG_OVERLAP_TOKENS: int = 64 @@ -73,3 +77,12 @@ MARKET_SWEEP_INTERVAL_SECONDS: int = 60 MARKET_DEFAULT_TTL_SECONDS: int = 86400 * 7 # 1 week MARKET_MAX_TTL_SECONDS: int = 86400 * 30 # 30 days MARKET_SEARCH_CACHE_MAX: int = 5000 + +# ── STT / TTS ───────────────────────────────────────────────────────────────── +STT_MAX_AUDIO_SECONDS: int = 300 + +# ── Translation ─────────────────────────────────────────────────────────────── +TRANSLATION_MAX_CHARS: int = 4000 + +# ── Rerank ──────────────────────────────────────────────────────────────────── +RERANK_MAX_DOCS: int = 100 diff --git a/hearthnet/crypto/__init__.py b/hearthnet/crypto/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cba93884a3799854af247e882a8293e5064dc0e8 --- /dev/null +++ b/hearthnet/crypto/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class CryptoError(Exception): + """Raised for all crypto-layer failures.""" diff --git a/hearthnet/crypto/envelope.py b/hearthnet/crypto/envelope.py new file mode 100644 index 0000000000000000000000000000000000000000..a25146636b90197fabad164f7326bdf1882d6c01 --- /dev/null +++ b/hearthnet/crypto/envelope.py @@ -0,0 +1,103 @@ +"""File-chunk envelope encryption for HearthNet blobs (M23 / M07 extension).""" +from __future__ import annotations + +import hashlib +import hmac +from dataclasses import dataclass + +from hearthnet.crypto import CryptoError + +try: + import nacl.secret + import nacl.utils + + _NACL_AVAILABLE = True +except ImportError: # pragma: no cover + _NACL_AVAILABLE = False + + +def _require_nacl() -> None: + if not _NACL_AVAILABLE: + raise ImportError( + "PyNaCl is required for envelope encryption. Install it with: pip install pynacl" + ) + + +# --------------------------------------------------------------------------- +# HKDF helper (local copy to keep this module self-contained) +# --------------------------------------------------------------------------- + + +def _hkdf_sha256(ikm: bytes, salt: bytes, info: bytes, length: int) -> bytes: + if not salt: + salt = b"\x00" * 32 + prk = hmac.new(salt, ikm, hashlib.sha256).digest() + t = b"" + okm = b"" + i = 0 + while len(okm) < length: + i += 1 + t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest() + okm += t + return okm[:length] + + +# --------------------------------------------------------------------------- +# EncryptedEnvelope +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EncryptedEnvelope: + """An encrypted blob chunk with metadata.""" + + ciphertext: bytes + nonce: bytes # 24 bytes (XSalsa20 nonce) + key_id: str # identifies which key was used (e.g., recipient node_id or blob CID) + + +# --------------------------------------------------------------------------- +# Envelope encrypt / decrypt +# --------------------------------------------------------------------------- + + +def envelope_encrypt(plaintext: bytes, key: bytes) -> EncryptedEnvelope: + """Encrypt plaintext with XSalsa20-Poly1305 using the given 32-byte key.""" + _require_nacl() + if len(key) != nacl.secret.SecretBox.KEY_SIZE: + raise CryptoError( + f"Key must be {nacl.secret.SecretBox.KEY_SIZE} bytes, got {len(key)}" + ) + box = nacl.secret.SecretBox(key) + nonce = nacl.utils.random(nacl.secret.SecretBox.NONCE_SIZE) + ciphertext = bytes(box.encrypt(plaintext, nonce).ciphertext) + return EncryptedEnvelope(ciphertext=ciphertext, nonce=nonce, key_id="") + + +def envelope_decrypt(envelope: EncryptedEnvelope, key: bytes) -> bytes: + """Decrypt an EncryptedEnvelope using the given 32-byte key.""" + _require_nacl() + if len(key) != nacl.secret.SecretBox.KEY_SIZE: + raise CryptoError( + f"Key must be {nacl.secret.SecretBox.KEY_SIZE} bytes, got {len(key)}" + ) + box = nacl.secret.SecretBox(key) + try: + return bytes(box.decrypt(envelope.ciphertext, envelope.nonce)) + except Exception as exc: + raise CryptoError(f"Envelope decryption failed: {exc}") from exc + + +# --------------------------------------------------------------------------- +# Per-recipient key derivation +# --------------------------------------------------------------------------- + + +def per_recipient_key(shared_secret: bytes, recipient_id: str, blob_cid: str) -> bytes: + """Derive a 32-byte per-recipient encryption key via HKDF-SHA256. + + Binds the key to the recipient identity and the specific blob CID so that + a key derived for one blob cannot decrypt another. + """ + info = f"HearthNet_BlobKey_v1:{recipient_id}:{blob_cid}".encode() + return _hkdf_sha256(shared_secret, salt=b"HearthNet_envelope", info=info, length=32) diff --git a/hearthnet/crypto/kem.py b/hearthnet/crypto/kem.py new file mode 100644 index 0000000000000000000000000000000000000000..9028911bdf497edc4cce03a4c56d3ff588b069c9 --- /dev/null +++ b/hearthnet/crypto/kem.py @@ -0,0 +1,253 @@ +"""X25519 key agreement + X3DH for HearthNet E2E encryption (M23).""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from hearthnet.crypto import CryptoError + +if TYPE_CHECKING: + from hearthnet.identity.keys import KeyPair + +try: + import nacl.bindings + import nacl.public + import nacl.signing + + _NACL_AVAILABLE = True +except ImportError: # pragma: no cover + _NACL_AVAILABLE = False + + +def _require_nacl() -> None: + if not _NACL_AVAILABLE: + raise ImportError( + "PyNaCl is required for E2E encryption. Install it with: pip install pynacl" + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _b64url_encode(b: bytes) -> str: + return base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64url_decode(s: str) -> bytes: + pad = 4 - len(s) % 4 + if pad != 4: + s += "=" * pad + return base64.urlsafe_b64decode(s) + + +def _hkdf_sha256(ikm: bytes, salt: bytes, info: bytes, length: int) -> bytes: + """HKDF-SHA256 (RFC 5869).""" + if not salt: + salt = b"\x00" * 32 + prk = hmac.new(salt, ikm, hashlib.sha256).digest() + t = b"" + okm = b"" + i = 0 + while len(okm) < length: + i += 1 + t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest() + okm += t + return okm[:length] + + +# --------------------------------------------------------------------------- +# X25519 key types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class X25519KeyPair: + """An X25519 Diffie-Hellman key pair.""" + + private: bytes # 32 bytes + public: bytes # 32 bytes + + +# --------------------------------------------------------------------------- +# X25519 primitives +# --------------------------------------------------------------------------- + + +def x25519_generate() -> X25519KeyPair: + """Generate a random X25519 key pair using PyNaCl.""" + _require_nacl() + priv_key = nacl.public.PrivateKey.generate() + return X25519KeyPair( + private=bytes(priv_key), + public=bytes(priv_key.public_key), + ) + + +def x25519_dh(our_priv: bytes, their_pub: bytes) -> bytes: + """Compute the X25519 Diffie-Hellman shared secret (32 bytes).""" + _require_nacl() + if len(our_priv) != 32: + raise CryptoError(f"Expected 32-byte private key, got {len(our_priv)}") + if len(their_pub) != 32: + raise CryptoError(f"Expected 32-byte public key, got {len(their_pub)}") + try: + return nacl.bindings.crypto_scalarmult(our_priv, their_pub) + except Exception as exc: + raise CryptoError(f"X25519 DH failed: {exc}") from exc + + +# --------------------------------------------------------------------------- +# Ed25519 β†’ X25519 conversion +# --------------------------------------------------------------------------- + + +def derive_identity_x25519_from_ed25519(ed_kp: KeyPair) -> X25519KeyPair: + """Convert an Ed25519 signing key to X25519 using the standard nacl Elligator conversion.""" + _require_nacl() + # Reconstruct the 64-byte Ed25519 secret key (seed || verify_key) via re-deriving + seed = bytes(ed_kp.signing_key) # 32-byte seed + _vk_bytes, sk_64 = nacl.bindings.crypto_sign_seed_keypair(seed) + x25519_priv = nacl.bindings.crypto_sign_ed25519_sk_to_curve25519(sk_64) + x25519_pub = nacl.bindings.crypto_sign_ed25519_pk_to_curve25519(bytes(ed_kp.verify_key)) + return X25519KeyPair(private=x25519_priv, public=x25519_pub) + + +# --------------------------------------------------------------------------- +# Prekey bundle +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PrekeyBundle: + """What a recipient publishes so senders can establish a session without them online.""" + + identity_key_pub: bytes # 32 bytes (X25519 public key derived from Ed25519 identity) + signed_prekey_pub: bytes # 32 bytes + signed_prekey_sig: bytes # 64 bytes (Ed25519 signature over signed_prekey_pub) + one_time_prekeys: list[str] # base64url-encoded 32-byte X25519 public keys + published_at: int # unix seconds + + +def build_prekey_bundle( + ed_kp: KeyPair, + num_one_time: int = 10, +) -> tuple[PrekeyBundle, X25519KeyPair, list[X25519KeyPair]]: + """Build a prekey bundle from an Ed25519 identity key pair. + + Returns (bundle, signed_prekey_full, one_time_prekeys_full). + Caller persists the private halves; publishes only the bundle. + """ + _require_nacl() + + # Identity X25519 key (derived from Ed25519) + identity_x = derive_identity_x25519_from_ed25519(ed_kp) + + # Signed prekey + signed_prekey_kp = x25519_generate() + + # Sign the signed_prekey_pub with the Ed25519 identity key + try: + signed_msg = ed_kp.signing_key.sign(signed_prekey_kp.public) + signed_prekey_sig = signed_msg.signature # 64 bytes + except Exception as exc: + raise CryptoError(f"Failed to sign prekey: {exc}") from exc + + # One-time prekeys + otp_kps: list[X25519KeyPair] = [x25519_generate() for _ in range(num_one_time)] + otp_pubs = [_b64url_encode(kp.public) for kp in otp_kps] + + bundle = PrekeyBundle( + identity_key_pub=identity_x.public, + signed_prekey_pub=signed_prekey_kp.public, + signed_prekey_sig=bytes(signed_prekey_sig), + one_time_prekeys=otp_pubs, + published_at=int(time.time()), + ) + return bundle, signed_prekey_kp, otp_kps + + +# --------------------------------------------------------------------------- +# X3DH key agreement +# --------------------------------------------------------------------------- + +_X3DH_F = b"\xff" * 32 # Fixed padding as per Signal spec + + +def _x3dh_kdf(*dh_outputs: bytes) -> bytes: + """KDF over concatenated DH outputs β†’ 32-byte shared secret.""" + ikm = _X3DH_F + b"".join(dh_outputs) + return _hkdf_sha256(ikm, salt=b"HearthNet_X3DH_v1", info=b"HearthNet_X3DH_v1", length=32) + + +def x3dh_initiator( + our_identity_x: X25519KeyPair, + our_ephemeral_kp: X25519KeyPair, + their_bundle: PrekeyBundle, +) -> tuple[bytes, dict]: + """X3DH from the initiator's side. + + Returns (shared_secret_32bytes, session_init_message). + session_init_message carries public material needed for the responder. + """ + _require_nacl() + + # DH1 = DH(IK_a, SPK_b) + dh1 = x25519_dh(our_identity_x.private, their_bundle.signed_prekey_pub) + # DH2 = DH(EK_a, IK_b) + dh2 = x25519_dh(our_ephemeral_kp.private, their_bundle.identity_key_pub) + # DH3 = DH(EK_a, SPK_b) + dh3 = x25519_dh(our_ephemeral_kp.private, their_bundle.signed_prekey_pub) + + # Optional: DH4 = DH(EK_a, OTP_b) + used_otp_index: int | None = None + dh_outputs = [dh1, dh2, dh3] + if their_bundle.one_time_prekeys: + used_otp_index = 0 + otp_pub_bytes = _b64url_decode(their_bundle.one_time_prekeys[0]) + dh4 = x25519_dh(our_ephemeral_kp.private, otp_pub_bytes) + dh_outputs.append(dh4) + + shared_secret = _x3dh_kdf(*dh_outputs) + + session_init_message = { + "identity_pub": _b64url_encode(our_identity_x.public), + "ephemeral_pub": _b64url_encode(our_ephemeral_kp.public), + "signed_prekey_pub": _b64url_encode(their_bundle.signed_prekey_pub), + "used_otp_index": used_otp_index, + "used_otp_pub": ( + their_bundle.one_time_prekeys[0] if used_otp_index is not None else None + ), + } + return shared_secret, session_init_message + + +def x3dh_responder( + our_identity_x: X25519KeyPair, + our_signed_prekey: X25519KeyPair, + used_one_time_prekey: X25519KeyPair | None, + their_ephemeral_pub: bytes, + their_identity_pub: bytes, +) -> bytes: + """X3DH from the responder's side. Returns the shared secret (32 bytes).""" + _require_nacl() + + # DH1 = DH(SPK_b, IK_a) + dh1 = x25519_dh(our_signed_prekey.private, their_identity_pub) + # DH2 = DH(IK_b, EK_a) + dh2 = x25519_dh(our_identity_x.private, their_ephemeral_pub) + # DH3 = DH(SPK_b, EK_a) + dh3 = x25519_dh(our_signed_prekey.private, their_ephemeral_pub) + + # Optional: DH4 = DH(OTP_b, EK_a) + dh_outputs = [dh1, dh2, dh3] + if used_one_time_prekey is not None: + dh4 = x25519_dh(used_one_time_prekey.private, their_ephemeral_pub) + dh_outputs.append(dh4) + + return _x3dh_kdf(*dh_outputs) diff --git a/hearthnet/crypto/prekeys.py b/hearthnet/crypto/prekeys.py new file mode 100644 index 0000000000000000000000000000000000000000..f81e3f8fc2eb0f624860fd22f45c37b97b314036 --- /dev/null +++ b/hearthnet/crypto/prekeys.py @@ -0,0 +1,139 @@ +"""Prekey bundle storage for HearthNet E2E encryption (M23).""" +from __future__ import annotations + +import base64 +import json +import sqlite3 +from pathlib import Path + +from hearthnet.crypto import CryptoError +from hearthnet.crypto.kem import PrekeyBundle, X25519KeyPair + + +def _b64url_encode(b: bytes) -> str: + return base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64url_decode(s: str) -> bytes: + pad = 4 - len(s) % 4 + if pad != 4: + s += "=" * pad + return base64.urlsafe_b64decode(s) + + +# --------------------------------------------------------------------------- +# PrekeyStore +# --------------------------------------------------------------------------- + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS prekey_bundles ( + id INTEGER PRIMARY KEY, + bundle_json TEXT NOT NULL, + private_keys_json TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) +); + +CREATE TABLE IF NOT EXISTS one_time_prekeys ( + pub_key_b64 TEXT PRIMARY KEY, + private_key_b64 TEXT NOT NULL, + used INTEGER NOT NULL DEFAULT 0 +); +""" + + +class PrekeyStore: + """Persistent store for prekey bundles backed by SQLite.""" + + def __init__(self, db_path: str | Path = ":memory:") -> None: + self._db_path = str(db_path) + self._conn = sqlite3.connect(self._db_path, check_same_thread=False) + self._conn.executescript(_SCHEMA) + self._conn.commit() + + def close(self) -> None: + self._conn.close() + + # ------------------------------------------------------------------ + # Bundle persistence + # ------------------------------------------------------------------ + + def store_bundle( + self, + bundle: PrekeyBundle, + signed_prekey_kp: X25519KeyPair, + otp_kps: list[X25519KeyPair], + ) -> None: + """Persist the full bundle (public) and private key halves.""" + bundle_dict = { + "identity_key_pub": _b64url_encode(bundle.identity_key_pub), + "signed_prekey_pub": _b64url_encode(bundle.signed_prekey_pub), + "signed_prekey_sig": _b64url_encode(bundle.signed_prekey_sig), + "one_time_prekeys": bundle.one_time_prekeys, + "published_at": bundle.published_at, + } + private_dict = { + "signed_prekey_priv": _b64url_encode(signed_prekey_kp.private), + } + with self._conn: + self._conn.execute( + "DELETE FROM prekey_bundles", + ) + self._conn.execute( + "INSERT INTO prekey_bundles (bundle_json, private_keys_json) VALUES (?, ?)", + (json.dumps(bundle_dict), json.dumps(private_dict)), + ) + # Store one-time prekeys + for kp in otp_kps: + self._conn.execute( + "INSERT OR REPLACE INTO one_time_prekeys (pub_key_b64, private_key_b64) " + "VALUES (?, ?)", + (_b64url_encode(kp.public), _b64url_encode(kp.private)), + ) + + def load_bundle(self) -> tuple[PrekeyBundle, dict]: + """Load the stored bundle and private key map. + + Returns (PrekeyBundle, private_keys_dict) where private_keys_dict has + 'signed_prekey_priv' as a base64url string. + Raises CryptoError if no bundle is stored. + """ + row = self._conn.execute( + "SELECT bundle_json, private_keys_json FROM prekey_bundles ORDER BY id DESC LIMIT 1" + ).fetchone() + if row is None: + raise CryptoError("No prekey bundle stored") + bundle_dict = json.loads(row[0]) + private_dict = json.loads(row[1]) + # Rebuild one_time_prekeys from the active (unused) OTPs stored in the DB + otp_rows = self._conn.execute( + "SELECT pub_key_b64 FROM one_time_prekeys WHERE used = 0" + ).fetchall() + active_otps = [r[0] for r in otp_rows] + bundle = PrekeyBundle( + identity_key_pub=_b64url_decode(bundle_dict["identity_key_pub"]), + signed_prekey_pub=_b64url_decode(bundle_dict["signed_prekey_pub"]), + signed_prekey_sig=_b64url_decode(bundle_dict["signed_prekey_sig"]), + one_time_prekeys=active_otps, + published_at=bundle_dict["published_at"], + ) + return bundle, private_dict + + def consume_one_time_prekey(self, pub_key_b64: str) -> X25519KeyPair | None: + """Mark a one-time prekey as used and return its key pair. + + Returns None if the key does not exist or was already consumed. + """ + row = self._conn.execute( + "SELECT private_key_b64 FROM one_time_prekeys WHERE pub_key_b64 = ? AND used = 0", + (pub_key_b64,), + ).fetchone() + if row is None: + return None + with self._conn: + self._conn.execute( + "UPDATE one_time_prekeys SET used = 1 WHERE pub_key_b64 = ?", + (pub_key_b64,), + ) + priv = _b64url_decode(row[0]) + pub = _b64url_decode(pub_key_b64) + return X25519KeyPair(private=priv, public=pub) diff --git a/hearthnet/crypto/ratchet.py b/hearthnet/crypto/ratchet.py new file mode 100644 index 0000000000000000000000000000000000000000..d8826b75fa71e4668160810b648e160dee85d310 --- /dev/null +++ b/hearthnet/crypto/ratchet.py @@ -0,0 +1,247 @@ +"""Double Ratchet session for HearthNet E2E encryption (M23).""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +from dataclasses import dataclass, field + +from hearthnet.crypto import CryptoError + +try: + import nacl.secret + import nacl.utils + + _NACL_AVAILABLE = True +except ImportError: # pragma: no cover + _NACL_AVAILABLE = False + + +def _require_nacl() -> None: + if not _NACL_AVAILABLE: + raise ImportError( + "PyNaCl is required for the Double Ratchet. Install it with: pip install pynacl" + ) + + +# --------------------------------------------------------------------------- +# Internal KDF helpers +# --------------------------------------------------------------------------- + + +def _hkdf_sha256(ikm: bytes, salt: bytes, info: bytes, length: int) -> bytes: + if not salt: + salt = b"\x00" * 32 + prk = hmac.new(salt, ikm, hashlib.sha256).digest() + t = b"" + okm = b"" + i = 0 + while len(okm) < length: + i += 1 + t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest() + okm += t + return okm[:length] + + +def _kdf_chain(chain_key: bytes) -> tuple[bytes, bytes]: + """Advance the symmetric chain. Returns (message_key, next_chain_key).""" + msg_key = hmac.new(chain_key, b"\x01", hashlib.sha256).digest() + next_ck = hmac.new(chain_key, b"\x02", hashlib.sha256).digest() + return msg_key, next_ck + + +def _kdf_root_key(root_key: bytes, dh_out: bytes) -> tuple[bytes, bytes]: + """KDF_RK: returns (new_root_key, new_chain_key) from root_key + DH output.""" + kdf_out = _hkdf_sha256(dh_out, salt=root_key, info=b"HearthNet_RK_v1", length=64) + return kdf_out[:32], kdf_out[32:] + + +def _b64url_encode(b: bytes) -> str: + return base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64url_decode(s: str) -> bytes: + pad = 4 - len(s) % 4 + if pad != 4: + s += "=" * pad + return base64.urlsafe_b64decode(s) + + +def _secretbox_encrypt(key: bytes, plaintext: bytes) -> bytes: + """Encrypt plaintext with XSalsa20-Poly1305. Returns nonce+ciphertext.""" + box = nacl.secret.SecretBox(key) + return bytes(box.encrypt(plaintext)) + + +def _secretbox_decrypt(key: bytes, data: bytes) -> bytes: + """Decrypt nonce+ciphertext with XSalsa20-Poly1305.""" + box = nacl.secret.SecretBox(key) + try: + return bytes(box.decrypt(data)) + except Exception as exc: + raise CryptoError(f"Ratchet decryption failed: {exc}") from exc + + +# --------------------------------------------------------------------------- +# Ratchet session state +# --------------------------------------------------------------------------- + +_MAX_SKIP = 100 # Safety cap on how many messages can be skipped + + +@dataclass +class RatchetSession: + """Bidirectional Double Ratchet session between two nodes.""" + + peer_node_id_full: str + root_key: bytes + chain_key: bytes # send chain key + recv_chain_key: bytes # receive chain key + message_keys: dict[tuple[int, int], bytes] = field(default_factory=dict) + send_counter: int = 0 + recv_counter: int = 0 + epoch: int = 0 # DH ratchet step index + + # X25519 ratchet keys (stored as raw bytes) + ratchet_priv: bytes | None = None # our current DH ratchet private key + ratchet_pub: bytes | None = None # our current DH ratchet public key + remote_ratchet_pub: bytes | None = None # peer's current DH ratchet public key + + # Set to True for the responder (Bob): the first receive must NOT trigger a + # DH ratchet step because the initial recv_chain_key is already established + # from X3DH. The DH ratchet only fires when epoch in the header advances. + is_initiator: bool = True + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def init_from_shared_secret( + shared_secret: bytes, + is_initiator: bool, + peer_node_id_full: str = "", +) -> RatchetSession: + """Initialise a new ratchet session from an X3DH shared secret.""" + _require_nacl() + + from hearthnet.crypto.kem import x25519_generate + + root_key = _hkdf_sha256( + shared_secret, + salt=b"HearthNet_X3DH", + info=b"RootKey_v1", + length=32, + ) + info_send = b"InitiatorChain_v1" if is_initiator else b"ResponderChain_v1" + info_recv = b"ResponderChain_v1" if is_initiator else b"InitiatorChain_v1" + chain_key = _hkdf_sha256(root_key, salt=b"", info=info_send, length=32) + recv_chain_key = _hkdf_sha256(root_key, salt=b"", info=info_recv, length=32) + + ratchet_kp = x25519_generate() + return RatchetSession( + peer_node_id_full=peer_node_id_full, + root_key=root_key, + chain_key=chain_key, + recv_chain_key=recv_chain_key, + ratchet_priv=ratchet_kp.private, + ratchet_pub=ratchet_kp.public, + is_initiator=is_initiator, + ) + + +def _dh_ratchet_step(session: RatchetSession, new_remote_ratchet_pub: bytes) -> None: + """Perform a DH ratchet step on receiving a new remote ratchet public key.""" + from hearthnet.crypto.kem import x25519_dh, x25519_generate + + assert session.ratchet_priv is not None + + # Step 1: derive new recv chain key from DH(our_current, their_new) + dh1 = x25519_dh(session.ratchet_priv, new_remote_ratchet_pub) + root_key, recv_chain_key = _kdf_root_key(session.root_key, dh1) + + # Step 2: generate new DH ratchet keypair + new_kp = x25519_generate() + + # Step 3: derive new send chain key from DH(our_new, their_new) + dh2 = x25519_dh(new_kp.private, new_remote_ratchet_pub) + root_key2, send_chain_key = _kdf_root_key(root_key, dh2) + + # Mutate session state + session.root_key = root_key2 + session.chain_key = send_chain_key + session.recv_chain_key = recv_chain_key + session.remote_ratchet_pub = new_remote_ratchet_pub + session.ratchet_priv = new_kp.private + session.ratchet_pub = new_kp.public + session.epoch += 1 + session.send_counter = 0 + session.recv_counter = 0 + + +def encrypt(session: RatchetSession, plaintext: bytes) -> tuple[bytes, dict]: + """Encrypt a message. Returns (ciphertext, header). + + header contains: ratchet_pub (b64url), index, epoch. + """ + _require_nacl() + + msg_key, next_ck = _kdf_chain(session.chain_key) + session.chain_key = next_ck + counter = session.send_counter + session.send_counter += 1 + + ciphertext = _secretbox_encrypt(msg_key, plaintext) + + header = { + "ratchet_pub": _b64url_encode(session.ratchet_pub or b"\x00" * 32), + "index": counter, + "epoch": session.epoch, + } + return ciphertext, header + + +def decrypt(session: RatchetSession, ciphertext: bytes, header: dict) -> bytes: + """Decrypt a message using ratchet state. Handles out-of-order messages (limited).""" + _require_nacl() + + ratchet_pub = _b64url_decode(header["ratchet_pub"]) + index: int = header["index"] + epoch: int = header["epoch"] + + # Check the skipped-message cache first + cached_key = session.message_keys.pop((epoch, index), None) + if cached_key is not None: + return _secretbox_decrypt(cached_key, ciphertext) + + # DH ratchet step rules: + # - Initiator's first decrypt: remote_ratchet_pub is None but epoch==0 + # means the responder is starting their first reply β€” do the DH step. + # - Responder's first decrypt (epoch==0): the initial recv_chain_key is + # already set from X3DH; just record the peer's ratchet pub, no DH step. + # - Any subsequent epoch mismatch: always do the DH step. + if ratchet_pub != session.remote_ratchet_pub: + if session.remote_ratchet_pub is None and not session.is_initiator and epoch == 0: + # Responder receiving initiator's first message β€” use pre-computed + # recv_chain_key as-is; just record the initiator's ratchet pub. + session.remote_ratchet_pub = ratchet_pub + else: + _dh_ratchet_step(session, ratchet_pub) + + # Advance recv chain to the target counter, caching skipped keys + skipped = 0 + while session.recv_counter < index: + if skipped >= _MAX_SKIP: + raise CryptoError("Too many skipped messages in ratchet") + msg_key, next_ck = _kdf_chain(session.recv_chain_key) + session.recv_chain_key = next_ck + session.message_keys[(session.epoch, session.recv_counter)] = msg_key + session.recv_counter += 1 + skipped += 1 + + msg_key, next_ck = _kdf_chain(session.recv_chain_key) + session.recv_chain_key = next_ck + session.recv_counter += 1 + + return _secretbox_decrypt(msg_key, ciphertext) diff --git a/hearthnet/dht/__init__.py b/hearthnet/dht/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/dht/bootstrap.py b/hearthnet/dht/bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..92ffc12491ecbd1863064da94304c33566f10662 --- /dev/null +++ b/hearthnet/dht/bootstrap.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class BootstrapConfig: + peers: list[str] + relay_url: str | None = None + + +_DEFAULT_PEERS: list[str] = ["relay.hearthnet.de:7080"] + + +def load_bootstrap(config_path: str | Path | None = None) -> BootstrapConfig: + """Load bootstrap configuration from a JSON file, or return defaults. + + The config file is optional and its absence or any error is handled + gracefully by returning the built-in default peers. + + Expected JSON schema:: + + { + "peers": ["host:port", ...], + "relay_url": "https://relay.hearthnet.de" // optional + } + """ + if config_path is not None: + path = Path(config_path) + if path.is_file(): + try: + data: Any = json.loads(path.read_text(encoding="utf-8")) + peers: list[str] = data.get("peers", _DEFAULT_PEERS) + relay_url: str | None = data.get("relay_url") + return BootstrapConfig(peers=peers, relay_url=relay_url) + except Exception: + pass # fall through to defaults + + # Auto-discover relay_url from XDG config if possible + try: + from hearthnet.config import _default_config_path, load # noqa: PLC0415 + + cfg_file = _default_config_path() + if cfg_file.is_file(): + cfg = load(cfg_file) + relay_url = getattr(cfg, "relay_url", None) + return BootstrapConfig(peers=list(_DEFAULT_PEERS), relay_url=relay_url) + except Exception: + pass + + return BootstrapConfig(peers=list(_DEFAULT_PEERS), relay_url=None) diff --git a/hearthnet/dht/kademlia.py b/hearthnet/dht/kademlia.py new file mode 100644 index 0000000000000000000000000000000000000000..e47be9a36ae170cc3a8723b247ae03e55fad8fa1 --- /dev/null +++ b/hearthnet/dht/kademlia.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import time +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class DhtContact: + node_key: bytes # 32-byte SHA-256 of node_id + endpoint: str # "host:port" + node_id: str # human-readable node identifier + last_seen: float # monotonic timestamp + + +@dataclass(frozen=True) +class DhtValue: + key: bytes # lookup key (arbitrary bytes) + payload: dict # stored data + expires_at: int # Unix epoch seconds + + +def _xor_distance(a: bytes, b: bytes) -> int: + """XOR metric over equal-length byte strings, returned as integer.""" + # Pad to same length if needed + la, lb = len(a), len(b) + if la < lb: + a = a.ljust(lb, b"\x00") + elif lb < la: + b = b.ljust(la, b"\x00") + result = 0 + for x, y in zip(a, b): + result = (result << 8) | (x ^ y) + return result + + +def _bucket_index(own_key: bytes, target_key: bytes) -> int: + """Return the Kademlia bucket index [0, 255] for the target key.""" + dist = _xor_distance(own_key, target_key) + if dist == 0: + return 0 + # Most-significant bit position of the XOR distance + bit_length = dist.bit_length() + return bit_length - 1 # 0-based, max 255 for 32-byte keys + + +class RoutingTable: + """256 buckets of K=8 contacts each.""" + + def __init__(self, own_key: bytes, k: int = 8) -> None: + self._own_key = own_key + self._k = k + self._buckets: list[list[DhtContact]] = [[] for _ in range(256)] + + def add_contact(self, contact: DhtContact) -> None: + if contact.node_key == self._own_key: + return + idx = _bucket_index(self._own_key, contact.node_key) + bucket = self._buckets[idx] + + # Replace existing entry for the same node_key + for i, existing in enumerate(bucket): + if existing.node_key == contact.node_key: + bucket[i] = contact + return + + if len(bucket) < self._k: + bucket.append(contact) + else: + # Replace the oldest (least recently seen) contact + oldest_idx = min(range(len(bucket)), key=lambda i: bucket[i].last_seen) + bucket[oldest_idx] = contact + + def find_closest(self, key: bytes, k: int = 8) -> list[DhtContact]: + """Return up to k contacts closest (by XOR) to key.""" + all_contacts = self.all_contacts() + all_contacts.sort(key=lambda c: _xor_distance(c.node_key, key)) + return all_contacts[:k] + + def size(self) -> int: + return sum(len(b) for b in self._buckets) + + def all_contacts(self) -> list[DhtContact]: + contacts: list[DhtContact] = [] + for bucket in self._buckets: + contacts.extend(bucket) + return contacts + + +class KademliaNode: + """Local Kademlia DHT node: routing table + local value store.""" + + def __init__(self, node_id: str, k: int = 8, alpha: int = 3) -> None: + self._node_id = node_id + self._k = k + self._alpha = alpha + + # Deterministic 32-byte key from node_id + self.node_key: bytes = hashlib.sha256(node_id.encode()).digest() + + self.routing_table = RoutingTable(own_key=self.node_key, k=k) + self.local_store: dict[bytes, DhtValue] = {} + + # ── Value store ─────────────────────────────────────────────────────────── + + def store(self, key: bytes, value: dict, ttl: int = 3600) -> None: + expires_at = int(time.time()) + ttl + self.local_store[key] = DhtValue(key=key, payload=value, expires_at=expires_at) + + def find_value(self, key: bytes) -> DhtValue | None: + entry = self.local_store.get(key) + if entry is None: + return None + if int(time.time()) > entry.expires_at: + del self.local_store[key] + return None + return entry + + # ── Routing ─────────────────────────────────────────────────────────────── + + def find_closest(self, key: bytes, k: int = 8) -> list[DhtContact]: + return self.routing_table.find_closest(key, k) + + def update_contact(self, contact: DhtContact) -> None: + self.routing_table.add_contact(contact) + + # ── Maintenance ─────────────────────────────────────────────────────────── + + def expire_stale(self) -> int: + """Remove expired values. Returns count of removed entries.""" + now = int(time.time()) + stale = [k for k, v in self.local_store.items() if now > v.expires_at] + for k in stale: + del self.local_store[k] + return len(stale) diff --git a/hearthnet/distributed_inference/__init__.py b/hearthnet/distributed_inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f378927afedf58eb2a4870374428d8847914fa9 --- /dev/null +++ b/hearthnet/distributed_inference/__init__.py @@ -0,0 +1,11 @@ +"""M26 β€” Distributed Inference (experimental, Phase 3). + +Gated by config.research.distributed_inference = True. +Layer-shards an LLM across multiple LAN nodes (Petals-style). +""" +from __future__ import annotations + +from hearthnet.distributed_inference.shard import ShardDescriptor, ShardServer +from hearthnet.distributed_inference.pipeline import Pipeline, PipelineOrchestrator + +__all__ = ["ShardDescriptor", "ShardServer", "Pipeline", "PipelineOrchestrator"] diff --git a/hearthnet/distributed_inference/pipeline.py b/hearthnet/distributed_inference/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..05bc0635eb421935dd58c2ac33446cc707589df9 --- /dev/null +++ b/hearthnet/distributed_inference/pipeline.py @@ -0,0 +1,97 @@ +"""Pipeline orchestrator for distributed inference (M26 β€” experimental).""" +from __future__ import annotations + +import time +from dataclasses import dataclass, field + +from hearthnet.distributed_inference.shard import ShardDescriptor + + +@dataclass +class Pipeline: + """A planned pipeline: ordered list of shards covering layers 0..N.""" + pipeline_id: str + model_id: str + shards: list[ShardDescriptor] + established_at: float = field(default_factory=time.time) + status: str = "planned" # "planned" | "active" | "failed" | "done" + + @property + def is_complete(self) -> bool: + """True if shards cover a contiguous range starting at 0.""" + if not self.shards: + return False + sorted_shards = sorted(self.shards, key=lambda s: s.layer_lo) + if sorted_shards[0].layer_lo != 0: + return False + expected_next = 0 + for shard in sorted_shards: + if shard.layer_lo != expected_next: + return False + expected_next = shard.layer_hi + 1 + return True + + +class PipelineOrchestrator: + """Constructs and executes layer pipelines across multiple shard servers. + + Experimental Phase 3 feature. Not production-ready. + The orchestrator: + 1. Finds available shards from the capability bus + 2. Plans a pipeline covering all layers + 3. Chains forward passes across shards + 4. Streams token output back to the caller + """ + + def __init__(self, bus=None) -> None: + self._bus = bus + self._pipelines: dict[str, Pipeline] = {} + + def plan(self, model_id: str, available_shards: list[ShardDescriptor]) -> Pipeline | None: + """Choose a minimal set of shards that covers layers 0..N continuously.""" + import uuid + model_shards = [s for s in available_shards if s.model_id == model_id] + if not model_shards: + return None + + # Greedy cover: sort by layer_lo, pick first shard that starts where we left off + sorted_shards = sorted(model_shards, key=lambda s: s.layer_lo) + chosen: list[ShardDescriptor] = [] + expected = 0 + for shard in sorted_shards: + if shard.layer_lo == expected: + chosen.append(shard) + expected = shard.layer_hi + 1 + + pipeline = Pipeline( + pipeline_id=str(uuid.uuid4()), + model_id=model_id, + shards=chosen, + ) + if pipeline.is_complete: + self._pipelines[pipeline.pipeline_id] = pipeline + return pipeline + return None + + async def run(self, pipeline_id: str, prompt_tokens: list[int]) -> list[int]: + """Execute a pipeline. Returns generated tokens. + + Experimental β€” raises NotImplementedError in current state. + """ + raise NotImplementedError( + "PipelineOrchestrator.run() is not yet implemented. " + "This is an experimental Phase 3 feature (M26). " + "Enable with config.research.distributed_inference = True." + ) + + def list_pipelines(self) -> list[dict]: + return [ + { + "pipeline_id": p.pipeline_id, + "model_id": p.model_id, + "shard_count": len(p.shards), + "complete": p.is_complete, + "status": p.status, + } + for p in self._pipelines.values() + ] diff --git a/hearthnet/distributed_inference/shard.py b/hearthnet/distributed_inference/shard.py new file mode 100644 index 0000000000000000000000000000000000000000..295943423f7ab2ade21347fc7e726179ce8713ad --- /dev/null +++ b/hearthnet/distributed_inference/shard.py @@ -0,0 +1,86 @@ +"""Shard descriptors and server for distributed inference (M26 β€” experimental).""" +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass, field +from typing import Any + +ShardID = str + + +@dataclass(frozen=True) +class ShardDescriptor: + """Describes one contiguous layer range hosted by a node.""" + shard_id: ShardID # ":-" + model_id: str + layer_lo: int + layer_hi: int # inclusive + node_id: str + endpoint: str + dtype: str = "float16" # "float16" | "bfloat16" | "int8" + advertised_at: float = field(default_factory=time.time) + + @property + def layer_count(self) -> int: + return self.layer_hi - self.layer_lo + 1 + + +class ShardServer: + """Hosts one contiguous shard. + + Loaded lazily on first use; evictable under memory pressure. + This is an experimental module β€” only active when + config.research.distributed_inference = True. + """ + + def __init__(self, descriptor: ShardDescriptor) -> None: + self._desc = descriptor + self._model: Any = None + self._loaded = False + + @property + def descriptor(self) -> ShardDescriptor: + return self._desc + + def is_loaded(self) -> bool: + return self._loaded + + def load(self) -> None: + """Load the shard weights. Raises ImportError if torch unavailable.""" + try: + import torch # noqa: F401 + except ImportError as exc: + raise ImportError( + "PyTorch is required for distributed inference. " + "Install: pip install torch" + ) from exc + # Actual weight loading would go here; placeholder for the research prototype. + self._loaded = True + + def evict(self) -> None: + """Free shard memory.""" + self._model = None + self._loaded = False + + async def forward(self, hidden_states: bytes, dtype: str = "float16") -> bytes: + """Run one forward pass through this shard. + + hidden_states: raw tensor bytes (X08 tensor-transport format) + Returns: raw output tensor bytes + """ + if not self._loaded: + self.load() + # Placeholder β€” real implementation uses torch to slice model and forward. + raise NotImplementedError( + "ShardServer.forward() is not yet implemented for this shard. " + "This is an experimental Phase 3 feature." + ) + + def health(self) -> dict: + return { + "shard_id": self._desc.shard_id, + "loaded": self._loaded, + "layers": f"{self._desc.layer_lo}-{self._desc.layer_hi}", + "status": "loaded" if self._loaded else "unloaded", + } diff --git a/hearthnet/evidence/__init__.py b/hearthnet/evidence/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6dffb76565700f1c04b51776d0520d37f1bcbd95 --- /dev/null +++ b/hearthnet/evidence/__init__.py @@ -0,0 +1,6 @@ +"""M30 β€” Evidence Graph package (experimental, Phase 3).""" +from __future__ import annotations + +from hearthnet.evidence.store import Claim, ClaimID, ClaimSource, ClaimStore, Attestation, Dispute + +__all__ = ["Claim", "ClaimID", "ClaimSource", "ClaimStore", "Attestation", "Dispute"] diff --git a/hearthnet/evidence/store.py b/hearthnet/evidence/store.py new file mode 100644 index 0000000000000000000000000000000000000000..ec4e9af23b1ed83c1090d654c10f6323331bcb26 --- /dev/null +++ b/hearthnet/evidence/store.py @@ -0,0 +1,131 @@ +"""M30 β€” Evidence Graph & EBKH Integration (experimental, Phase 3). + +Content-addressed claim graph alongside the event log. +Events record what happened; claims record what is believed and by whom. +Gated by config.research.evidence_graph = True. +""" +from __future__ import annotations + +import hashlib +import json +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, NewType + +ClaimID = NewType("ClaimID", str) +SourceID = NewType("SourceID", str) + + +@dataclass(frozen=True) +class ClaimSource: + source_id: SourceID + source_type: str # "event" | "external" | "ebkh" | "manual" + url: str | None = None + retrieved_at: float | None = None + reliability_score: float = 1.0 + + +@dataclass(frozen=True) +class Claim: + """An assertion by a node about some fact, with provenance.""" + claim_id: ClaimID + subject: str # what the claim is about (URI or free text) + predicate: str # what is being claimed + object_: str # the claimed value + asserted_by: str # NodeID of the asserting node + sources: tuple[ClaimSource, ...] + community_id: str + asserted_at: float = field(default_factory=time.time) + confidence: float = 1.0 + signature: bytes = b"" + + def content_id(self) -> ClaimID: + """Stable content-addressed ID based on subject/predicate/object.""" + payload = f"{self.subject}\x00{self.predicate}\x00{self.object_}" + return ClaimID("claim:" + hashlib.sha256(payload.encode()).hexdigest()[:16]) + + +@dataclass(frozen=True) +class Attestation: + """A second node vouches for a claim.""" + claim_id: ClaimID + attested_by: str + attested_at: float = field(default_factory=time.time) + signature: bytes = b"" + + +@dataclass(frozen=True) +class Dispute: + """A node disputes a claim.""" + claim_id: ClaimID + disputed_by: str + reason: str + disputed_at: float = field(default_factory=time.time) + counter_claim_id: ClaimID | None = None + + +class ClaimStore: + """Append-only content-addressed claim store (in-memory prototype). + + Production implementation will use a Merkle-DAG store backed by SQLite. + EBKH adapter (PostGIS + OSINT) plugs in via the `import_ebkh_record` method. + """ + + def __init__(self) -> None: + self._claims: dict[ClaimID, Claim] = {} + self._attestations: dict[ClaimID, list[Attestation]] = {} + self._disputes: dict[ClaimID, list[Dispute]] = {} + + def add_claim(self, claim: Claim) -> ClaimID: + cid = claim.content_id() + if cid not in self._claims: + self._claims[cid] = claim + return cid + + def attest(self, attestation: Attestation) -> None: + self._attestations.setdefault(attestation.claim_id, []).append(attestation) + + def dispute(self, dispute: Dispute) -> None: + self._disputes.setdefault(dispute.claim_id, []).append(dispute) + + def get_claim(self, claim_id: ClaimID) -> Claim | None: + return self._claims.get(claim_id) + + def find_by_subject(self, subject: str) -> list[Claim]: + return [c for c in self._claims.values() if c.subject == subject] + + def attestation_count(self, claim_id: ClaimID) -> int: + return len(self._attestations.get(claim_id, [])) + + def is_disputed(self, claim_id: ClaimID) -> bool: + return bool(self._disputes.get(claim_id)) + + def import_ebkh_record(self, record: dict[str, Any], asserted_by: str, community_id: str) -> ClaimID: + """Import a record from Christof's EBKH system as a Claim. + + Expects record to have at minimum: subject, predicate, object, source_url. + """ + source = ClaimSource( + source_id=SourceID(record.get("ebkh_id", str(uuid.uuid4()))), + source_type="ebkh", + url=record.get("source_url"), + reliability_score=float(record.get("reliability", 1.0)), + ) + claim = Claim( + claim_id=ClaimID(str(uuid.uuid4())), + subject=str(record.get("subject", "")), + predicate=str(record.get("predicate", "asserts")), + object_=str(record.get("object", "")), + asserted_by=asserted_by, + sources=(source,), + community_id=community_id, + ) + return self.add_claim(claim) + + def summary(self) -> dict: + return { + "claims": len(self._claims), + "attestations": sum(len(v) for v in self._attestations.values()), + "disputes": sum(len(v) for v in self._disputes.values()), + } diff --git a/hearthnet/federation/__init__.py b/hearthnet/federation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d48db4f9f85e1752cf424c49ee18a6907c3f160 --- /dev/null +++ b/hearthnet/federation/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/hearthnet/federation/manifest.py b/hearthnet/federation/manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..c9e65ab82185620cc59e4798df28a4672584c49e --- /dev/null +++ b/hearthnet/federation/manifest.py @@ -0,0 +1,266 @@ +"""Federation manifest builder and verifier (M14).""" +from __future__ import annotations + +import base64 +import json +import time +from dataclasses import dataclass +from typing import Any + +try: + import nacl.bindings + import nacl.exceptions + + _NACL_AVAILABLE = True +except ImportError: # pragma: no cover + nacl = None # type: ignore[assignment] + _NACL_AVAILABLE = False + + +class ManifestError(Exception): + """Raised for federation manifest validation failures.""" + + +# --------------------------------------------------------------------------- +# Value types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class FederationScope: + """What one community grants the other.""" + + capabilities: list[str] + data_visibility: str = "public_corpora_only" # "public_corpora_only"|"members_only"|"open" + + +@dataclass(frozen=True) +class FederationManifest: + """A bilateral federation agreement between two communities.""" + + schema_version: int + federation_id: str + community_a_id: str + community_a_name: str + community_b_id: str + community_b_name: str + scope_a_to_b: FederationScope # what A grants B + scope_b_to_a: FederationScope # what B grants A + sig_a: str # Ed25519 sig from anchor of community A + sig_b: str # Ed25519 sig from anchor of community B + co_signers_a: list[str] # additional anchor signatures from community A + co_signers_b: list[str] # additional anchor signatures from community B + created_at: int # unix seconds + expires_at: int # unix seconds + bootstrap_endpoints_a: list[str] + bootstrap_endpoints_b: list[str] + + def is_expired(self, now: int | None = None) -> bool: + ts = now if now is not None else int(time.time()) + return ts >= self.expires_at + + +@dataclass(frozen=True) +class FederationProposal: + """A draft federation proposal from community A to community B.""" + + community_a: str # community_id of proposer + community_b: str # community_id of target + scope_a: FederationScope # scope A proposes to grant B + scope_b: FederationScope # scope A requests from B + bootstrap_a: list[str] # endpoints for community A + bootstrap_b: list[str] # expected endpoints for community B + proposed_at: int # unix seconds + proposer_sig: str # Ed25519 sig over the proposal body by an anchor of A + + +# --------------------------------------------------------------------------- +# Encoding helpers +# --------------------------------------------------------------------------- + + +def _b64url_encode(b: bytes) -> str: + return base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64url_decode(s: str) -> bytes: + pad = 4 - len(s) % 4 + if pad != 4: + s += "=" * pad + return base64.urlsafe_b64decode(s) + + +def _scope_to_dict(s: FederationScope) -> dict: + return {"capabilities": list(s.capabilities), "data_visibility": s.data_visibility} + + +def _scope_from_dict(d: dict) -> FederationScope: + return FederationScope( + capabilities=list(d.get("capabilities", [])), + data_visibility=d.get("data_visibility", "public_corpora_only"), + ) + + +def _proposal_body(proposal: FederationProposal) -> bytes: + """Canonical bytes for signing a proposal.""" + body = { + "community_a": proposal.community_a, + "community_b": proposal.community_b, + "scope_a": _scope_to_dict(proposal.scope_a), + "scope_b": _scope_to_dict(proposal.scope_b), + "bootstrap_a": proposal.bootstrap_a, + "bootstrap_b": proposal.bootstrap_b, + "proposed_at": proposal.proposed_at, + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + + +def _manifest_body(manifest: FederationManifest) -> bytes: + """Canonical bytes for signing a manifest (excludes sig_a, sig_b, co_signers).""" + body = { + "schema_version": manifest.schema_version, + "federation_id": manifest.federation_id, + "community_a_id": manifest.community_a_id, + "community_a_name": manifest.community_a_name, + "community_b_id": manifest.community_b_id, + "community_b_name": manifest.community_b_name, + "scope_a_to_b": _scope_to_dict(manifest.scope_a_to_b), + "scope_b_to_a": _scope_to_dict(manifest.scope_b_to_a), + "created_at": manifest.created_at, + "expires_at": manifest.expires_at, + "bootstrap_endpoints_a": manifest.bootstrap_endpoints_a, + "bootstrap_endpoints_b": manifest.bootstrap_endpoints_b, + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + + +def build_federation_proposal( + our_manifest: Any, + our_keypair: Any, + their_community_id: str, + their_community_name: str, + scope_we_grant: FederationScope, + scope_they_grant: FederationScope, + bootstrap_endpoints: list[str], +) -> FederationProposal: + """Create a signed federation proposal to send to a peer community.""" + if not _NACL_AVAILABLE: + raise ManifestError("PyNaCl is required for federation. Install pynacl.") + + our_community_id = getattr(our_manifest, "community_id", "") + now = int(time.time()) + + # Build an unsigned proposal first to produce the body bytes for signing + unsigned_proposal = FederationProposal( + community_a=our_community_id, + community_b=their_community_id, + scope_a=scope_we_grant, + scope_b=scope_they_grant, + bootstrap_a=bootstrap_endpoints, + bootstrap_b=[], + proposed_at=now, + proposer_sig="", + ) + body = _proposal_body(unsigned_proposal) + try: + signed = our_keypair.signing_key.sign(body) + sig_b64 = _b64url_encode(signed.signature) + except Exception as exc: + raise ManifestError(f"Signing proposal failed: {exc}") from exc + + return FederationProposal( + community_a=our_community_id, + community_b=their_community_id, + scope_a=scope_we_grant, + scope_b=scope_they_grant, + bootstrap_a=bootstrap_endpoints, + bootstrap_b=[], + proposed_at=now, + proposer_sig=sig_b64, + ) + + +def co_sign_federation(proposal: FederationProposal, keypair: Any, role: str) -> dict: + """Co-sign a federation proposal on behalf of a community anchor. + + Returns {signed_by: node_id, signature: b64url, role: str}. + """ + if not _NACL_AVAILABLE: + raise ManifestError("PyNaCl is required. Install pynacl.") + + from hearthnet.identity.keys import full_node_id + + body = _proposal_body(proposal) + try: + signed = keypair.signing_key.sign(body) + sig_b64 = _b64url_encode(signed.signature) + except Exception as exc: + raise ManifestError(f"Co-signing failed: {exc}") from exc + + node_id = full_node_id(bytes(keypair.verify_key)) + return {"signed_by": node_id, "signature": sig_b64, "role": role} + + +def finalize_federation_manifest( + proposal: FederationProposal, + sig_a: str, + sig_b: str, + community_a_name: str = "", + community_b_name: str = "", + ttl_seconds: int = 365 * 24 * 3600, +) -> FederationManifest: + """Combine a proposal and both anchor signatures into a finalized manifest.""" + from hearthnet.events.types import new_ulid + + now = int(time.time()) + return FederationManifest( + schema_version=1, + federation_id=new_ulid(), + community_a_id=proposal.community_a, + community_a_name=community_a_name, + community_b_id=proposal.community_b, + community_b_name=community_b_name, + scope_a_to_b=proposal.scope_a, + scope_b_to_a=proposal.scope_b, + sig_a=sig_a, + sig_b=sig_b, + co_signers_a=[], + co_signers_b=[], + created_at=now, + expires_at=now + ttl_seconds, + bootstrap_endpoints_a=proposal.bootstrap_a, + bootstrap_endpoints_b=proposal.bootstrap_b, + ) + + +def verify_federation_manifest( + manifest: FederationManifest, + community_a_verify_key: Any, + community_b_verify_key: Any, +) -> None: + """Verify both anchor signatures on a manifest. Raises ManifestError if invalid.""" + if not _NACL_AVAILABLE: + raise ManifestError("PyNaCl is required. Install pynacl.") + + body = _manifest_body(manifest) + + for label, sig_str, vk in [ + ("community_a", manifest.sig_a, community_a_verify_key), + ("community_b", manifest.sig_b, community_b_verify_key), + ]: + if not sig_str: + raise ManifestError(f"Missing signature for {label}") + try: + import nacl.exceptions + + sig_bytes = _b64url_decode(sig_str) + vk.verify(body, sig_bytes) + except nacl.exceptions.BadSignatureError as exc: + raise ManifestError(f"Invalid signature for {label}: {exc}") from exc + except Exception as exc: + raise ManifestError(f"Signature verification error for {label}: {exc}") from exc diff --git a/hearthnet/federation/peering.py b/hearthnet/federation/peering.py new file mode 100644 index 0000000000000000000000000000000000000000..854b49771f60e64555e19a68cd19046efeb5ab90 --- /dev/null +++ b/hearthnet/federation/peering.py @@ -0,0 +1,234 @@ +"""Cross-community peering store and HTTP client (M14).""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + +from hearthnet.federation.manifest import ( + FederationManifest, + FederationProposal, + ManifestError, + _scope_from_dict, + _scope_to_dict, + co_sign_federation, +) + +# --------------------------------------------------------------------------- +# Serialisation helpers +# --------------------------------------------------------------------------- + + +def _manifest_to_dict(m: FederationManifest) -> dict: + return { + "schema_version": m.schema_version, + "federation_id": m.federation_id, + "community_a_id": m.community_a_id, + "community_a_name": m.community_a_name, + "community_b_id": m.community_b_id, + "community_b_name": m.community_b_name, + "scope_a_to_b": _scope_to_dict(m.scope_a_to_b), + "scope_b_to_a": _scope_to_dict(m.scope_b_to_a), + "sig_a": m.sig_a, + "sig_b": m.sig_b, + "co_signers_a": m.co_signers_a, + "co_signers_b": m.co_signers_b, + "created_at": m.created_at, + "expires_at": m.expires_at, + "bootstrap_endpoints_a": m.bootstrap_endpoints_a, + "bootstrap_endpoints_b": m.bootstrap_endpoints_b, + } + + +def _manifest_from_dict(d: dict) -> FederationManifest: + return FederationManifest( + schema_version=int(d["schema_version"]), + federation_id=d["federation_id"], + community_a_id=d["community_a_id"], + community_a_name=d.get("community_a_name", ""), + community_b_id=d["community_b_id"], + community_b_name=d.get("community_b_name", ""), + scope_a_to_b=_scope_from_dict(d["scope_a_to_b"]), + scope_b_to_a=_scope_from_dict(d["scope_b_to_a"]), + sig_a=d.get("sig_a", ""), + sig_b=d.get("sig_b", ""), + co_signers_a=list(d.get("co_signers_a", [])), + co_signers_b=list(d.get("co_signers_b", [])), + created_at=int(d["created_at"]), + expires_at=int(d["expires_at"]), + bootstrap_endpoints_a=list(d.get("bootstrap_endpoints_a", [])), + bootstrap_endpoints_b=list(d.get("bootstrap_endpoints_b", [])), + ) + + +def _proposal_to_dict(p: FederationProposal) -> dict: + return { + "community_a": p.community_a, + "community_b": p.community_b, + "scope_a": _scope_to_dict(p.scope_a), + "scope_b": _scope_to_dict(p.scope_b), + "bootstrap_a": p.bootstrap_a, + "bootstrap_b": p.bootstrap_b, + "proposed_at": p.proposed_at, + "proposer_sig": p.proposer_sig, + } + + +def _proposal_from_dict(d: dict) -> FederationProposal: + return FederationProposal( + community_a=d["community_a"], + community_b=d["community_b"], + scope_a=_scope_from_dict(d["scope_a"]), + scope_b=_scope_from_dict(d["scope_b"]), + bootstrap_a=list(d.get("bootstrap_a", [])), + bootstrap_b=list(d.get("bootstrap_b", [])), + proposed_at=int(d["proposed_at"]), + proposer_sig=d.get("proposer_sig", ""), + ) + + +# --------------------------------------------------------------------------- +# FederationStore (SQLite) +# --------------------------------------------------------------------------- + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS federation_manifests ( + federation_id TEXT PRIMARY KEY, + community_id TEXT NOT NULL, + manifest_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_fed_community ON federation_manifests (community_id); +""" + + +class FederationStore: + """SQLite-backed store for active federation manifests.""" + + def __init__(self, db_path: str | Path = ":memory:") -> None: + self._db_path = str(db_path) + self._conn = sqlite3.connect(self._db_path, check_same_thread=False) + self._conn.executescript(_SCHEMA) + self._conn.commit() + + def close(self) -> None: + self._conn.close() + + def add_manifest(self, m: FederationManifest) -> None: + """Insert or replace a federation manifest.""" + data = json.dumps(_manifest_to_dict(m)) + with self._conn: + self._conn.execute( + "INSERT OR REPLACE INTO federation_manifests " + "(federation_id, community_id, manifest_json, created_at, expires_at) " + "VALUES (?, ?, ?, ?, ?)", + (m.federation_id, m.community_b_id, data, m.created_at, m.expires_at), + ) + # Also index by community_a so lookups from either side work + self._conn.execute( + "INSERT OR REPLACE INTO federation_manifests " + "(federation_id, community_id, manifest_json, created_at, expires_at) " + "VALUES (?, ?, ?, ?, ?)", + ( + m.federation_id + "_rev", + m.community_a_id, + data, + m.created_at, + m.expires_at, + ), + ) + + def get_manifest(self, community_id: str) -> FederationManifest | None: + """Return the most recent active manifest for a community, or None.""" + import time + + now = int(time.time()) + row = self._conn.execute( + "SELECT manifest_json FROM federation_manifests " + "WHERE community_id = ? AND expires_at > ? " + "ORDER BY created_at DESC LIMIT 1", + (community_id, now), + ).fetchone() + if row is None: + return None + return _manifest_from_dict(json.loads(row[0])) + + def list_active(self) -> list[FederationManifest]: + """Return all non-expired manifests (deduplicated by federation_id).""" + import time + + now = int(time.time()) + rows = self._conn.execute( + "SELECT DISTINCT manifest_json FROM federation_manifests WHERE expires_at > ?", + (now,), + ).fetchall() + seen: set[str] = set() + result: list[FederationManifest] = [] + for (row,) in rows: + m = _manifest_from_dict(json.loads(row)) + if m.federation_id not in seen: + seen.add(m.federation_id) + result.append(m) + return result + + def remove(self, federation_id: str) -> None: + """Remove a federation manifest by its ID.""" + with self._conn: + self._conn.execute( + "DELETE FROM federation_manifests WHERE federation_id = ? OR federation_id = ?", + (federation_id, federation_id + "_rev"), + ) + + +# --------------------------------------------------------------------------- +# PeeringClient β€” HTTP handshake helpers +# --------------------------------------------------------------------------- + + +class PeeringClient: + """HTTP client for cross-community federation handshake. + + Uses the injected http_client which must implement .post(url, json=...) -> dict. + """ + + def __init__(self, http_client: Any) -> None: + self._http = http_client + + def propose(self, remote_url: str, proposal: FederationProposal) -> FederationProposal: + """Send a federation proposal to a peer community. + + Returns the peer's (possibly updated) proposal echo or raises ManifestError. + """ + endpoint = remote_url.rstrip("/") + "/federation/propose" + body = _proposal_to_dict(proposal) + try: + resp = self._http.post(endpoint, json=body) + except Exception as exc: + raise ManifestError(f"Peering propose failed: {exc}") from exc + if isinstance(resp, dict) and "error" in resp: + raise ManifestError(f"Remote rejected proposal: {resp['error']}") + return _proposal_from_dict(resp) + + def co_sign( + self, + remote_url: str, + proposal: FederationProposal, + keypair: Any, + role: str, + ) -> FederationManifest: + """Co-sign a proposal and submit it to produce a finalized manifest.""" + cosig = co_sign_federation(proposal, keypair, role) + endpoint = remote_url.rstrip("/") + "/federation/cosign" + body = { + "proposal": _proposal_to_dict(proposal), + "co_sig": cosig, + } + try: + resp = self._http.post(endpoint, json=body) + except Exception as exc: + raise ManifestError(f"Peering co-sign failed: {exc}") from exc + if isinstance(resp, dict) and "error" in resp: + raise ManifestError(f"Remote rejected co-sign: {resp['error']}") + return _manifest_from_dict(resp) diff --git a/hearthnet/federation/service.py b/hearthnet/federation/service.py new file mode 100644 index 0000000000000000000000000000000000000000..9712b30efbd1bfd7aae27d8e357742ef4590da39 --- /dev/null +++ b/hearthnet/federation/service.py @@ -0,0 +1,164 @@ +"""FederationService β€” registers federation.* capabilities on the bus (M14).""" +from __future__ import annotations + +from typing import Any + +from hearthnet.federation.manifest import ( + FederationManifest, + ManifestError, + finalize_federation_manifest, +) +from hearthnet.federation.peering import ( + FederationStore, + _proposal_from_dict, +) + + +class FederationService: + """Manages bilateral community federation. + + Registers: + federation.peer.list@1.0 + federation.peer.add@1.0 + federation.peer.remove@1.0 + """ + + name = "federation" + + def __init__( + self, + keypair: Any, + community_manifest: Any | None = None, + store: FederationStore | None = None, + bus: Any | None = None, + ) -> None: + self._kp = keypair + self._community_manifest = community_manifest + self._store = store or FederationStore() + self._bus = bus + + # ------------------------------------------------------------------ + # Registration + # ------------------------------------------------------------------ + + def register(self, bus: Any) -> None: + """Register all federation capabilities with the bus Registry.""" + from hearthnet.bus.capability import CapabilityDescriptor + + self._bus = bus + registry = getattr(bus, "registry", None) + if registry is None: + return + + descriptors = [ + ("federation.peer.list", "1.0", self._handle_list), + ("federation.peer.add", "1.0", self._handle_add), + ("federation.peer.remove", "1.0", self._handle_remove), + ] + for name, version, handler in descriptors: + desc = CapabilityDescriptor( + name=name, + version=version, + stability="stable", + params={}, + max_concurrent=2, + ) + registry.register_local(desc, handler) + + # ------------------------------------------------------------------ + # Handlers + # ------------------------------------------------------------------ + + def _handle_list(self, params: dict) -> dict: + """federation.peer.list@1.0 β€” list active federation peers. + + returns: {peers: list[{community_id, community_name, scope, expires_at}]} + """ + manifests = self._store.list_active() + our_community_id = getattr(self._community_manifest, "community_id", "") + peers = [] + for m in manifests: + # Determine which side we are to pick the correct scope + if m.community_a_id == our_community_id: + peer_id = m.community_b_id + peer_name = m.community_b_name + scope = m.scope_b_to_a # scope they grant us + else: + peer_id = m.community_a_id + peer_name = m.community_a_name + scope = m.scope_a_to_b + peers.append({ + "community_id": peer_id, + "community_name": peer_name, + "federation_id": m.federation_id, + "scope": { + "capabilities": list(scope.capabilities), + "data_visibility": scope.data_visibility, + }, + "expires_at": m.expires_at, + }) + return {"peers": peers} + + def _handle_add(self, params: dict) -> dict: + """federation.peer.add@1.0 β€” accept a signed proposal + co-sig and activate. + + params: {proposal_json: str, co_sig_json: str, + community_a_name?: str, community_b_name?: str} + returns: {federation_id: str, active: bool} + """ + import json as _json + + try: + proposal_dict = _json.loads(params.get("proposal_json", "{}")) + co_sig_dict = _json.loads(params.get("co_sig_json", "{}")) + except Exception as exc: + return {"error": f"JSON parse error: {exc}", "active": False, "federation_id": ""} + + try: + proposal = _proposal_from_dict(proposal_dict) + sig_a = proposal.proposer_sig + sig_b = co_sig_dict.get("signature", "") + community_a_name = params.get("community_a_name", "") + community_b_name = params.get("community_b_name", "") + manifest = finalize_federation_manifest( + proposal, + sig_a=sig_a, + sig_b=sig_b, + community_a_name=community_a_name, + community_b_name=community_b_name, + ) + self._store.add_manifest(manifest) + return {"federation_id": manifest.federation_id, "active": True} + except ManifestError as exc: + return {"error": str(exc), "active": False, "federation_id": ""} + + def _handle_remove(self, params: dict) -> dict: + """federation.peer.remove@1.0 β€” deactivate federation with a community. + + params: {community_id: str} + returns: {removed: bool} + """ + community_id = params.get("community_id", "") + if not community_id: + return {"removed": False, "error": "community_id required"} + m = self._store.get_manifest(community_id) + if m is None: + return {"removed": False} + self._store.remove(m.federation_id) + return {"removed": True} + + # ------------------------------------------------------------------ + # Direct API + # ------------------------------------------------------------------ + + def add_manifest(self, manifest: FederationManifest) -> None: + """Directly add a finalized manifest (bypasses the bus).""" + self._store.add_manifest(manifest) + + def get_peer(self, community_id: str) -> FederationManifest | None: + """Return the active manifest for a peer community, or None.""" + return self._store.get_manifest(community_id) + + def list_peers(self) -> list[FederationManifest]: + """Return all active federation manifests.""" + return self._store.list_active() diff --git a/hearthnet/fedlearn/__init__.py b/hearthnet/fedlearn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d98f5f50a1186ea1b7d1c5c12363d20712f730a6 --- /dev/null +++ b/hearthnet/fedlearn/__init__.py @@ -0,0 +1,6 @@ +"""M28 β€” Federated Learning package (experimental, Phase 3).""" +from __future__ import annotations + +from hearthnet.fedlearn.coordinator import FedLearnCoordinator, RoundManifest, ParticipantSubmission + +__all__ = ["FedLearnCoordinator", "RoundManifest", "ParticipantSubmission"] diff --git a/hearthnet/fedlearn/coordinator.py b/hearthnet/fedlearn/coordinator.py new file mode 100644 index 0000000000000000000000000000000000000000..10317108476b860c60302182f03d82717e1b2aca --- /dev/null +++ b/hearthnet/fedlearn/coordinator.py @@ -0,0 +1,109 @@ +"""M28 β€” Federated Learning / LoRA aggregation (experimental, Phase 3). + +FedAvg on LoRA adapter weight deltas. Each node trains locally; +only adapter deltas (not raw data or full weights) are shared. +Gated by config.research.federated_learning = True. +""" +from __future__ import annotations + +import time +import uuid +from dataclasses import dataclass, field +from typing import NewType + +RoundID = NewType("RoundID", str) + + +@dataclass(frozen=True) +class RoundManifest: + """Describes a federated learning round.""" + round_id: RoundID + base_model_id: str + coordinator_node_id: str + community_id: str + lora_rank: int = 16 + lora_alpha: float = 32.0 + learning_rate: float = 2e-4 + min_participants: int = 2 + max_participants: int = 20 + round_timeout_seconds: int = 3600 + created_at: float = field(default_factory=time.time) + coordinator_sig: bytes = b"" + + +@dataclass +class ParticipantSubmission: + round_id: RoundID + participant_node_id: str + delta_bytes: bytes # serialised LoRA state dict subset (safetensors format) + num_samples: int + submitted_at: float = field(default_factory=time.time) + participant_sig: bytes = b"" + + +class FedLearnCoordinator: + """Orchestrates a federated learning round. + + Experimental. Requires peft and torch. + Only active when config.research.federated_learning = True. + """ + + def __init__(self, keypair=None, bus=None) -> None: + self._keypair = keypair + self._bus = bus + self._rounds: dict[RoundID, RoundManifest] = {} + self._submissions: dict[RoundID, list[ParticipantSubmission]] = {} + + def create_round( + self, + base_model_id: str, + community_id: str, + **kwargs, + ) -> RoundManifest: + """Create a new federated learning round manifest.""" + round_id = RoundID(str(uuid.uuid4())) + manifest = RoundManifest( + round_id=round_id, + base_model_id=base_model_id, + coordinator_node_id=getattr(self._keypair, "node_id_short", "unknown"), + community_id=community_id, + **kwargs, + ) + self._rounds[round_id] = manifest + self._submissions[round_id] = [] + return manifest + + def submit(self, submission: ParticipantSubmission) -> bool: + """Accept a participant's LoRA delta submission.""" + if submission.round_id not in self._rounds: + return False + self._submissions[submission.round_id].append(submission) + return True + + def aggregate(self, round_id: RoundID) -> bytes | None: + """FedAvg: weighted average of submitted LoRA deltas. + + Returns aggregated delta bytes or None if not enough participants. + Raises NotImplementedError β€” actual aggregation requires peft+torch. + """ + subs = self._submissions.get(round_id, []) + manifest = self._rounds.get(round_id) + if manifest is None or len(subs) < manifest.min_participants: + return None + raise NotImplementedError( + "FedLearnCoordinator.aggregate() requires peft and torch. " + "This is an experimental Phase 3 feature (M28)." + ) + + def round_status(self, round_id: RoundID) -> dict: + manifest = self._rounds.get(round_id) + if manifest is None: + return {"error": "not_found"} + subs = self._submissions.get(round_id, []) + return { + "round_id": round_id, + "base_model_id": manifest.base_model_id, + "participants": len(subs), + "min_required": manifest.min_participants, + "ready_to_aggregate": len(subs) >= manifest.min_participants, + } diff --git a/hearthnet/identity/tokens.py b/hearthnet/identity/tokens.py index cdf11911480c58c4cc917ce5990025e325322b78..7b21619bf7451fc976f5e4e824671d01c2e9f702 100644 --- a/hearthnet/identity/tokens.py +++ b/hearthnet/identity/tokens.py @@ -1,42 +1,238 @@ -from __future__ import annotations - -"""tokens.py β€” Phase 2 stub for capability tokens. +"""hearthnet/identity/tokens.py β€” Capability tokens (M16, Phase 2). -All functions raise NotImplementedError. Implement in Phase 2. +Token format: hntoken://v1/.. """ +from __future__ import annotations +import base64 +import json +import time from dataclasses import dataclass from typing import Any +try: + import nacl.public # noqa: F401 β€” presence check only + + _NACL_AVAILABLE = True +except ImportError: # pragma: no cover + _NACL_AVAILABLE = False + + +class TokenError(Exception): + """Raised for all token-layer failures.""" + + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class TokenScope: + """Scope granted by a capability token.""" + + capabilities: list[str] + max_uses: int | None = None + max_calls_total: int | None = None + @dataclass(frozen=True) class CapabilityToken: - """Stub for a signed capability token (Phase 2).""" + """A signed Ed25519 capability token.""" + + iss: str # issuer node_id (full form "ed25519:…") + sub: str # subject node_id or "*" for bearer token + aud: str # audience community_id or "" + iat: int # issued-at unix seconds + exp: int # expires-at unix seconds + nbf: int # not-before unix seconds + scope: TokenScope + jti: str # unique token ID (ULID) + issued_via: str # "federation"|"onboarding"|"manual"|"relay" + + +# --------------------------------------------------------------------------- +# Encoding helpers +# --------------------------------------------------------------------------- + +_TOKEN_SCHEME = "hntoken://v1/" + +_HEADER = json.dumps({"alg": "EdDSA", "typ": "hntoken", "v": 1}, separators=(",", ":")) +_HEADER_B64 = base64.urlsafe_b64encode(_HEADER.encode()).rstrip(b"=").decode("ascii") + + +def _b64url_encode(b: bytes) -> str: + return base64.urlsafe_b64encode(b).rstrip(b"=").decode("ascii") + + +def _b64url_decode(s: str) -> bytes: + pad = 4 - len(s) % 4 + if pad != 4: + s += "=" * pad + return base64.urlsafe_b64decode(s) - token_id: str - issuer_node_id: str - subject_node_id: str - capability: str - issued_at: str - expires_at: str - signature: str + +def _scope_to_dict(scope: TokenScope) -> dict[str, Any]: + return { + "capabilities": list(scope.capabilities), + "max_uses": scope.max_uses, + "max_calls_total": scope.max_calls_total, + } + + +def _scope_from_dict(d: dict[str, Any]) -> TokenScope: + return TokenScope( + capabilities=list(d.get("capabilities", [])), + max_uses=d.get("max_uses"), + max_calls_total=d.get("max_calls_total"), + ) + + +def _payload_to_dict(tok: CapabilityToken) -> dict[str, Any]: + return { + "iss": tok.iss, + "sub": tok.sub, + "aud": tok.aud, + "iat": tok.iat, + "exp": tok.exp, + "nbf": tok.nbf, + "scope": _scope_to_dict(tok.scope), + "jti": tok.jti, + "issued_via": tok.issued_via, + } + + +# --------------------------------------------------------------------------- +# Issue +# --------------------------------------------------------------------------- def issue_token( issuer_kp: Any, subject_node_id: str, - capability: str, - ttl_seconds: int = 300, -) -> CapabilityToken: - """Issue a signed capability token. Phase 2 β€” not yet implemented.""" - raise NotImplementedError("CapabilityToken issuance is a Phase 2 feature.") + audience: str, + scope: TokenScope, + ttl_seconds: int = 3600, + issued_via: str = "manual", +) -> tuple[CapabilityToken, str]: + """Issue a signed capability token. + + Returns (CapabilityToken, encoded_token_string). + """ + if not _NACL_AVAILABLE: + raise TokenError("PyNaCl is required for token issuance. Install pynacl.") + + from hearthnet.events.types import new_ulid + from hearthnet.identity.keys import full_node_id + + now = int(time.time()) + jti = new_ulid() + iss = full_node_id(bytes(issuer_kp.verify_key)) + + tok = CapabilityToken( + iss=iss, + sub=subject_node_id, + aud=audience, + iat=now, + exp=now + ttl_seconds, + nbf=now, + scope=scope, + jti=jti, + issued_via=issued_via, + ) + payload_bytes = json.dumps(_payload_to_dict(tok), separators=(",", ":")).encode() + payload_b64 = _b64url_encode(payload_bytes) + signing_input = f"{_HEADER_B64}.{payload_b64}".encode() + + try: + signed = issuer_kp.signing_key.sign(signing_input) + sig_b64 = _b64url_encode(signed.signature) + except Exception as exc: + raise TokenError(f"Signing failed: {exc}") from exc + + encoded = encode_token(tok, sig_b64) + return tok, encoded + + +def encode_token(tok: CapabilityToken, signature_b64: str) -> str: + """Encode a CapabilityToken + pre-computed signature to the hntoken:// string.""" + payload_bytes = json.dumps(_payload_to_dict(tok), separators=(",", ":")).encode() + payload_b64 = _b64url_encode(payload_bytes) + return f"{_TOKEN_SCHEME}{_HEADER_B64}.{payload_b64}.{signature_b64}" + + +# --------------------------------------------------------------------------- +# Decode (structural only, no sig verify) +# --------------------------------------------------------------------------- + + +def decode_token(text: str) -> CapabilityToken: + """Parse an hntoken:// string. Validates structure; does NOT verify the signature.""" + if not text.startswith(_TOKEN_SCHEME): + raise TokenError(f"Not a HearthNet token (expected 'hntoken://v1/'): {text[:40]!r}") + body = text[len(_TOKEN_SCHEME):] + parts = body.split(".") + if len(parts) != 3: + raise TokenError("Token must have exactly 3 dot-separated parts") + _header_b64, payload_b64, _sig_b64 = parts + try: + payload_bytes = _b64url_decode(payload_b64) + pd = json.loads(payload_bytes) + except Exception as exc: + raise TokenError(f"Failed to decode token payload: {exc}") from exc + + required = {"iss", "sub", "aud", "iat", "exp", "nbf", "scope", "jti", "issued_via"} + missing = required - pd.keys() + if missing: + raise TokenError(f"Token payload missing fields: {missing}") + + try: + scope = _scope_from_dict(pd["scope"]) + return CapabilityToken( + iss=pd["iss"], + sub=pd["sub"], + aud=pd["aud"], + iat=int(pd["iat"]), + exp=int(pd["exp"]), + nbf=int(pd["nbf"]), + scope=scope, + jti=pd["jti"], + issued_via=pd["issued_via"], + ) + except (KeyError, TypeError, ValueError) as exc: + raise TokenError(f"Malformed token payload: {exc}") from exc + +# --------------------------------------------------------------------------- +# Verify +# --------------------------------------------------------------------------- -def verify_token(token: CapabilityToken) -> bool: - """Verify a capability token's signature and expiry. Phase 2 β€” not yet implemented.""" - raise NotImplementedError("CapabilityToken verification is a Phase 2 feature.") +def verify_token( + tok: CapabilityToken, + community_manifest: Any | None = None, + now: int | None = None, +) -> None: + """Verify token validity (expiry, not-before). Raises TokenError on failure. -def revoke_token(token_id: str) -> None: - """Revoke a capability token by ID. Phase 2 β€” not yet implemented.""" - raise NotImplementedError("CapabilityToken revocation is a Phase 2 feature.") + If community_manifest is provided, the issuer is checked against membership. + """ + ts = now if now is not None else int(time.time()) + if ts < tok.nbf: + raise TokenError(f"Token not yet valid (nbf={tok.nbf}, now={ts})") + if ts >= tok.exp: + raise TokenError(f"Token expired (exp={tok.exp}, now={ts})") + if not tok.iss.startswith("ed25519:"): + raise TokenError(f"Issuer must be a full node_id: {tok.iss!r}") + if not tok.jti: + raise TokenError("Token has no jti") + if community_manifest is not None: + members = getattr(community_manifest, "members", None) or {} + if isinstance(members, dict): + member_ids = set(members.keys()) + elif isinstance(members, list): + member_ids = set(members) + else: + member_ids = set() + if member_ids and tok.iss not in member_ids: + raise TokenError(f"Token issuer {tok.iss!r} is not a member of the community") diff --git a/hearthnet/lora/__init__.py b/hearthnet/lora/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2f3e7fdd750c4e1bfda561be7e9e789fa77657d2 --- /dev/null +++ b/hearthnet/lora/__init__.py @@ -0,0 +1,6 @@ +"""LoRa hardware beacons package (experimental, Phase 3 β€” M29).""" +from __future__ import annotations + +from hearthnet.lora.service import LoraBeacon, LoraBeaconService, decode_beacon_frame, encode_beacon_frame + +__all__ = ["LoraBeacon", "LoraBeaconService", "decode_beacon_frame", "encode_beacon_frame"] diff --git a/hearthnet/lora/service.py b/hearthnet/lora/service.py new file mode 100644 index 0000000000000000000000000000000000000000..ae45878bccb138b75a7d948a6f03b8602650e22e --- /dev/null +++ b/hearthnet/lora/service.py @@ -0,0 +1,119 @@ +"""M29 β€” LoRa Hardware Beacons (experimental, Phase 3). + +868 MHz LoRa "I'm still here" beacons for offline emergency presence. +No AI traffic, no chat, no file transfer β€” only 32-byte heartbeat frames. +Gated by config.research.lora_beacons = True. +""" +from __future__ import annotations + +import struct +import time +from dataclasses import dataclass, field +from typing import NewType + +LoraBeaconID = NewType("LoraBeaconID", str) +LoraDeviceID = NewType("LoraDeviceID", str) + +# Frame layout: 32 bytes +# [0:4] magic (b"HN01") +# [4:8] sequence (uint32 big-endian) +# [8:16] node_id_hash (first 8 bytes of SHA-256 of node_id_full) +# [16:17] flags (bit0=emergency, bit1=panic) +# [17:32] reserved (zeros) +FRAME_MAGIC = b"HN01" +FRAME_SIZE = 32 + + +@dataclass(frozen=True) +class LoraBeacon: + beacon_id: LoraBeaconID + device_id: LoraDeviceID + node_id_hash: bytes # 8 bytes + sequence: int + flags: int # bit0=emergency, bit1=panic + rssi: int | None = None # dBm, if available + received_at: float = field(default_factory=time.time) + + @property + def is_emergency(self) -> bool: + return bool(self.flags & 0x01) + + @property + def is_panic(self) -> bool: + return bool(self.flags & 0x02) + + +def encode_beacon_frame(node_id_full: str, sequence: int, flags: int = 0) -> bytes: + """Encode a 32-byte LoRa beacon frame.""" + import hashlib + node_hash = hashlib.sha256(node_id_full.encode()).digest()[:8] + header = struct.pack(">4sI8sB", FRAME_MAGIC, sequence, node_hash, flags) + return header + b"\x00" * (FRAME_SIZE - len(header)) + + +def decode_beacon_frame(raw: bytes, device_id: str = "unknown") -> LoraBeacon | None: + """Decode a 32-byte LoRa frame. Returns None if invalid.""" + if len(raw) < FRAME_SIZE: + return None + magic, sequence, node_hash, flags = struct.unpack_from(">4sI8sB", raw) + if magic != FRAME_MAGIC: + return None + return LoraBeacon( + beacon_id=LoraBeaconID(f"{device_id}:{sequence}"), + device_id=LoraDeviceID(device_id), + node_id_hash=node_hash, + sequence=sequence, + flags=flags, + ) + + +class LoraBeaconService: + """Sends and receives LoRa beacons. + + Requires a USB LoRa stick (RFM95W, sx1276, sx1262 via serial bridge). + Falls back to simulation mode if no hardware detected. + Only active when config.research.lora_beacons = True. + """ + + def __init__(self, serial_port: str | None = None, node_id_full: str = "") -> None: + self._serial_port = serial_port + self._node_id_full = node_id_full + self._sequence = 0 + self._received: list[LoraBeacon] = [] + self._simulated = serial_port is None + + def send_heartbeat(self, flags: int = 0) -> bytes: + """Encode and (if hardware present) transmit a heartbeat frame.""" + frame = encode_beacon_frame(self._node_id_full, self._sequence, flags) + self._sequence += 1 + if not self._simulated: + self._transmit(frame) + return frame + + def _transmit(self, frame: bytes) -> None: + """Write frame to serial LoRa hardware (stub β€” real impl needs pyserial).""" + try: + import serial # type: ignore[import-untyped] + with serial.Serial(self._serial_port, baudrate=9600, timeout=1) as ser: + ser.write(frame) + except ImportError: + pass # pyserial not installed β€” silently skip + + def receive_frame(self, raw: bytes, device_id: str = "unknown") -> LoraBeacon | None: + """Decode an incoming frame and record it.""" + beacon = decode_beacon_frame(raw, device_id) + if beacon is not None: + self._received.append(beacon) + return beacon + + def recent_beacons(self, window_seconds: float = 300.0) -> list[LoraBeacon]: + cutoff = time.time() - window_seconds + return [b for b in self._received if b.received_at >= cutoff] + + def health(self) -> dict: + return { + "hardware": "detected" if not self._simulated else "simulated", + "serial_port": self._serial_port, + "sent": self._sequence, + "received": len(self._received), + } diff --git a/hearthnet/mobile/__init__.py b/hearthnet/mobile/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..042a58c3c0f6c82c4732cc9166907cc8987ca11e --- /dev/null +++ b/hearthnet/mobile/__init__.py @@ -0,0 +1,9 @@ +"""M22 β€” Mobile Native β€” Python anchor-side helpers. + +The Flutter mobile app lives in a separate repo (``mobile-native/``). +This package provides the Python-side services that a community anchor +exposes specifically for mobile clients: + +- ``invite.py`` β€” mobile-targeted invite QR / deep-link generation +- ``push_authority.py`` β€” push token registry + notification dispatch +""" diff --git a/hearthnet/mobile/invite.py b/hearthnet/mobile/invite.py new file mode 100644 index 0000000000000000000000000000000000000000..a52845b27bd14655d6bf36978e4f573e557b308b --- /dev/null +++ b/hearthnet/mobile/invite.py @@ -0,0 +1,168 @@ +"""M22 β€” Mobile invite helpers. + +Generates mobile-targeted invite deep links (``hnapp://``) and QR codes +for the mobile native client (Flutter). Builds on top of the Phase 1 +onboarding module (M13). +""" +from __future__ import annotations + +import base64 +import hashlib +import json +import time +from dataclasses import dataclass, field +from typing import Any + + +# --------------------------------------------------------------------------- +# Invite blob for mobile clients +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class MobileInviteBlob: + """Compact invite that the mobile app can parse from a QR or deep link. + + Wire format: ``hnapp://v1/`` + The payload is JSON with fields defined below. + """ + + community_id: str + community_name: str + anchor_endpoints: list[str] + """HTTP(S) or WebSocket URLs of community anchors the app can reach.""" + + invited_by: str + """node_id of the inviting user (display hint only).""" + + relay_url: str | None = None + """Optional relay tier URL for NAT-traversal push delivery (M15).""" + + invite_token: str | None = None + """One-time capability token the app exchanges on first contact (M16).""" + + created_at: float = field(default_factory=time.time) + expires_at: float | None = None + + def is_expired(self, now: float | None = None) -> bool: + t = now if now is not None else time.time() + return self.expires_at is not None and t > self.expires_at + + # ------------------------------------------------------------------ + # Serialization + # ------------------------------------------------------------------ + + def to_dict(self) -> dict: + return { + "v": 1, + "community_id": self.community_id, + "community_name": self.community_name, + "anchor_endpoints": self.anchor_endpoints, + "invited_by": self.invited_by, + "relay_url": self.relay_url, + "invite_token": self.invite_token, + "created_at": self.created_at, + "expires_at": self.expires_at, + } + + def to_deep_link(self) -> str: + """Encode as ``hnapp://v1/``.""" + payload = json.dumps(self.to_dict(), separators=(",", ":"), sort_keys=True) + b64 = base64.urlsafe_b64encode(payload.encode()).rstrip(b"=").decode() + return f"hnapp://v1/{b64}" + + def fingerprint(self) -> str: + """SHA-256 (hex, 16 chars) of the JSON payload for logging.""" + raw = json.dumps(self.to_dict(), separators=(",", ":"), sort_keys=True) + return hashlib.sha256(raw.encode()).hexdigest()[:16] + + @classmethod + def from_deep_link(cls, deep_link: str) -> "MobileInviteBlob": + """Parse a deep link produced by :meth:`to_deep_link`.""" + if not deep_link.startswith("hnapp://v1/"): + raise ValueError(f"Not a valid hnapp:// deep link: {deep_link!r}") + b64 = deep_link[len("hnapp://v1/"):] + # Re-add padding + padding = 4 - len(b64) % 4 + if padding != 4: + b64 += "=" * padding + payload = json.loads(base64.urlsafe_b64decode(b64).decode()) + return cls( + community_id=payload["community_id"], + community_name=payload["community_name"], + anchor_endpoints=payload["anchor_endpoints"], + invited_by=payload["invited_by"], + relay_url=payload.get("relay_url"), + invite_token=payload.get("invite_token"), + created_at=payload.get("created_at", time.time()), + expires_at=payload.get("expires_at"), + ) + + +# --------------------------------------------------------------------------- +# QR code rendering (qrcode optional) +# --------------------------------------------------------------------------- + + +def render_qr_svg(blob: MobileInviteBlob) -> str | None: + """Return an SVG string for the invite QR code, or None if ``qrcode`` is + not installed. The SVG can be embedded directly in HTML.""" + try: + import qrcode # type: ignore + import qrcode.image.svg # type: ignore + + factory = qrcode.image.svg.SvgPathImage + qr = qrcode.make(blob.to_deep_link(), image_factory=factory) + import io + + buf = io.BytesIO() + qr.save(buf) + return buf.getvalue().decode("utf-8") + except ImportError: + return None + + +def render_qr_terminal(blob: MobileInviteBlob) -> str: + """Return the QR code as ASCII art (uses ``qrcode`` if available, else + falls back to the raw deep link).""" + try: + import qrcode # type: ignore + + qr = qrcode.QRCode() + qr.add_data(blob.to_deep_link()) + qr.make(fit=True) + import io + + buf = io.StringIO() + qr.print_ascii(out=buf) + return buf.getvalue() + except ImportError: + return blob.to_deep_link() + + +# --------------------------------------------------------------------------- +# Factory helper +# --------------------------------------------------------------------------- + + +def build_mobile_invite( + community_id: str, + community_name: str, + anchor_endpoints: list[str], + invited_by: str, + relay_url: str | None = None, + invite_token: str | None = None, + ttl_seconds: float = 86_400 * 7, # 7 days default +) -> MobileInviteBlob: + """Create a :class:`MobileInviteBlob` with a default 7-day TTL.""" + now = time.time() + return MobileInviteBlob( + community_id=community_id, + community_name=community_name, + anchor_endpoints=anchor_endpoints, + invited_by=invited_by, + relay_url=relay_url, + invite_token=invite_token, + created_at=now, + expires_at=now + ttl_seconds, + ) diff --git a/hearthnet/mobile/push_authority.py b/hearthnet/mobile/push_authority.py new file mode 100644 index 0000000000000000000000000000000000000000..3e23b0f38eb71b65015c9ef5fd106b387ff1b3dd --- /dev/null +++ b/hearthnet/mobile/push_authority.py @@ -0,0 +1,238 @@ +"""M22 β€” Mobile push token authority. + +The anchor-side service that mobile clients (Flutter) call to: +1. Register their APNs/FCM push token β†’ ``mobile.push.register@1.0`` +2. Deregister (logout / token rotation) β†’ ``mobile.push.deregister@1.0`` +3. Receive in-app notifications sent by other services + +Push delivery itself is *out of scope* for the local-first anchor β€” if the +relay tier (M15) is configured the anchor forwards the notification there. +""" +from __future__ import annotations + +import hashlib +import time +from dataclasses import dataclass, field +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from hearthnet.bus.router import Router + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass +class PushToken: + """A mobile push token registration entry.""" + + node_id: str + """The HearthNet node ID of the mobile device.""" + + token: str + """Raw APNs device token or FCM registration ID.""" + + platform: str + """``apns`` | ``fcm`` | ``simulator``.""" + + registered_at: float = field(default_factory=time.time) + last_seen_at: float = field(default_factory=time.time) + + def touch(self) -> None: + self.last_seen_at = time.time() + + @property + def token_hash(self) -> str: + """SHA-256 of the raw token (for logging without leaking the token).""" + return hashlib.sha256(self.token.encode()).hexdigest()[:12] + + +@dataclass(frozen=True) +class PushNotification: + """A notification to be delivered to one or more mobile devices.""" + + title: str + body: str + data: dict = field(default_factory=dict) + """Arbitrary key/value payload forwarded to the app.""" + + badge: int | None = None + """iOS badge count; ``None`` leaves the current badge unchanged.""" + + +# --------------------------------------------------------------------------- +# In-memory token store +# --------------------------------------------------------------------------- + + +class PushTokenRegistry: + """Thread-safe in-memory registry of push tokens. + + In production you would back this with the SQLite event store (M07 / X02) + or a dedicated table. For Phase 1 and 2 the in-memory approach is + sufficient for single-process deployments. + """ + + def __init__(self) -> None: + self._tokens: dict[str, list[PushToken]] = {} + """node_id β†’ list of tokens (a user may have multiple devices).""" + + def register(self, node_id: str, token: str, platform: str) -> PushToken: + entry = PushToken(node_id=node_id, token=token, platform=platform) + self._tokens.setdefault(node_id, []) + # Deduplicate by token value + existing = {t.token for t in self._tokens[node_id]} + if token not in existing: + self._tokens[node_id].append(entry) + else: + for t in self._tokens[node_id]: + if t.token == token: + t.touch() + entry = t + return entry + + def deregister(self, node_id: str, token: str) -> bool: + before = self._tokens.get(node_id, []) + after = [t for t in before if t.token != token] + self._tokens[node_id] = after + return len(after) < len(before) + + def get_tokens(self, node_id: str) -> list[PushToken]: + return list(self._tokens.get(node_id, [])) + + def all_node_ids(self) -> list[str]: + return list(self._tokens.keys()) + + +# --------------------------------------------------------------------------- +# Bus service +# --------------------------------------------------------------------------- + + +class MobilePushService: + """Registers as a bus capability provider for mobile push operations. + + Capabilities provided: + - ``mobile.push.register@1.0`` + - ``mobile.push.deregister@1.0`` + - ``mobile.push.notify@1.0`` (internal β€” other services call this) + """ + + def __init__( + self, + relay_url: str | None = None, + bus: "Router | None" = None, + ) -> None: + self._registry = PushTokenRegistry() + self._relay_url = relay_url + self._bus = bus + + # ------------------------------------------------------------------ + # Health + # ------------------------------------------------------------------ + + def health(self) -> dict: + total_tokens = sum(len(v) for v in self._registry._tokens.values()) + return { + "status": "ok", + "registered_devices": total_tokens, + "relay_url": self._relay_url, + } + + # ------------------------------------------------------------------ + # Handlers called by the bus router + # ------------------------------------------------------------------ + + def handle_register(self, params: dict) -> dict: + """``mobile.push.register@1.0`` handler. + + Expected input:: + + { + "node_id": "", + "token": "", + "platform": "apns" | "fcm" | "simulator" + } + """ + inp = params.get("input", params) + node_id = str(inp.get("node_id", "")) + token = str(inp.get("token", "")) + platform = str(inp.get("platform", "unknown")) + if not node_id or not token: + return {"output": {"error": "bad_request", "detail": "node_id and token required"}, "meta": {}} + entry = self._registry.register(node_id, token, platform) + return {"output": {"status": "registered", "token_hash": entry.token_hash}, "meta": {}} + + def handle_deregister(self, params: dict) -> dict: + """``mobile.push.deregister@1.0`` handler.""" + inp = params.get("input", params) + node_id = str(inp.get("node_id", "")) + token = str(inp.get("token", "")) + removed = self._registry.deregister(node_id, token) + return {"output": {"status": "removed" if removed else "not_found"}, "meta": {}} + + async def handle_notify(self, params: dict) -> dict: + """``mobile.push.notify@1.0`` β€” internal notification dispatch. + + Sends a :class:`PushNotification` to all registered devices of + ``target_node_id``. If a relay URL is configured, forwards there; + otherwise logs and returns ``{"status": "no_relay"}``. + """ + inp = params.get("input", params) + target = str(inp.get("target_node_id", "")) + notif = PushNotification( + title=str(inp.get("title", "")), + body=str(inp.get("body", "")), + data=inp.get("data", {}), + badge=inp.get("badge"), + ) + tokens = self._registry.get_tokens(target) + if not tokens: + return {"output": {"status": "no_tokens", "delivered": 0}, "meta": {}} + + if self._relay_url: + delivered = await self._forward_to_relay(target, tokens, notif) + return {"output": {"status": "forwarded", "delivered": delivered}, "meta": {}} + + # No relay configured β€” local delivery only (simulator / test) + return {"output": {"status": "no_relay", "delivered": 0}, "meta": {}} + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + async def _forward_to_relay( + self, + target_node_id: str, + tokens: list[PushToken], + notif: PushNotification, + ) -> int: + """POST the notification to the relay tier (M15) for each token.""" + try: + import httpx # type: ignore + except ImportError: + return 0 + + payload = { + "target_node_id": target_node_id, + "tokens": [{"token": t.token, "platform": t.platform} for t in tokens], + "notification": { + "title": notif.title, + "body": notif.body, + "data": notif.data, + "badge": notif.badge, + }, + } + delivered = 0 + async with httpx.AsyncClient(timeout=10) as client: + try: + resp = await client.post( + f"{self._relay_url}/push/v1/send", + json=payload, + ) + if resp.status_code == 200: + delivered = len(tokens) + except httpx.HTTPError: + pass + return delivered diff --git a/hearthnet/moe/__init__.py b/hearthnet/moe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b479374fbc667a21a7e473624da2875818eb17a3 --- /dev/null +++ b/hearthnet/moe/__init__.py @@ -0,0 +1,6 @@ +"""M27 β€” MoE Expert Routing package (experimental, Phase 3).""" +from __future__ import annotations + +from hearthnet.moe.router import ExpertDescriptor, ExpertRegistry, MoeRouter, RouteResult + +__all__ = ["ExpertDescriptor", "ExpertRegistry", "MoeRouter", "RouteResult"] diff --git a/hearthnet/moe/router.py b/hearthnet/moe/router.py new file mode 100644 index 0000000000000000000000000000000000000000..a112c1eb25695036d450554efe0862a1e2e22ee7 --- /dev/null +++ b/hearthnet/moe/router.py @@ -0,0 +1,154 @@ +"""M27 β€” MoE Expert Routing (experimental, Phase 3). + +Routes queries to the best expert: local model, service capability, human, or external. +Gated by config.research.moe_routing = True. +""" +from __future__ import annotations + +import time +import uuid +from dataclasses import dataclass, field +from typing import Any + +ExpertID = str +ThreadID = str + + +@dataclass(frozen=True) +class ExpertDescriptor: + """Describes an available expert (model, service, human, or external).""" + expert_id: ExpertID # "human:" | "model:" | "service:" | "external:" + expert_type: str # "human" | "model" | "service" | "external" + topic_tags: frozenset[str] + confidence_score: float # 0.0-1.0, self-reported + community_id: str + name: str | None = None + description: str | None = None + expires_at: float | None = None + + def is_expired(self, now: float | None = None) -> bool: + if self.expires_at is None: + return False + return (now or time.time()) > self.expires_at + + +@dataclass +class RouteCandidate: + expert_id: ExpertID + score: float + reason: str + expert_type: str + name: str | None = None + + +@dataclass +class RouteResult: + candidates: list[RouteCandidate] + query_summary: str + routed_at: float = field(default_factory=time.time) + + +@dataclass +class Handoff: + """A pending handoff to a human expert.""" + handoff_id: str + expert_id: ExpertID + query: str + created_at: float = field(default_factory=time.time) + resolved_at: float | None = None + status: str = "pending" # "pending" | "accepted" | "declined" | "timeout" + thread_id: ThreadID | None = None + + +class ExpertRegistry: + """Tracks registered experts and their declared topics.""" + + def __init__(self) -> None: + self._experts: dict[ExpertID, ExpertDescriptor] = {} + + def register(self, descriptor: ExpertDescriptor) -> None: + self._experts[descriptor.expert_id] = descriptor + + def unregister(self, expert_id: ExpertID) -> bool: + if expert_id in self._experts: + del self._experts[expert_id] + return True + return False + + def list_active(self, now: float | None = None) -> list[ExpertDescriptor]: + now = now or time.time() + return [e for e in self._experts.values() if not e.is_expired(now)] + + def find_by_tags(self, tags: set[str], now: float | None = None) -> list[ExpertDescriptor]: + active = self.list_active(now) + return [e for e in active if e.topic_tags & tags] + + +class MoeRouter: + """Recommends experts for a query. + + Uses a simple rule-based scorer in Phase 3. + A learned scorer (embedding-based) is planned but not yet implemented. + Only active when config.research.moe_routing = True. + """ + + def __init__(self, registry: ExpertRegistry | None = None, bus=None) -> None: + self._registry = registry or ExpertRegistry() + self._bus = bus + self._pending_handoffs: dict[str, Handoff] = {} + + @property + def registry(self) -> ExpertRegistry: + return self._registry + + def route(self, query: str, top_k: int = 3, tags: set[str] | None = None) -> RouteResult: + """Return top-K expert candidates for a query.""" + candidates_src = self._registry.list_active() + if tags: + candidates_src = [e for e in candidates_src if e.topic_tags & tags] + + # Simple scoring: exact tag matches + confidence weight + query_words = set(query.lower().split()) + scored: list[RouteCandidate] = [] + for expert in candidates_src: + tag_overlap = len(expert.topic_tags & query_words) + score = expert.confidence_score * (1.0 + 0.2 * tag_overlap) + scored.append(RouteCandidate( + expert_id=expert.expert_id, + score=min(score, 1.0), + reason=f"tag_overlap={tag_overlap}, confidence={expert.confidence_score:.2f}", + expert_type=expert.expert_type, + name=expert.name, + )) + + scored.sort(key=lambda c: c.score, reverse=True) + return RouteResult( + candidates=scored[:top_k], + query_summary=query[:200], + ) + + def initiate_handoff(self, expert_id: ExpertID, query: str, thread_id: str | None = None) -> Handoff: + """Create a pending handoff to a human expert.""" + h = Handoff( + handoff_id=str(uuid.uuid4()), + expert_id=expert_id, + query=query, + thread_id=thread_id, + ) + self._pending_handoffs[h.handoff_id] = h + return h + + def resolve_handoff(self, handoff_id: str, status: str) -> bool: + if handoff_id in self._pending_handoffs: + h = self._pending_handoffs[handoff_id] + self._pending_handoffs[handoff_id] = Handoff( + handoff_id=h.handoff_id, + expert_id=h.expert_id, + query=h.query, + created_at=h.created_at, + resolved_at=time.time(), + status=status, + thread_id=h.thread_id, + ) + return True + return False diff --git a/hearthnet/observability/federated.py b/hearthnet/observability/federated.py new file mode 100644 index 0000000000000000000000000000000000000000..40fd0b3c2e85fcac1567e18cd20cb66dbeecbe6a --- /dev/null +++ b/hearthnet/observability/federated.py @@ -0,0 +1,298 @@ +"""Federated metrics aggregation (X07).""" +from __future__ import annotations + +import asyncio +import logging +import math +import time +from collections import defaultdict, deque +from dataclasses import dataclass, field +from typing import Any + +logger = logging.getLogger(__name__) + +# ── Dataclasses ─────────────────────────────────────────────────────────────── + + +@dataclass +class NodeMetricsTick: + """Single per-node metrics sample.""" + + node_id: str + community_id: str + tick_at: float + + active_capabilities: int = 0 + events_per_min: float = 0.0 + peers_online: int = 0 + + llm_requests_total: int = 0 + rag_requests_total: int = 0 + + gpu_memory_mb: float | None = None + cpu_percent: float = 0.0 + memory_mb: float = 0.0 + + online_seconds: int = 0 + + +@dataclass +class CommunityMetrics: + """Aggregated metrics across all members of a community (full detail).""" + + community_id: str + member_count: int + online_count: int + + events_per_min_total: float + capabilities_total: int + + ticks: list[NodeMetricsTick] # per-node detail + sampled_at: float + + +@dataclass +class AggregatedSnapshot: + """ + Anonymised/banded aggregate for federated peers (less information + at greater distance per X07 design rule). + """ + + community_id: str + member_count_band: str # e.g. "10-20" + online_count_band: str + events_per_min_band: str + capabilities_count: int + federation_links_count: int + sampled_at: float + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _band(value: float, steps: list[int]) -> str: + """Return a band string like '10-20' for *value* given boundary *steps*.""" + for i, upper in enumerate(steps): + lower = steps[i - 1] if i > 0 else 0 + if value < upper: + return f"{lower}-{upper}" + last = steps[-1] if steps else 0 + return f"{last}+" + + +_MEMBER_BANDS = [5, 10, 20, 50, 100, 250, 500, 1000] +_ONLINE_BANDS = [2, 5, 10, 25, 50, 100, 250, 500] +_EPM_BANDS = [10, 50, 100, 500, 1000, 5000, 10000] + + +def _collect_system_metrics() -> dict[str, float]: + """Snapshot CPU / memory using psutil if available; otherwise zeros.""" + try: + import psutil # type: ignore[import] + cpu = psutil.cpu_percent(interval=None) + mem = psutil.virtual_memory().used / (1024 * 1024) + return {"cpu_percent": cpu, "memory_mb": mem} + except ImportError: + return {"cpu_percent": 0.0, "memory_mb": 0.0} + except Exception: + return {"cpu_percent": 0.0, "memory_mb": 0.0} + + +def _collect_gpu_memory() -> float | None: + """Return GPU memory usage in MB if pynvml is available.""" + try: + import pynvml # type: ignore[import] + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + info = pynvml.nvmlDeviceGetMemoryInfo(handle) + return info.used / (1024 * 1024) + except Exception: + return None + + +# ── FederatedMetricsExporter ────────────────────────────────────────────────── + + +class FederatedMetricsExporter: + """ + Snapshots local metrics and publishes them to the community bus topic + and optionally to an OTLP collector. + """ + + def __init__( + self, + node_id: str = "", + community_id: str = "", + bus: Any = None, + ) -> None: + self._node_id = node_id + self._community_id = community_id + self._bus = bus + self._start_time: float = time.time() + + def collect_tick(self, bus: Any = None) -> NodeMetricsTick: + """Snapshot current metrics into a :class:`NodeMetricsTick`.""" + _bus = bus or self._bus + sys_metrics = _collect_system_metrics() + + # Collect capability count from bus + active_caps = 0 + if _bus is not None: + try: + caps = _bus.list_capabilities() + active_caps = len(caps) if caps else 0 + except Exception: + pass + + # Collect request counters from prometheus registry if available + llm_total = 0 + rag_total = 0 + try: + from hearthnet.observability.metrics import _STD # noqa: PLC0415 + llm_counter = _STD.get("hearthnet_llm_requests_total") + rag_counter = _STD.get("hearthnet_rag_requests_total") + if llm_counter is not None and hasattr(llm_counter, "_value"): + llm_total = int(llm_counter._value.get() or 0) + if rag_counter is not None and hasattr(rag_counter, "_value"): + rag_total = int(rag_counter._value.get() or 0) + except Exception: + pass + + online_secs = int(time.time() - self._start_time) + + return NodeMetricsTick( + node_id=self._node_id, + community_id=self._community_id, + tick_at=time.time(), + active_capabilities=active_caps, + events_per_min=0.0, # filled by aggregator from event log + peers_online=0, # filled by aggregator from peer registry + llm_requests_total=llm_total, + rag_requests_total=rag_total, + gpu_memory_mb=_collect_gpu_memory(), + cpu_percent=sys_metrics["cpu_percent"], + memory_mb=sys_metrics["memory_mb"], + online_seconds=online_secs, + ) + + async def push_to_community(self, tick: NodeMetricsTick, bus: Any = None) -> None: + """Publish *tick* to the bus topic ``observability.metrics.tick.``.""" + _bus = bus or self._bus + if _bus is None: + logger.debug("FederatedMetricsExporter.push_to_community: no bus configured") + return + topic = f"observability.metrics.tick.{tick.node_id}" + payload: dict[str, Any] = { + "node_id": tick.node_id, + "community_id": tick.community_id, + "tick_at": tick.tick_at, + "active_capabilities": tick.active_capabilities, + "events_per_min": tick.events_per_min, + "peers_online": tick.peers_online, + "llm_requests_total": tick.llm_requests_total, + "rag_requests_total": tick.rag_requests_total, + "gpu_memory_mb": tick.gpu_memory_mb, + "cpu_percent": tick.cpu_percent, + "memory_mb": tick.memory_mb, + "online_seconds": tick.online_seconds, + } + try: + result = _bus.call( + "bus.publish", + (1, 0), + {"topic": topic, "event": "metrics_tick", "data": payload}, + ) + if asyncio.iscoroutine(result): + await result + except Exception as exc: + logger.warning("FederatedMetricsExporter.push_to_community failed: %s", exc) + + async def push_otlp(self, endpoint: str, tick: NodeMetricsTick) -> None: + """ + Export *tick* via OTLP HTTP. Requires opentelemetry-exporter-otlp-proto-http. + + Delegates to :class:`OtlpExporter`. + """ + try: + from hearthnet.observability.otlp_export import OtlpExporter # noqa: PLC0415 + exporter = OtlpExporter(endpoint) + await exporter.export_metrics(tick) + except ImportError: + logger.debug("push_otlp: opentelemetry not installed β€” skipping") + except Exception as exc: + logger.warning("FederatedMetricsExporter.push_otlp failed: %s", exc) + + +# ── MetricsAggregator ───────────────────────────────────────────────────────── + + +class MetricsAggregator: + """ + Receives NodeMetricsTick events from all community members and builds + community-level and federated snapshots. + """ + + def __init__( + self, + community_id: str, + max_ticks_per_node: int = 60, + ) -> None: + self._community_id = community_id + self._max_ticks = max_ticks_per_node + # node_id β†’ deque of ticks (newest last) + self._ticks: dict[str, deque[NodeMetricsTick]] = defaultdict( + lambda: deque(maxlen=self._max_ticks) + ) + self._federation_links: dict[str, int] = {} # peer_community_id β†’ count + + def apply_tick(self, tick: NodeMetricsTick) -> None: + """Incorporate a new tick from a community member.""" + self._ticks[tick.node_id].append(tick) + + def community_snapshot(self) -> CommunityMetrics: + """Return the latest community-wide aggregate.""" + now = time.time() + latest_ticks: list[NodeMetricsTick] = [] + online_cutoff = now - 120 # consider online if tick within 2 min + + for node_deque in self._ticks.values(): + if node_deque: + latest_ticks.append(node_deque[-1]) + + online = [t for t in latest_ticks if t.tick_at >= online_cutoff] + total_epm = sum(t.events_per_min for t in online) + total_caps = sum(t.active_capabilities for t in online) + + return CommunityMetrics( + community_id=self._community_id, + member_count=len(self._ticks), + online_count=len(online), + events_per_min_total=total_epm, + capabilities_total=total_caps, + ticks=list(latest_ticks), + sampled_at=now, + ) + + def federated_snapshot(self, peer_community_id: str) -> AggregatedSnapshot: + """ + Return a banded/anonymised snapshot suitable for sharing with a + federated peer community. + """ + snap = self.community_snapshot() + fed_links = len(self._federation_links) + + return AggregatedSnapshot( + community_id=self._community_id, + member_count_band=_band(snap.member_count, _MEMBER_BANDS), + online_count_band=_band(snap.online_count, _ONLINE_BANDS), + events_per_min_band=_band(snap.events_per_min_total, _EPM_BANDS), + capabilities_count=snap.capabilities_total, + federation_links_count=fed_links, + sampled_at=snap.sampled_at, + ) + + def record_federation_link(self, peer_community_id: str) -> None: + """Track that we have an active federation link to *peer_community_id*.""" + self._federation_links[peer_community_id] = self._federation_links.get( + peer_community_id, 0 + ) + 1 diff --git a/hearthnet/observability/otlp_export.py b/hearthnet/observability/otlp_export.py new file mode 100644 index 0000000000000000000000000000000000000000..9ceec2efa48d33229662b4057f3aa173b055542c --- /dev/null +++ b/hearthnet/observability/otlp_export.py @@ -0,0 +1,173 @@ +"""OTLP metrics and trace export (X07 optional OpenTelemetry integration).""" +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from hearthnet.observability.federated import NodeMetricsTick + +logger = logging.getLogger(__name__) + +# Optional OpenTelemetry imports +try: + from opentelemetry import metrics as otel_metrics # type: ignore[import] + from opentelemetry.sdk.metrics import MeterProvider # type: ignore[import] + from opentelemetry.sdk.metrics.export import ( # type: ignore[import] + PeriodicExportingMetricReader, + ) + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( # type: ignore[import] + OTLPMetricExporter, + ) + HAS_OTEL_METRICS = True +except ImportError: + HAS_OTEL_METRICS = False + +try: + from opentelemetry.sdk.trace import TracerProvider # type: ignore[import] + from opentelemetry.sdk.trace.export import SimpleSpanProcessor # type: ignore[import] + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # type: ignore[import] + OTLPSpanExporter, + ) + HAS_OTEL_TRACES = True +except ImportError: + HAS_OTEL_TRACES = False + + +class OtlpExporter: + """ + Sends HearthNet metrics and traces to an OTLP HTTP collector. + + Both opentelemetry-sdk and opentelemetry-exporter-otlp-proto-http are + optional. All methods return False / empty and log a debug message + when the libraries are not installed. + """ + + def __init__(self, endpoint: str) -> None: + self._endpoint = endpoint.rstrip("/") + self._meter_provider: Any = None + self._tracer_provider: Any = None + + def _get_meter_provider(self) -> Any: + """Lazily initialise the OTLP MeterProvider.""" + if self._meter_provider is not None: + return self._meter_provider + if not HAS_OTEL_METRICS: + return None + try: + exporter = OTLPMetricExporter(endpoint=f"{self._endpoint}/v1/metrics") # type: ignore[possibly-undefined] + reader = PeriodicExportingMetricReader(exporter, export_interval_millis=60_000) # type: ignore[possibly-undefined] + provider = MeterProvider(metric_readers=[reader]) # type: ignore[possibly-undefined] + self._meter_provider = provider + return provider + except Exception as exc: + logger.warning("OtlpExporter: failed to create MeterProvider: %s", exc) + return None + + def _get_tracer_provider(self) -> Any: + """Lazily initialise the OTLP TracerProvider.""" + if self._tracer_provider is not None: + return self._tracer_provider + if not HAS_OTEL_TRACES: + return None + try: + exporter = OTLPSpanExporter(endpoint=f"{self._endpoint}/v1/traces") # type: ignore[possibly-undefined] + provider = TracerProvider() # type: ignore[possibly-undefined] + provider.add_span_processor(SimpleSpanProcessor(exporter)) # type: ignore[possibly-undefined] + self._tracer_provider = provider + return provider + except Exception as exc: + logger.warning("OtlpExporter: failed to create TracerProvider: %s", exc) + return None + + async def export_metrics(self, tick: NodeMetricsTick) -> bool: + """ + Export a :class:`NodeMetricsTick` as OTLP gauge metrics. + + Returns True if data was exported, False if opentelemetry is not + installed or an error occurred. + """ + if not HAS_OTEL_METRICS: + logger.debug( + "OtlpExporter.export_metrics: opentelemetry not installed β€” skipping" + ) + return False + provider = self._get_meter_provider() + if provider is None: + return False + try: + meter = provider.get_meter("hearthnet") # type: ignore[union-attr] + # Record gauges for each numeric field on the tick + fields: dict[str, float | int | None] = { + "active_capabilities": tick.active_capabilities, + "events_per_min": tick.events_per_min, + "peers_online": tick.peers_online, + "llm_requests_total": tick.llm_requests_total, + "rag_requests_total": tick.rag_requests_total, + "cpu_percent": tick.cpu_percent, + "memory_mb": tick.memory_mb, + "online_seconds": tick.online_seconds, + "gpu_memory_mb": tick.gpu_memory_mb, + } + attrs = { + "node_id": tick.node_id, + "community_id": tick.community_id, + } + for name, value in fields.items(): + if value is None: + continue + g = meter.create_gauge( + f"hearthnet_{name}", + description=f"HearthNet {name}", + ) + g.set(float(value), attrs) + return True + except Exception as exc: + logger.warning("OtlpExporter.export_metrics error: %s", exc) + return False + + async def export_traces(self, spans: list[dict]) -> bool: + """ + Export a list of span dicts as OTLP traces. + + Each span dict should have at minimum: ``name``, ``trace_id``, + ``span_id``, ``start_time``, ``end_time``. + + Returns True if spans were submitted, False otherwise. + """ + if not HAS_OTEL_TRACES: + logger.debug( + "OtlpExporter.export_traces: opentelemetry not installed β€” skipping" + ) + return False + provider = self._get_tracer_provider() + if provider is None: + return False + try: + tracer = provider.get_tracer("hearthnet") # type: ignore[union-attr] + for span_dict in spans: + name = span_dict.get("name", "hearthnet.span") + with tracer.start_as_current_span(name) as span: + for k, v in span_dict.items(): + if k not in ("name", "trace_id", "span_id", "start_time", "end_time"): + try: + span.set_attribute(k, str(v)) + except Exception: + pass + return True + except Exception as exc: + logger.warning("OtlpExporter.export_traces error: %s", exc) + return False + + async def shutdown(self) -> None: + """Flush and shut down the underlying providers.""" + if self._meter_provider is not None: + try: + self._meter_provider.shutdown() # type: ignore[union-attr] + except Exception: + pass + if self._tracer_provider is not None: + try: + self._tracer_provider.shutdown() # type: ignore[union-attr] + except Exception: + pass diff --git a/hearthnet/relay/__init__.py b/hearthnet/relay/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..512b55161d212720995f61cb8f5e42c97379d93e --- /dev/null +++ b/hearthnet/relay/__init__.py @@ -0,0 +1,2 @@ +# hearthnet/relay β€” client-side relay helpers (M15) +from __future__ import annotations diff --git a/hearthnet/relay/client.py b/hearthnet/relay/client.py new file mode 100644 index 0000000000000000000000000000000000000000..8b4565581d4818435f80ae9539346a936487379e --- /dev/null +++ b/hearthnet/relay/client.py @@ -0,0 +1,209 @@ +"""Relay client β€” registers with a relay server for NAT traversal (M15).""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +from dataclasses import dataclass + +logger = logging.getLogger(__name__) + +# Optional httpx +try: + import httpx + HAS_HTTPX = True +except ImportError: + httpx = None # type: ignore[assignment] + HAS_HTTPX = False + + +@dataclass(frozen=True) +class RelayRegistration: + """Immutable record of a successful relay registration.""" + + relay_url: str + node_id: str + registered_at: float + expires_at: float + forwarding_url: str + + +class RelayClient: + """ + Client-side helper for registering with a HearthNet relay server. + + All methods degrade gracefully when the relay is unreachable: they log a + warning and return empty / False rather than raising. + """ + + def __init__( + self, + relay_url: str, + http_client: object | None = None, + keypair: object | None = None, + ) -> None: + self._relay_url = relay_url.rstrip("/") + self._http_client = http_client # Optional HttpClient instance + self._keypair = keypair + self._httpx_client: object | None = None + self._keepalive_task: asyncio.Task | None = None + + # ── Internal helpers ────────────────────────────────────────────────────── + + def _get_httpx(self) -> object: + """Return or create a shared httpx.AsyncClient.""" + if not HAS_HTTPX: + raise ImportError("httpx is required for RelayClient: pip install httpx") + if self._httpx_client is None: + import httpx as _httpx # noqa: PLC0415 + self._httpx_client = _httpx.AsyncClient(timeout=10.0) + return self._httpx_client + + def _sign_payload(self, payload: dict) -> dict: + """Attach a signature if a keypair is present.""" + if self._keypair is None: + return payload + try: + if hasattr(self._keypair, "sign"): + sig = self._keypair.sign(json.dumps(payload, sort_keys=True).encode()) + payload = dict(payload) + if hasattr(sig, "hex"): + payload["_sig"] = sig.hex() + else: + payload["_sig"] = str(sig) + except Exception as exc: + logger.debug("RelayClient._sign_payload: %s", exc) + return payload + + async def _post(self, path: str, body: dict) -> dict | None: + """POST *body* to relay path. Returns parsed JSON or None on error.""" + url = f"{self._relay_url}{path}" + try: + client = self._get_httpx() + resp = await client.post(url, json=body) # type: ignore[union-attr] + resp.raise_for_status() + return resp.json() + except Exception as exc: + logger.warning("RelayClient POST %s failed: %s", path, exc) + return None + + async def _get(self, path: str) -> dict | list | None: + """GET relay path. Returns parsed JSON or None on error.""" + url = f"{self._relay_url}{path}" + try: + client = self._get_httpx() + resp = await client.get(url) # type: ignore[union-attr] + resp.raise_for_status() + return resp.json() + except Exception as exc: + logger.warning("RelayClient GET %s failed: %s", path, exc) + return None + + # ── Public API ──────────────────────────────────────────────────────────── + + async def register( + self, + node_id: str, + endpoints: list[str], + community_id: str, + ttl: int = 60, + ) -> RelayRegistration: + """ + Register this node with the relay server. + + Returns a :class:`RelayRegistration`. On failure, returns a + registration with ``forwarding_url=""`` and ``expires_at`` in the past. + """ + now = time.time() + payload = self._sign_payload( + { + "node_id": node_id, + "endpoints": endpoints, + "community_id": community_id, + "ttl": ttl, + "registered_at": now, + } + ) + data = await self._post("/relay/v1/register", payload) + if data and isinstance(data, dict): + return RelayRegistration( + relay_url=self._relay_url, + node_id=node_id, + registered_at=now, + expires_at=data.get("expires_at", now + ttl), + forwarding_url=data.get("forwarding_url", ""), + ) + # Degraded: return sentinel with past expiry + return RelayRegistration( + relay_url=self._relay_url, + node_id=node_id, + registered_at=now, + expires_at=now - 1, + forwarding_url="", + ) + + async def heartbeat(self, node_id: str) -> bool: + """Renew the relay registration. Returns True on success.""" + payload = self._sign_payload({"node_id": node_id, "ts": time.time()}) + data = await self._post("/relay/v1/heartbeat", payload) + return bool(data and data.get("ok")) + + async def deregister(self, node_id: str) -> bool: + """Cleanly remove the relay registration. Returns True on success.""" + payload = self._sign_payload({"node_id": node_id}) + data = await self._post("/relay/v1/deregister", payload) + return bool(data and data.get("ok")) + + async def lookup_community(self, community_id: str) -> list[str]: + """ + Look up the current bridge endpoint URLs for *community_id*. + + Returns an empty list on error. + """ + data = await self._get(f"/relay/v1/community/{community_id}") + if data is None: + return [] + if isinstance(data, list): + return [str(e) for e in data] + if isinstance(data, dict): + # Accept {"endpoints": [...]} envelope + return [str(e) for e in data.get("endpoints", [])] + return [] + + async def start_keepalive( + self, + node_id: str, + interval: int = 30, + ) -> asyncio.Task: + """ + Start a background asyncio task that sends a heartbeat every + *interval* seconds. Cancels any previously running keepalive. + """ + if self._keepalive_task is not None and not self._keepalive_task.done(): + self._keepalive_task.cancel() + + async def _loop() -> None: + while True: + await asyncio.sleep(interval) + ok = await self.heartbeat(node_id) + if not ok: + logger.warning("RelayClient keepalive heartbeat failed for %s", node_id) + + self._keepalive_task = asyncio.create_task(_loop()) + return self._keepalive_task + + async def close(self) -> None: + """Cancel keepalive and close the internal httpx client.""" + if self._keepalive_task is not None and not self._keepalive_task.done(): + self._keepalive_task.cancel() + try: + await self._keepalive_task + except asyncio.CancelledError: + pass + if self._httpx_client is not None: + try: + await self._httpx_client.aclose() # type: ignore[union-attr] + except Exception: + pass + self._httpx_client = None diff --git a/hearthnet/relay/push_subscriber.py b/hearthnet/relay/push_subscriber.py new file mode 100644 index 0000000000000000000000000000000000000000..542f0b513abc5b973df202e6510a4e8e6251745e --- /dev/null +++ b/hearthnet/relay/push_subscriber.py @@ -0,0 +1,105 @@ +"""Push token registry for relay-mediated mobile notifications (M15).""" +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +logger = logging.getLogger(__name__) + +try: + import httpx + HAS_HTTPX = True +except ImportError: + httpx = None # type: ignore[assignment] + HAS_HTTPX = False + + +@dataclass +class PushTokenRecord: + """Record of a mobile push token registered with a relay.""" + + node_id: str + platform: str # "apns" | "fcm" | "generic" + device_token: str + relay_url: str + registered_at: float + + +class PushSubscriber: + """ + Registers and unregisters APNs / FCM device tokens with a relay server. + + Degrades gracefully when the relay is unreachable. + """ + + def __init__( + self, + relay_url: str, + http_client: object | None = None, + ) -> None: + self._relay_url = relay_url.rstrip("/") + self._http_client = http_client + self._httpx_client: object | None = None + + # ── Internal helpers ────────────────────────────────────────────────────── + + def _get_httpx(self) -> object: + if not HAS_HTTPX: + raise ImportError("httpx is required for PushSubscriber: pip install httpx") + if self._httpx_client is None: + import httpx as _httpx # noqa: PLC0415 + self._httpx_client = _httpx.AsyncClient(timeout=10.0) + return self._httpx_client + + async def _post(self, path: str, body: dict) -> dict | None: + url = f"{self._relay_url}{path}" + try: + client = self._get_httpx() + resp = await client.post(url, json=body) # type: ignore[union-attr] + resp.raise_for_status() + return resp.json() + except Exception as exc: + logger.warning("PushSubscriber POST %s failed: %s", path, exc) + return None + + # ── Public API ──────────────────────────────────────────────────────────── + + async def register_push_token( + self, + node_id: str, + platform: str, + device_token: str, + ) -> bool: + """ + Register a mobile push token with the relay. + + Returns True on success, False on failure. + """ + payload = { + "node_id": node_id, + "platform": platform, + "device_token": device_token, + "registered_at": time.time(), + } + data = await self._post("/relay/v1/push/register", payload) + return bool(data and data.get("ok")) + + async def unregister(self, node_id: str, device_token: str) -> bool: + """ + Remove a push token registration from the relay. + + Returns True on success, False on failure. + """ + payload = {"node_id": node_id, "device_token": device_token} + data = await self._post("/relay/v1/push/unregister", payload) + return bool(data and data.get("ok")) + + async def close(self) -> None: + """Close the internal httpx client.""" + if self._httpx_client is not None: + try: + await self._httpx_client.aclose() # type: ignore[union-attr] + except Exception: + pass + self._httpx_client = None diff --git a/hearthnet/services/auth/__init__.py b/hearthnet/services/auth/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d48db4f9f85e1752cf424c49ee18a6907c3f160 --- /dev/null +++ b/hearthnet/services/auth/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/hearthnet/services/auth/service.py b/hearthnet/services/auth/service.py new file mode 100644 index 0000000000000000000000000000000000000000..5bb0e058ab5ca97e437f9d16d2adb3f373ebf0c3 --- /dev/null +++ b/hearthnet/services/auth/service.py @@ -0,0 +1,181 @@ +"""AuthService β€” registers auth.token.* capabilities on the bus (M16).""" +from __future__ import annotations + +from typing import Any + +from hearthnet.identity.tokens import ( + CapabilityToken, + TokenError, + TokenScope, + decode_token, + issue_token, + verify_token, +) + + +class AuthService: + """Manages capability token issuance, verification, and revocation. + + Registers: + auth.token.issue@1.0 + auth.token.verify@1.0 + auth.token.revoke@1.0 + """ + + name = "auth" + + def __init__( + self, + keypair: Any, + community_manifest: Any | None = None, + bus: Any | None = None, + ) -> None: + self._kp = keypair + self._community_manifest = community_manifest + self._bus = bus + self._revoked_jtis: set[str] = set() + + # ------------------------------------------------------------------ + # Registration + # ------------------------------------------------------------------ + + def register(self, bus: Any) -> None: + """Register all auth capabilities with the bus Registry.""" + from hearthnet.bus.capability import CapabilityDescriptor + + self._bus = bus + registry = getattr(bus, "registry", None) + if registry is None: + return + + descriptors = [ + ("auth.token.issue", "1.0", self._handle_issue), + ("auth.token.verify", "1.0", self._handle_verify), + ("auth.token.revoke", "1.0", self._handle_revoke), + ] + for name, version, handler in descriptors: + desc = CapabilityDescriptor( + name=name, + version=version, + stability="stable", + params={}, + max_concurrent=4, + ) + registry.register_local(desc, handler) + + # ------------------------------------------------------------------ + # Handlers + # ------------------------------------------------------------------ + + def _handle_issue(self, params: dict) -> dict: + """auth.token.issue@1.0 handler. + + params: {subject, audience, capabilities: list[str], + ttl_seconds=3600, issued_via="manual", + max_uses=None, max_calls_total=None} + returns: {token: str, expires_at: int} + """ + subject = params.get("subject", "") + audience = params.get("audience", "") + capabilities = list(params.get("capabilities", [])) + ttl = int(params.get("ttl_seconds", 3600)) + issued_via = str(params.get("issued_via", "manual")) + max_uses = params.get("max_uses") + max_calls_total = params.get("max_calls_total") + + scope = TokenScope( + capabilities=capabilities, + max_uses=max_uses, + max_calls_total=max_calls_total, + ) + try: + tok, encoded = issue_token( + self._kp, + subject_node_id=subject, + audience=audience, + scope=scope, + ttl_seconds=ttl, + issued_via=issued_via, + ) + except TokenError as exc: + return {"error": str(exc)} + + return {"token": encoded, "expires_at": tok.exp} + + def _handle_verify(self, params: dict) -> dict: + """auth.token.verify@1.0 handler. + + params: {token: str} + returns: {valid: bool, subject: str | None, capabilities: list[str], expires_at: int} + """ + text = params.get("token", "") + try: + tok = decode_token(text) + except TokenError as exc: + return {"valid": False, "subject": None, "capabilities": [], "expires_at": 0, + "error": str(exc)} + + # Check revocation + if tok.jti in self._revoked_jtis: + return {"valid": False, "subject": tok.sub, "capabilities": list(tok.scope.capabilities), + "expires_at": tok.exp, "error": "Token has been revoked"} + + try: + verify_token(tok, community_manifest=self._community_manifest) + except TokenError as exc: + return {"valid": False, "subject": tok.sub, "capabilities": list(tok.scope.capabilities), + "expires_at": tok.exp, "error": str(exc)} + + return { + "valid": True, + "subject": tok.sub, + "capabilities": list(tok.scope.capabilities), + "expires_at": tok.exp, + } + + def _handle_revoke(self, params: dict) -> dict: + """auth.token.revoke@1.0 handler. + + params: {jti: str} + returns: {revoked: bool} + """ + jti = params.get("jti", "") + if not jti: + return {"revoked": False, "error": "No jti provided"} + self._revoked_jtis.add(jti) + return {"revoked": True} + + # ------------------------------------------------------------------ + # Direct API (for use without the bus) + # ------------------------------------------------------------------ + + def issue( + self, + subject: str, + audience: str, + capabilities: list[str], + ttl_seconds: int = 3600, + issued_via: str = "manual", + ) -> tuple[CapabilityToken, str]: + """Issue a token directly (bypasses the bus).""" + scope = TokenScope(capabilities=capabilities) + return issue_token( + self._kp, + subject_node_id=subject, + audience=audience, + scope=scope, + ttl_seconds=ttl_seconds, + issued_via=issued_via, + ) + + def verify(self, text: str) -> CapabilityToken: + """Verify a token string directly. Raises TokenError if invalid.""" + tok = decode_token(text) + if tok.jti in self._revoked_jtis: + raise TokenError(f"Token {tok.jti!r} has been revoked") + verify_token(tok, community_manifest=self._community_manifest) + return tok + + def revoke(self, jti: str) -> None: + """Revoke a token by JTI (in-memory, not persisted across restart).""" + self._revoked_jtis.add(jti) diff --git a/hearthnet/services/chat/thread_service.py b/hearthnet/services/chat/thread_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e6da0002569484466f246978c7c98edd8cee9ffc --- /dev/null +++ b/hearthnet/services/chat/thread_service.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import time +import uuid +from typing import Any + +from hearthnet.bus.capability import CapabilityDescriptor, RouteRequest +from hearthnet.services.chat.thread_views import ThreadViewStore + + +class ThreadService: + """Group-chat thread service. + + Registers: + chat.thread.create@1.0 + chat.thread.send@1.0 + chat.thread.history@1.0 + chat.thread.invite@1.0 + chat.thread.leave@1.0 + """ + + name = "chat.threads" + + def __init__( + self, + node_id: str, + event_log: Any = None, + bus: Any = None, + db_path: str | None = None, + ) -> None: + self._node_id = node_id + self._event_log = event_log + self._bus = bus + self._store = ThreadViewStore(db_path=db_path) + + # ── Registration ────────────────────────────────────────────────────────── + + def capabilities(self) -> list[tuple]: + return [ + (CapabilityDescriptor(name="chat.thread.create", max_concurrent=4, idempotent=False), self.create_thread, None), + (CapabilityDescriptor(name="chat.thread.send", max_concurrent=8, idempotent=False), self.send_message, None), + (CapabilityDescriptor(name="chat.thread.history", max_concurrent=8, idempotent=True), self.get_history, None), + (CapabilityDescriptor(name="chat.thread.invite", max_concurrent=4, idempotent=True), self.invite_member, None), + (CapabilityDescriptor(name="chat.thread.leave", max_concurrent=4, idempotent=False), self.leave_thread, None), + ] + + def register(self, bus: Any) -> None: + self._bus = bus + for cap, handler, predicate in self.capabilities(): + bus.register_local(cap, handler, predicate) + + # ── Handlers ────────────────────────────────────────────────────────────── + + async def create_thread(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + name: str = params.get("name", "") + members: list[str] = list(params.get("members", [])) + e2e_enabled: bool = bool(params.get("e2e_enabled", False)) + + caller = req.caller or self._node_id + if caller not in members: + members.append(caller) + + thread_id = f"thread:{uuid.uuid4().hex}" + created_at = time.time() + + event = { + "event_id": f"evt:{uuid.uuid4().hex}", + "event_type": "chat.thread.created", + "author": caller, + "payload": { + "thread_id": thread_id, + "name": name, + "members": members, + "created_at": created_at, + "e2e_enabled": e2e_enabled, + }, + } + + if self._event_log is not None: + try: + logged = self._event_log.append_local( + event_type="chat.thread.created", + author=caller, + payload=event["payload"], + ) + self._store.apply({ + "event_id": logged.event_id, + "event_type": "chat.thread.created", + "author": caller, + "payload": event["payload"], + }) + except Exception: + self._store.apply(event) + else: + self._store.apply(event) + + return {"output": {"thread_id": thread_id, "created_at": created_at}, "meta": {}} + + async def send_message(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + thread_id: str | None = params.get("thread_id") + content: str = params.get("content", "") + caller = req.caller or self._node_id + + if not thread_id: + return {"error": "bad_request", "message": "thread_id required"} + if not content: + return {"error": "bad_request", "message": "content required"} + + # Verify thread exists and caller is a member + thread = self._store.get_thread(thread_id) + if thread is None: + return {"error": "not_found", "message": f"thread {thread_id} not found"} + if caller not in thread.members: + return {"error": "forbidden", "message": "not a member of this thread"} + + event_id = f"msg:{uuid.uuid4().hex}" + sent_at = time.time() + + event = { + "event_id": event_id, + "event_type": "chat.thread.message.sent", + "author": caller, + "payload": { + "thread_id": thread_id, + "sender": caller, + "content": content, + "sent_at": sent_at, + }, + } + + if self._event_log is not None: + try: + logged = self._event_log.append_local( + event_type="chat.thread.message.sent", + author=caller, + payload=event["payload"], + ) + self._store.apply({ + "event_id": logged.event_id, + "event_type": "chat.thread.message.sent", + "author": caller, + "payload": event["payload"], + }) + event_id = logged.event_id + except Exception: + self._store.apply(event) + else: + self._store.apply(event) + + return {"output": {"event_id": event_id, "sent_at": sent_at}, "meta": {}} + + async def get_history(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + thread_id: str | None = params.get("thread_id") + since: float | None = params.get("since") + limit: int = int(params.get("limit", 50)) + caller = req.caller or self._node_id + + if not thread_id: + return {"error": "bad_request", "message": "thread_id required"} + + thread = self._store.get_thread(thread_id) + if thread is None: + return {"error": "not_found", "message": f"thread {thread_id} not found"} + if caller not in thread.members: + return {"error": "forbidden", "message": "not a member of this thread"} + + messages = self._store.get_messages(thread_id, since=since, limit=limit) + return { + "output": { + "messages": [ + { + "event_id": m.event_id, + "thread_id": m.thread_id, + "sender": m.sender, + "content": m.content, + "sent_at": m.sent_at, + "delivered_to": list(m.delivered_to), + } + for m in messages + ] + }, + "meta": {}, + } + + async def invite_member(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + thread_id: str | None = params.get("thread_id") + member_id: str | None = params.get("member_id") + caller = req.caller or self._node_id + + if not thread_id or not member_id: + return {"error": "bad_request", "message": "thread_id and member_id required"} + + thread = self._store.get_thread(thread_id) + if thread is None: + return {"error": "not_found", "message": f"thread {thread_id} not found"} + if caller not in thread.members: + return {"error": "forbidden", "message": "not a member of this thread"} + + event = { + "event_id": f"evt:{uuid.uuid4().hex}", + "event_type": "chat.thread.member.added", + "author": caller, + "payload": {"thread_id": thread_id, "member_id": member_id}, + } + + if self._event_log is not None: + try: + self._event_log.append_local( + event_type="chat.thread.member.added", + author=caller, + payload=event["payload"], + ) + except Exception: + pass + + self._store.apply(event) + return {"output": {"success": True}, "meta": {}} + + async def leave_thread(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + thread_id: str | None = params.get("thread_id") + caller = req.caller or self._node_id + + if not thread_id: + return {"error": "bad_request", "message": "thread_id required"} + + thread = self._store.get_thread(thread_id) + if thread is None: + return {"error": "not_found", "message": f"thread {thread_id} not found"} + if caller not in thread.members: + return {"error": "forbidden", "message": "not a member of this thread"} + + event = { + "event_id": f"evt:{uuid.uuid4().hex}", + "event_type": "chat.thread.member.removed", + "author": caller, + "payload": {"thread_id": thread_id, "member_id": caller}, + } + + if self._event_log is not None: + try: + self._event_log.append_local( + event_type="chat.thread.member.removed", + author=caller, + payload=event["payload"], + ) + except Exception: + pass + + self._store.apply(event) + return {"output": {"success": True}, "meta": {}} diff --git a/hearthnet/services/chat/thread_views.py b/hearthnet/services/chat/thread_views.py new file mode 100644 index 0000000000000000000000000000000000000000..17c1a5afc828605d9a55ce9e6705910ddeee2e8d --- /dev/null +++ b/hearthnet/services/chat/thread_views.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +import sqlite3 +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class Thread: + thread_id: str + name: str + members: list[str] + created_at: float + archived: bool + e2e_enabled: bool + + +@dataclass(frozen=True) +class ThreadMessage: + event_id: str + thread_id: str + sender: str + content: str + sent_at: float + delivered_to: frozenset[str] + + +class ThreadViewStore: + """Materialised view of thread state from chat.thread.* events. + + Uses SQLite when available, falls back to in-memory dicts. + """ + + def __init__(self, db_path: str | Path | None = None) -> None: + self._lock = threading.Lock() + self._db: sqlite3.Connection | None = None + + # In-memory fallback structures + self._threads: dict[str, dict] = {} # thread_id -> thread data + self._members: dict[str, set[str]] = {} # thread_id -> set of member_ids + self._messages: dict[str, dict] = {} # event_id -> message data + self._msg_by_thread: dict[str, list[str]] = {} # thread_id -> [event_id, ...] + # read receipts: thread_id -> {member_id -> last_read_ts} + self._read_receipts: dict[str, dict[str, float]] = {} + + if db_path: + try: + self._db = sqlite3.connect(str(db_path), check_same_thread=False) + self._db.execute("PRAGMA journal_mode=WAL") + self._init_schema() + except Exception: + self._db = None + + def _init_schema(self) -> None: + assert self._db is not None + self._db.executescript(""" + CREATE TABLE IF NOT EXISTS threads ( + thread_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at REAL NOT NULL, + archived INTEGER NOT NULL DEFAULT 0, + e2e_enabled INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS thread_members ( + thread_id TEXT NOT NULL, + member_id TEXT NOT NULL, + PRIMARY KEY (thread_id, member_id) + ); + CREATE TABLE IF NOT EXISTS thread_messages ( + event_id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL, + sender TEXT NOT NULL, + content TEXT NOT NULL, + sent_at REAL NOT NULL + ); + CREATE TABLE IF NOT EXISTS delivered_to ( + event_id TEXT NOT NULL, + member_id TEXT NOT NULL, + PRIMARY KEY (event_id, member_id) + ); + CREATE TABLE IF NOT EXISTS read_receipts ( + thread_id TEXT NOT NULL, + member_id TEXT NOT NULL, + last_read_ts REAL NOT NULL, + PRIMARY KEY (thread_id, member_id) + ); + """) + self._db.commit() + + # ── Apply event ─────────────────────────────────────────────────────────── + + def apply(self, event: dict) -> None: + etype = event.get("event_type", "") + payload = event.get("payload", {}) + author = event.get("author", "") + event_id = event.get("event_id", "") + + with self._lock: + if etype == "chat.thread.created": + self._apply_thread_created(event_id, payload, author) + elif etype == "chat.thread.message.sent": + self._apply_message_sent(event_id, payload, author) + elif etype == "chat.thread.member.added": + self._apply_member_added(payload) + elif etype == "chat.thread.member.removed": + self._apply_member_removed(payload) + elif etype == "chat.thread.archived": + self._apply_archived(payload) + + def _apply_thread_created(self, event_id: str, payload: dict, author: str) -> None: + thread_id = payload.get("thread_id", event_id) + members: list[str] = list(payload.get("members", [])) + if author and author not in members: + members.append(author) + name = payload.get("name", "") + created_at = payload.get("created_at", time.time()) + e2e_enabled = bool(payload.get("e2e_enabled", False)) + + if self._db: + self._db.execute( + "INSERT OR IGNORE INTO threads (thread_id, name, created_at, archived, e2e_enabled) VALUES (?,?,?,0,?)", + (thread_id, name, created_at, int(e2e_enabled)), + ) + for m in members: + self._db.execute( + "INSERT OR IGNORE INTO thread_members (thread_id, member_id) VALUES (?,?)", + (thread_id, m), + ) + self._db.commit() + else: + if thread_id not in self._threads: + self._threads[thread_id] = { + "thread_id": thread_id, + "name": name, + "created_at": created_at, + "archived": False, + "e2e_enabled": e2e_enabled, + } + self._members[thread_id] = set(members) + self._msg_by_thread[thread_id] = [] + + def _apply_message_sent(self, event_id: str, payload: dict, author: str) -> None: + thread_id = payload.get("thread_id", "") + sender = payload.get("sender", author) + content = payload.get("content", "") + sent_at = payload.get("sent_at", time.time()) + + if self._db: + self._db.execute( + "INSERT OR IGNORE INTO thread_messages (event_id, thread_id, sender, content, sent_at) VALUES (?,?,?,?,?)", + (event_id, thread_id, sender, content, sent_at), + ) + self._db.commit() + else: + if event_id not in self._messages: + self._messages[event_id] = { + "event_id": event_id, + "thread_id": thread_id, + "sender": sender, + "content": content, + "sent_at": sent_at, + "delivered_to": set(), + } + self._msg_by_thread.setdefault(thread_id, []).append(event_id) + + def _apply_member_added(self, payload: dict) -> None: + thread_id = payload.get("thread_id", "") + member_id = payload.get("member_id", "") + if not thread_id or not member_id: + return + + if self._db: + self._db.execute( + "INSERT OR IGNORE INTO thread_members (thread_id, member_id) VALUES (?,?)", + (thread_id, member_id), + ) + self._db.commit() + else: + self._members.setdefault(thread_id, set()).add(member_id) + + def _apply_member_removed(self, payload: dict) -> None: + thread_id = payload.get("thread_id", "") + member_id = payload.get("member_id", "") + if not thread_id or not member_id: + return + + if self._db: + self._db.execute( + "DELETE FROM thread_members WHERE thread_id=? AND member_id=?", + (thread_id, member_id), + ) + self._db.commit() + else: + self._members.get(thread_id, set()).discard(member_id) + + def _apply_archived(self, payload: dict) -> None: + thread_id = payload.get("thread_id", "") + if not thread_id: + return + + if self._db: + self._db.execute( + "UPDATE threads SET archived=1 WHERE thread_id=?", (thread_id,) + ) + self._db.commit() + else: + if thread_id in self._threads: + t = self._threads[thread_id] + t["archived"] = True + + # ── Queries ─────────────────────────────────────────────────────────────── + + def get_thread(self, thread_id: str) -> Thread | None: + with self._lock: + if self._db: + row = self._db.execute( + "SELECT thread_id, name, created_at, archived, e2e_enabled FROM threads WHERE thread_id=?", + (thread_id,), + ).fetchone() + if not row: + return None + members_rows = self._db.execute( + "SELECT member_id FROM thread_members WHERE thread_id=?", (thread_id,) + ).fetchall() + members = [r[0] for r in members_rows] + return Thread( + thread_id=row[0], + name=row[1], + members=members, + created_at=row[2], + archived=bool(row[3]), + e2e_enabled=bool(row[4]), + ) + else: + t = self._threads.get(thread_id) + if not t: + return None + return Thread( + thread_id=t["thread_id"], + name=t["name"], + members=list(self._members.get(thread_id, set())), + created_at=t["created_at"], + archived=t["archived"], + e2e_enabled=t["e2e_enabled"], + ) + + def list_threads(self, member_id: str) -> list[Thread]: + with self._lock: + if self._db: + rows = self._db.execute( + """SELECT t.thread_id, t.name, t.created_at, t.archived, t.e2e_enabled + FROM threads t + JOIN thread_members tm ON t.thread_id=tm.thread_id + WHERE tm.member_id=? + ORDER BY t.created_at DESC""", + (member_id,), + ).fetchall() + result = [] + for row in rows: + thread_id = row[0] + members_rows = self._db.execute( + "SELECT member_id FROM thread_members WHERE thread_id=?", (thread_id,) + ).fetchall() + members = [r[0] for r in members_rows] + result.append(Thread( + thread_id=thread_id, + name=row[1], + members=members, + created_at=row[2], + archived=bool(row[3]), + e2e_enabled=bool(row[4]), + )) + return result + else: + results = [] + for tid, members in self._members.items(): + if member_id in members: + t = self._threads.get(tid) + if t: + results.append(Thread( + thread_id=t["thread_id"], + name=t["name"], + members=list(members), + created_at=t["created_at"], + archived=t["archived"], + e2e_enabled=t["e2e_enabled"], + )) + results.sort(key=lambda x: x.created_at, reverse=True) + return results + + def get_messages( + self, + thread_id: str, + since: float | None = None, + limit: int = 50, + ) -> list[ThreadMessage]: + with self._lock: + if self._db: + if since is not None: + rows = self._db.execute( + "SELECT event_id, thread_id, sender, content, sent_at FROM thread_messages " + "WHERE thread_id=? AND sent_at>? ORDER BY sent_at ASC LIMIT ?", + (thread_id, since, limit), + ).fetchall() + else: + rows = self._db.execute( + "SELECT event_id, thread_id, sender, content, sent_at FROM thread_messages " + "WHERE thread_id=? ORDER BY sent_at ASC LIMIT ?", + (thread_id, limit), + ).fetchall() + messages = [] + for row in rows: + eid = row[0] + delivered_rows = self._db.execute( + "SELECT member_id FROM delivered_to WHERE event_id=?", (eid,) + ).fetchall() + delivered = frozenset(r[0] for r in delivered_rows) + messages.append(ThreadMessage( + event_id=eid, + thread_id=row[1], + sender=row[2], + content=row[3], + sent_at=row[4], + delivered_to=delivered, + )) + return messages + else: + eids = self._msg_by_thread.get(thread_id, []) + msgs = [] + for eid in eids: + m = self._messages.get(eid) + if m and (since is None or m["sent_at"] > since): + msgs.append(ThreadMessage( + event_id=m["event_id"], + thread_id=m["thread_id"], + sender=m["sender"], + content=m["content"], + sent_at=m["sent_at"], + delivered_to=frozenset(m["delivered_to"]), + )) + msgs.sort(key=lambda x: x.sent_at) + return msgs[:limit] + + def unread_count(self, thread_id: str, member_id: str) -> int: + with self._lock: + if self._db: + row = self._db.execute( + "SELECT last_read_ts FROM read_receipts WHERE thread_id=? AND member_id=?", + (thread_id, member_id), + ).fetchone() + last_read = row[0] if row else 0.0 + count = self._db.execute( + "SELECT COUNT(*) FROM thread_messages WHERE thread_id=? AND sent_at>? AND sender!=?", + (thread_id, last_read, member_id), + ).fetchone()[0] + return int(count) + else: + last_read = self._read_receipts.get(thread_id, {}).get(member_id, 0.0) + eids = self._msg_by_thread.get(thread_id, []) + count = 0 + for eid in eids: + m = self._messages.get(eid) + if m and m["sent_at"] > last_read and m["sender"] != member_id: + count += 1 + return count diff --git a/hearthnet/services/image/__init__.py b/hearthnet/services/image/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/image/backends/__init__.py b/hearthnet/services/image/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/image/backends/base.py b/hearthnet/services/image/backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..b897be76ec50b56a365aaa017050ad571b37eaaa --- /dev/null +++ b/hearthnet/services/image/backends/base.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + + +@dataclass(frozen=True) +class ImageDescription: + caption: str + tags: list[str] + objects: list[str] + ocr_text: str | None + backend: str + ms: int + + +@dataclass(frozen=True) +class GenerationResult: + image_bytes: bytes + width: int + height: int + prompt_used: str + backend: str + ms: int + + +@runtime_checkable +class ImageDescribeBackend(Protocol): + name: str + + async def describe( + self, + image_bytes: bytes, + mode: str = "caption", + ) -> ImageDescription: ... + + def health(self) -> dict: ... + + +@runtime_checkable +class ImageGenerateBackend(Protocol): + name: str + + async def generate( + self, + prompt: str, + width: int = 512, + height: int = 512, + steps: int = 20, + lora: str | None = None, + ) -> GenerationResult: ... + + def health(self) -> dict: ... diff --git a/hearthnet/services/image/backends/florence2.py b/hearthnet/services/image/backends/florence2.py new file mode 100644 index 0000000000000000000000000000000000000000..8fb9a8b92cc896b684a86c814c080da4a73242ec --- /dev/null +++ b/hearthnet/services/image/backends/florence2.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +from hearthnet.services.image.backends.base import ImageDescription, GenerationResult + +if TYPE_CHECKING: + pass + +_TASK_MAP = { + "caption": "", + "detailed_caption": "", + "ocr": "", + "object_detection": "", +} + + +class Florence2Backend: + """Vision backend using Microsoft Florence-2.""" + + name = "florence2" + + def __init__( + self, + model: str = "microsoft/Florence-2-large", + device: str = "auto", + ) -> None: + self._model_id = model + self._device = device + self._processor = None + self._model = None + self._loaded = False + self._load_error: str | None = None + + def _load(self) -> bool: + if self._loaded: + return True + if self._load_error: + return False + try: + from transformers import AutoProcessor, AutoModelForCausalLM # type: ignore[import-untyped] + import torch # type: ignore[import-untyped] + + device = self._device + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + + self._processor = AutoProcessor.from_pretrained( + self._model_id, trust_remote_code=True + ) + self._model = AutoModelForCausalLM.from_pretrained( + self._model_id, + torch_dtype=torch.float16 if device == "cuda" else torch.float32, + trust_remote_code=True, + ).to(device) + self._device = device + self._loaded = True + return True + except ImportError as exc: + self._load_error = f"transformers/torch not installed: {exc}" + return False + except Exception as exc: + self._load_error = str(exc) + return False + + def _run_task(self, image, task_prompt: str) -> str: + """Run a single Florence-2 task prompt and return raw text result.""" + import torch # type: ignore[import-untyped] + + inputs = self._processor(text=task_prompt, images=image, return_tensors="pt").to( + self._device + ) + with torch.no_grad(): + generated_ids = self._model.generate( + input_ids=inputs["input_ids"], + pixel_values=inputs["pixel_values"], + max_new_tokens=1024, + num_beams=3, + do_sample=False, + ) + generated_text = self._processor.batch_decode( + generated_ids, skip_special_tokens=False + )[0] + parsed = self._processor.post_process_generation( + generated_text, + task=task_prompt, + image_size=(image.width, image.height), + ) + # parsed is typically {task_prompt: } + raw = parsed.get(task_prompt, "") + if isinstance(raw, dict): + return str(raw) + return str(raw) + + async def describe( + self, + image_bytes: bytes, + mode: str = "caption", + ) -> ImageDescription: + t0 = time.monotonic() + + if not self._load(): + return ImageDescription( + caption=f"[florence2 unavailable: {self._load_error}]", + tags=[], + objects=[], + ocr_text=None, + backend=self.name, + ms=0, + ) + + try: + from PIL import Image as PILImage # type: ignore[import-untyped] + import io + + pil_image = PILImage.open(io.BytesIO(image_bytes)).convert("RGB") + + task_key = _TASK_MAP.get(mode, "") + + caption = "" + tags: list[str] = [] + objects: list[str] = [] + ocr_text: str | None = None + + if mode == "ocr": + raw = self._run_task(pil_image, "") + ocr_text = raw + caption = raw[:200] if raw else "" + elif mode == "object_detection": + raw = self._run_task(pil_image, "") + # raw is a string repr of dict like {'': {'bboxes': [...], 'labels': [...]}} + # Try to extract labels + cap_text = self._run_task(pil_image, "") + caption = cap_text + try: + import ast + parsed = ast.literal_eval(raw) + if isinstance(parsed, dict): + inner = next(iter(parsed.values()), {}) + objects = inner.get("labels", []) if isinstance(inner, dict) else [] + except Exception: + objects = [] + else: + raw = self._run_task(pil_image, task_key) + caption = raw + + elapsed_ms = int((time.monotonic() - t0) * 1000) + return ImageDescription( + caption=caption, + tags=tags, + objects=objects, + ocr_text=ocr_text, + backend=self.name, + ms=elapsed_ms, + ) + except Exception as exc: + return ImageDescription( + caption=f"[florence2 error: {exc}]", + tags=[], + objects=[], + ocr_text=None, + backend=self.name, + ms=int((time.monotonic() - t0) * 1000), + ) + + def health(self) -> dict: + available = self._load_error is None + return { + "backend": self.name, + "model": self._model_id, + "loaded": self._loaded, + "available": available, + "error": self._load_error, + } diff --git a/hearthnet/services/image/describe_service.py b/hearthnet/services/image/describe_service.py new file mode 100644 index 0000000000000000000000000000000000000000..06b5a9837d7f455b91d6597591ff6cb4140e47ca --- /dev/null +++ b/hearthnet/services/image/describe_service.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import base64 +import time +from typing import Any + +from hearthnet.bus.capability import CapabilityDescriptor, RouteRequest +from hearthnet.services.image.backends.base import ImageDescribeBackend, ImageDescription +from hearthnet.services.image.backends.florence2 import Florence2Backend + + +class ImageDescribeService: + """Service wrapping image-description backends. + + Registers: img.describe@1.0 + """ + + name = "image.describe" + + def __init__(self, backends: list[ImageDescribeBackend] | None = None, bus: Any = None) -> None: + if backends is not None: + self._backends: list[ImageDescribeBackend] = backends + else: + self._backends = [Florence2Backend()] + self._bus = bus + self._by_name: dict[str, ImageDescribeBackend] = {b.name: b for b in self._backends} + + # ── Service registration ────────────────────────────────────────────────── + + def capabilities(self) -> list[tuple]: + return [ + ( + CapabilityDescriptor( + name="img.describe", + max_concurrent=2, + idempotent=True, + timeout_seconds=60, + ), + self.describe, + None, + ), + ] + + def register(self, bus: Any) -> None: + self._bus = bus + for cap, handler, predicate in self.capabilities(): + bus.register_local(cap, handler, predicate) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def describe(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + + image_cid: str | None = params.get("image_cid") + image_b64: str | None = params.get("image_b64") + mode: str = params.get("mode", "caption") + backend_name: str | None = params.get("backend") + + # Resolve image bytes + image_bytes: bytes | None = None + if image_b64: + try: + image_bytes = base64.b64decode(image_b64) + except Exception as exc: + return {"error": "bad_request", "message": f"invalid image_b64: {exc}"} + elif image_cid: + # Attempt to resolve from blob store if available + try: + from hearthnet.blobs.store import BlobStore # type: ignore[import-untyped] + # If bus has a blob store reference, use it; otherwise return error + if hasattr(self._bus, "blob_store"): + store: Any = self._bus.blob_store + image_bytes = store.get(image_cid) + except Exception: + pass + if image_bytes is None: + return {"error": "not_found", "message": f"blob {image_cid} not found"} + else: + return {"error": "bad_request", "message": "image_cid or image_b64 required"} + + # Select backend + backend: ImageDescribeBackend | None = None + if backend_name: + backend = self._by_name.get(backend_name) + if backend is None: + return {"error": "bad_request", "message": f"unknown backend: {backend_name}"} + elif self._backends: + backend = self._backends[0] + else: + return {"error": "unavailable", "message": "no image backends configured"} + + result: ImageDescription = await backend.describe(image_bytes, mode=mode) + return { + "output": { + "caption": result.caption, + "tags": result.tags, + "objects": result.objects, + "ocr_text": result.ocr_text, + "backend": result.backend, + "ms": result.ms, + }, + "meta": {}, + } + + def health(self) -> dict: + return { + "service": self.name, + "backends": [b.health() for b in self._backends], + } diff --git a/hearthnet/services/image/generate_service.py b/hearthnet/services/image/generate_service.py new file mode 100644 index 0000000000000000000000000000000000000000..00d37bbfcc6623f91b208a56f9aa5399afa670ab --- /dev/null +++ b/hearthnet/services/image/generate_service.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import base64 +from typing import Any + +from hearthnet.bus.capability import CapabilityDescriptor, RouteRequest +from hearthnet.services.image.backends.base import ImageGenerateBackend, GenerationResult + + +class ImageGenerateService: + """Service wrapping image-generation backends. + + Registers: img.generate@1.0 + """ + + name = "image.generate" + + def __init__( + self, + backends: list[ImageGenerateBackend] | None = None, + bus: Any = None, + ) -> None: + self._backends: list[ImageGenerateBackend] = backends if backends is not None else [] + self._bus = bus + self._by_name: dict[str, ImageGenerateBackend] = {b.name: b for b in self._backends} + + # ── Service registration ────────────────────────────────────────────────── + + def capabilities(self) -> list[tuple]: + return [ + ( + CapabilityDescriptor( + name="img.generate", + max_concurrent=1, + idempotent=False, + timeout_seconds=120, + ), + self.generate, + None, + ), + ] + + def register(self, bus: Any) -> None: + self._bus = bus + for cap, handler, predicate in self.capabilities(): + bus.register_local(cap, handler, predicate) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def generate(self, req: RouteRequest) -> dict: + if not self._backends: + return { + "error": "unavailable", + "message": "no image generation backends installed", + } + + params: dict = req.body.get("input", {}) + prompt: str | None = params.get("prompt") + if not prompt: + return {"error": "bad_request", "message": "prompt required"} + + width: int = int(params.get("width", 512)) + height: int = int(params.get("height", 512)) + steps: int = int(params.get("steps", 20)) + lora: str | None = params.get("lora") + backend_name: str | None = params.get("backend") + + # Clamp dimensions to sane limits + width = max(64, min(width, 2048)) + height = max(64, min(height, 2048)) + steps = max(1, min(steps, 200)) + + # Select backend + backend: ImageGenerateBackend | None = None + if backend_name: + backend = self._by_name.get(backend_name) + if backend is None: + return {"error": "bad_request", "message": f"unknown backend: {backend_name}"} + else: + backend = self._backends[0] + + result: GenerationResult = await backend.generate( + prompt, width=width, height=height, steps=steps, lora=lora + ) + image_b64 = base64.b64encode(result.image_bytes).decode("ascii") + return { + "output": { + "image_b64": image_b64, + "width": result.width, + "height": result.height, + "backend": result.backend, + "ms": result.ms, + }, + "meta": {}, + } + + def health(self) -> dict: + return { + "service": self.name, + "backends": [b.health() for b in self._backends], + "available": len(self._backends) > 0, + } diff --git a/hearthnet/services/llm/tools.py b/hearthnet/services/llm/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..a3bda0e5a3227b42e2622c0f0bf58688f1d15d46 --- /dev/null +++ b/hearthnet/services/llm/tools.py @@ -0,0 +1,263 @@ +"""M21 β€” LLM Tool Calls. + +Provides the data types and ToolExecutor that allows the LLM to call +HearthNet capabilities mid-generation and inject the results back. + +The actual LLM backends live in M04 (service.py); this module is the +orchestration layer that sits on top. +""" +from __future__ import annotations + +import json +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from hearthnet.bus.router import Router + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolDefinition: + """A tool the caller offers to the LLM. + + Translated by the LLM backend into its native tool format (e.g. the + ``tools`` parameter accepted by Ollama, OpenAI, llama.cpp, etc.). + """ + + name: str + """Short identifier visible to the LLM (must be ``[a-zA-Z0-9_-]``).""" + + description: str + """Human-readable description; drives LLM tool selection.""" + + parameters_schema: dict + """JSON Schema object describing the ``arguments`` the LLM should pass.""" + + bound_capability: str | None = None + """If set, :class:`ToolExecutor` will dispatch via the bus using this cap.""" + + bound_version: tuple[int, int] | None = None + """Minimum version for the ``bound_capability``; ``None`` means any.""" + + side_effects: bool = False + """True if execution has side effects β€” affects retry semantics.""" + + def to_openai_schema(self) -> dict: + """Render as an OpenAI / Ollama ``tools`` entry.""" + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters_schema, + }, + } + + +@dataclass(frozen=True) +class ToolCall: + """A request from the LLM to execute a tool.""" + + id: str + """Opaque identifier generated by the LLM; echoed in :class:`ToolResult`.""" + + name: str + """Matches :attr:`ToolDefinition.name`.""" + + arguments: dict + """Deserialized JSON object; validated against the definition schema.""" + + @classmethod + def from_openai_delta(cls, delta: dict) -> "ToolCall": + """Parse an OpenAI-style ``tool_calls[*]`` dict.""" + func = delta.get("function", {}) + raw_args = func.get("arguments", "{}") + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + args = {"_raw": raw_args} + else: + args = raw_args + return cls( + id=delta.get("id") or str(uuid.uuid4()), + name=func.get("name", ""), + arguments=args, + ) + + +@dataclass(frozen=True) +class ToolResult: + """The result of executing a :class:`ToolCall`, fed back to the LLM.""" + + tool_call_id: str + """Must match :attr:`ToolCall.id`.""" + + name: str + """Matches :attr:`ToolCall.name`.""" + + content: str | dict + """Serialisable result; if dict it will be JSON-encoded before injection.""" + + is_error: bool = False + + def to_message(self) -> dict: + """Render as an OpenAI / Ollama ``tool`` role message.""" + body = ( + json.dumps(self.content) if isinstance(self.content, dict) else self.content + ) + return { + "role": "tool", + "tool_call_id": self.tool_call_id, + "name": self.name, + "content": body, + } + + +# --------------------------------------------------------------------------- +# ToolExecutor +# --------------------------------------------------------------------------- + + +class ToolExecutor: + """Orchestrates the tool-call loop between the LLM and the bus. + + Usage:: + + executor = ToolExecutor(bus=node.bus, tools=[my_tool]) + result = await executor.execute(tool_call) + """ + + def __init__( + self, + bus: "Router | None" = None, + tools: list[ToolDefinition] | None = None, + custom_handlers: dict[str, Callable[..., Any]] | None = None, + ) -> None: + self._bus = bus + self._tools: dict[str, ToolDefinition] = { + t.name: t for t in (tools or []) + } + self._custom: dict[str, Callable[..., Any]] = custom_handlers or {} + + # ------------------------------------------------------------------ + # Public helpers + # ------------------------------------------------------------------ + + def definitions(self) -> list[ToolDefinition]: + return list(self._tools.values()) + + def openai_schema(self) -> list[dict]: + """Return all tools rendered as OpenAI ``tools`` array entries.""" + return [t.to_openai_schema() for t in self._tools.values()] + + async def execute(self, call: ToolCall) -> ToolResult: + """Dispatch *call* and return a :class:`ToolResult`. + + Lookup order: + 1. ``custom_handlers[call.name]`` (sync or async callable) + 2. ``ToolDefinition.bound_capability`` dispatched via bus + 3. Fallback: error result + """ + definition = self._tools.get(call.name) + + # 1. Custom handler + if call.name in self._custom: + handler = self._custom[call.name] + try: + import asyncio + + if asyncio.iscoroutinefunction(handler): + out = await handler(**call.arguments) + else: + out = handler(**call.arguments) + return ToolResult( + tool_call_id=call.id, + name=call.name, + content=out if isinstance(out, (str, dict)) else str(out), + is_error=False, + ) + except Exception as exc: # noqa: BLE001 + return ToolResult( + tool_call_id=call.id, + name=call.name, + content={"error": str(exc)}, + is_error=True, + ) + + # 2. Bus-dispatched capability + if definition and definition.bound_capability and self._bus is not None: + try: + from hearthnet.bus.router import RouteRequest + + req = RouteRequest( + capability=definition.bound_capability, + version_req=definition.bound_version or (1, 0), + body={"input": call.arguments}, + caller="tool-executor", + trace_id=str(uuid.uuid4()), + ) + resp = await self._bus.call(req) + out = resp.get("output", resp) + return ToolResult( + tool_call_id=call.id, + name=call.name, + content=out if isinstance(out, (str, dict)) else str(out), + is_error=False, + ) + except Exception as exc: # noqa: BLE001 + return ToolResult( + tool_call_id=call.id, + name=call.name, + content={"error": str(exc)}, + is_error=True, + ) + + # 3. Fallback + return ToolResult( + tool_call_id=call.id, + name=call.name, + content={"error": f"No handler registered for tool '{call.name}'"}, + is_error=True, + ) + + async def run_loop( + self, + initial_messages: list[dict], + max_rounds: int = 5, + ) -> list[dict]: + """Run a full tool-call loop and return the accumulated message list. + + Each round: + 1. Calls the LLM (not implemented here β€” caller provides messages so far) + 2. If the LLM response contains tool calls, executes them + 3. Appends tool results and continues + + This is a *stub* for the protocol loop β€” the actual LLM invocation + happens in the LLM service (M04) which calls back via the bus. This + helper is used by the Gradio UI (M08) and tests. + + Returns the extended message list after all tool rounds are complete. + """ + messages = list(initial_messages) + for _ in range(max_rounds): + # Check for pending tool calls in the last assistant message + last = messages[-1] if messages else {} + tool_calls_raw = last.get("tool_calls", []) + if not tool_calls_raw: + break + # Execute each tool call + tool_results = [] + for tc_raw in tool_calls_raw: + tc = ToolCall.from_openai_delta(tc_raw) + result = await self.execute(tc) + tool_results.append(result) + messages.append(result.to_message()) + # If all results are returned, no more rounds needed + break + return messages diff --git a/hearthnet/services/ocr/__init__.py b/hearthnet/services/ocr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/ocr/backends/__init__.py b/hearthnet/services/ocr/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/ocr/backends/base.py b/hearthnet/services/ocr/backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..3318fe7ddd57c73db8c6bbf658800eece2766e66 --- /dev/null +++ b/hearthnet/services/ocr/backends/base.py @@ -0,0 +1,51 @@ +"""OCR backend protocol and result types.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Protocol, runtime_checkable + + +@dataclass(frozen=True) +class OcrBlock: + text: str + confidence: float + bbox: tuple[int, int, int, int] | None = None + language: str | None = None + + +@dataclass(frozen=True) +class OcrPageResult: + page: int + blocks: list[OcrBlock] + full_text: str + confidence_avg: float + ms: int + + +@dataclass(frozen=True) +class OcrResult: + pages: list[OcrPageResult] + detected_languages: list[str] + backend: str + ms: int + + +@runtime_checkable +class OcrBackend(Protocol): + name: str + supported_languages: list[str] + + async def ocr_image( + self, + image_bytes: bytes, + languages: list[str] | None = None, + ) -> OcrResult: ... + + async def ocr_pdf( + self, + pdf_bytes: bytes, + pages: list[int] | None = None, + languages: list[str] | None = None, + ) -> OcrResult: ... + + def health(self) -> dict: ... diff --git a/hearthnet/services/ocr/backends/tesseract.py b/hearthnet/services/ocr/backends/tesseract.py new file mode 100644 index 0000000000000000000000000000000000000000..c3967db8ebb0343d3f4d7d9eb37712d929cf7d13 --- /dev/null +++ b/hearthnet/services/ocr/backends/tesseract.py @@ -0,0 +1,215 @@ +"""Tesseract OCR backend via pytesseract (optional dependency).""" +from __future__ import annotations + +import asyncio +import io +import subprocess +import time +from typing import Any + + +class TesseractBackend: + name = "tesseract" + + def __init__( + self, + min_confidence: float = 0.5, + data_dir: str | None = None, + ) -> None: + self._min_confidence = min_confidence + self._data_dir = data_dir + self._supported_languages: list[str] | None = None + + @property + def supported_languages(self) -> list[str]: + if self._supported_languages is None: + self._supported_languages = self._read_langs() + return self._supported_languages + + def _read_langs(self) -> list[str]: + try: + result = subprocess.run( + ["tesseract", "--list-langs"], + capture_output=True, + text=True, + timeout=10, + ) + lines = (result.stdout + result.stderr).splitlines() + # Output starts with "List of available tessdata languages..." then one lang per line + return [ln.strip() for ln in lines if ln.strip() and not ln.startswith("List")] + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + return [] + + def health(self) -> dict: + try: + import pytesseract # noqa: F401 + except ImportError: + return {"backend": self.name, "status": "unavailable", "reason": "pytesseract not installed"} + + try: + subprocess.run(["tesseract", "--version"], capture_output=True, timeout=5, check=True) + except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return {"backend": self.name, "status": "unavailable", "reason": "tesseract binary not found"} + + return {"backend": self.name, "status": "ok", "languages": len(self.supported_languages)} + + async def ocr_image( + self, + image_bytes: bytes, + languages: list[str] | None = None, + ) -> Any: + from hearthnet.services.ocr.backends.base import OcrBlock, OcrPageResult, OcrResult + + t0 = time.monotonic() + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, self._ocr_image_sync, image_bytes, languages) + ms = int((time.monotonic() - t0) * 1000) + result.pages[0] = OcrPageResult( + page=result.pages[0].page, + blocks=result.pages[0].blocks, + full_text=result.pages[0].full_text, + confidence_avg=result.pages[0].confidence_avg, + ms=ms, + ) + return OcrResult( + pages=result.pages, + detected_languages=result.detected_languages, + backend=self.name, + ms=ms, + ) + + def _ocr_image_sync(self, image_bytes: bytes, languages: list[str] | None) -> Any: + from hearthnet.services.ocr.backends.base import OcrBlock, OcrPageResult, OcrResult + + try: + import pytesseract + from PIL import Image + except ImportError as e: + raise RuntimeError(f"pytesseract/Pillow not installed: {e}") from e + + t0 = time.monotonic() + image = Image.open(io.BytesIO(image_bytes)) + + kwargs: dict = {} + if self._data_dir: + kwargs["config"] = f"--tessdata-dir {self._data_dir}" + lang_str = "+".join(languages) if languages else None + if lang_str: + kwargs["lang"] = lang_str + + data = pytesseract.image_to_data(image, output_type=pytesseract.Output.DICT, **kwargs) + + blocks: list[OcrBlock] = [] + full_text_parts: list[str] = [] + confidences: list[float] = [] + detected_langs: list[str] = [] + + n = len(data["text"]) + for i in range(n): + word = str(data["text"][i]).strip() + if not word: + continue + conf_raw = data["conf"][i] + conf = float(conf_raw) / 100.0 if conf_raw != -1 else 0.0 + if conf < self._min_confidence and conf_raw != -1: + continue + x, y, w, h = data["left"][i], data["top"][i], data["width"][i], data["height"][i] + blocks.append(OcrBlock(text=word, confidence=conf, bbox=(x, y, w, h), language=None)) + full_text_parts.append(word) + if conf_raw != -1: + confidences.append(conf) + + full_text = " ".join(full_text_parts) + confidence_avg = sum(confidences) / len(confidences) if confidences else 0.0 + ms = int((time.monotonic() - t0) * 1000) + + page = OcrPageResult( + page=1, + blocks=blocks, + full_text=full_text, + confidence_avg=confidence_avg, + ms=ms, + ) + return OcrResult(pages=[page], detected_languages=detected_langs, backend=self.name, ms=ms) + + async def ocr_pdf( + self, + pdf_bytes: bytes, + pages: list[int] | None = None, + languages: list[str] | None = None, + ) -> Any: + from hearthnet.services.ocr.backends.base import OcrResult + + t0 = time.monotonic() + loop = asyncio.get_event_loop() + result_pages = await loop.run_in_executor( + None, self._ocr_pdf_sync, pdf_bytes, pages, languages + ) + ms = int((time.monotonic() - t0) * 1000) + all_langs: list[str] = [] + for p in result_pages: + all_langs.extend(p.detected_languages) + return OcrResult( + pages=[p.pages[0] for p in result_pages], + detected_languages=list(dict.fromkeys(all_langs)), + backend=self.name, + ms=ms, + ) + + def _ocr_pdf_sync( + self, pdf_bytes: bytes, pages: list[int] | None, languages: list[str] | None + ) -> list[Any]: + """Convert PDF pages to images via pdf2image, then OCR each.""" + try: + from pdf2image import convert_from_bytes + except ImportError: + # Fallback: try pypdf text extraction (no image OCR) + return self._ocr_pdf_pypdf(pdf_bytes, pages) + + images = convert_from_bytes(pdf_bytes, dpi=200) + results = [] + for idx, img in enumerate(images, start=1): + if pages and idx not in pages: + continue + buf = io.BytesIO() + img.save(buf, format="PNG") + r = self._ocr_image_sync(buf.getvalue(), languages) + # Re-tag page number + from hearthnet.services.ocr.backends.base import OcrPageResult + + old_page = r.pages[0] + new_page = OcrPageResult( + page=idx, + blocks=old_page.blocks, + full_text=old_page.full_text, + confidence_avg=old_page.confidence_avg, + ms=old_page.ms, + ) + from hearthnet.services.ocr.backends.base import OcrResult + + results.append(OcrResult(pages=[new_page], detected_languages=[], backend=self.name, ms=0)) + return results + + def _ocr_pdf_pypdf(self, pdf_bytes: bytes, pages: list[int] | None) -> list[Any]: + """Best-effort text extraction from PDF using pypdf (no image rendering).""" + try: + import pypdf + except ImportError: + raise RuntimeError("Neither pdf2image nor pypdf is installed") from None + + from hearthnet.services.ocr.backends.base import OcrBlock, OcrPageResult, OcrResult + + reader = pypdf.PdfReader(io.BytesIO(pdf_bytes)) + results = [] + for idx, pdf_page in enumerate(reader.pages, start=1): + if pages and idx not in pages: + continue + text = pdf_page.extract_text() or "" + block = OcrBlock(text=text, confidence=1.0, bbox=None, language=None) + page_result = OcrPageResult( + page=idx, blocks=[block], full_text=text, confidence_avg=1.0, ms=0 + ) + results.append( + OcrResult(pages=[page_result], detected_languages=[], backend=self.name, ms=0) + ) + return results diff --git a/hearthnet/services/ocr/backends/trocr.py b/hearthnet/services/ocr/backends/trocr.py new file mode 100644 index 0000000000000000000000000000000000000000..f3099e3522ec0549b1c88f903922b504552285f7 --- /dev/null +++ b/hearthnet/services/ocr/backends/trocr.py @@ -0,0 +1,132 @@ +"""TrOCR backend via Hugging Face Transformers (optional dependency).""" +from __future__ import annotations + +import asyncio +import io +import time +from typing import Any + + +class TrocrBackend: + name = "trocr" + + def __init__( + self, + model: str = "microsoft/trocr-large-handwritten", + device: str = "auto", + ) -> None: + self._model_name = model + self._device = device + self._processor: Any = None + self._model: Any = None + self._loaded = False + + @property + def supported_languages(self) -> list[str]: + # TrOCR is primarily English/handwriting; can be fine-tuned for others + return ["eng", "deu"] + + def _resolve_device(self) -> str: + if self._device != "auto": + return self._device + try: + import torch + return "cuda" if torch.cuda.is_available() else "cpu" + except ImportError: + return "cpu" + + def _load_model_sync(self) -> None: + from transformers import TrOCRProcessor, VisionEncoderDecoderModel # type: ignore[import] + + device = self._resolve_device() + self._processor = TrOCRProcessor.from_pretrained(self._model_name) + self._model = VisionEncoderDecoderModel.from_pretrained(self._model_name) + self._model.to(device) + self._loaded = True + + async def _ensure_loaded(self) -> None: + if not self._loaded: + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, self._load_model_sync) + + def health(self) -> dict: + try: + import transformers # noqa: F401 + except ImportError: + return {"backend": self.name, "status": "unavailable", "reason": "transformers not installed"} + return {"backend": self.name, "status": "ok", "model": self._model_name} + + def _run_trocr_sync(self, image_bytes: bytes) -> tuple[str, float]: + from PIL import Image # type: ignore[import] + import torch + + device = self._resolve_device() + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + pixel_values = self._processor(images=image, return_tensors="pt").pixel_values.to(device) + with torch.no_grad(): + generated_ids = self._model.generate(pixel_values) + text = self._processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + return text, 1.0 + + async def ocr_image( + self, + image_bytes: bytes, + languages: list[str] | None = None, + ) -> Any: + from hearthnet.services.ocr.backends.base import OcrBlock, OcrPageResult, OcrResult + + await self._ensure_loaded() + t0 = time.monotonic() + loop = asyncio.get_event_loop() + text, confidence = await loop.run_in_executor(None, self._run_trocr_sync, image_bytes) + ms = int((time.monotonic() - t0) * 1000) + + block = OcrBlock(text=text, confidence=confidence, bbox=None, language=None) + page = OcrPageResult( + page=1, blocks=[block], full_text=text, confidence_avg=confidence, ms=ms + ) + return OcrResult(pages=[page], detected_languages=[], backend=self.name, ms=ms) + + async def ocr_pdf( + self, + pdf_bytes: bytes, + pages: list[int] | None = None, + languages: list[str] | None = None, + ) -> Any: + from hearthnet.services.ocr.backends.base import OcrResult + + try: + from pdf2image import convert_from_bytes # type: ignore[import] + except ImportError: + from hearthnet.services.ocr.backends.base import OcrBlock, OcrPageResult + + return OcrResult( + pages=[OcrPageResult(page=1, blocks=[], full_text="", confidence_avg=0.0, ms=0)], + detected_languages=[], + backend=self.name, + ms=0, + ) + + t0 = time.monotonic() + images = convert_from_bytes(pdf_bytes, dpi=200) + page_results = [] + for idx, img in enumerate(images, start=1): + if pages and idx not in pages: + continue + buf = io.BytesIO() + img.save(buf, format="PNG") + partial = await self.ocr_image(buf.getvalue(), languages) + from hearthnet.services.ocr.backends.base import OcrPageResult + + old = partial.pages[0] + page_results.append( + OcrPageResult( + page=idx, + blocks=old.blocks, + full_text=old.full_text, + confidence_avg=old.confidence_avg, + ms=old.ms, + ) + ) + ms = int((time.monotonic() - t0) * 1000) + return OcrResult(pages=page_results, detected_languages=[], backend=self.name, ms=ms) diff --git a/hearthnet/services/ocr/service.py b/hearthnet/services/ocr/service.py new file mode 100644 index 0000000000000000000000000000000000000000..6b415c4e0327100366447ce6ba453462f81049cf --- /dev/null +++ b/hearthnet/services/ocr/service.py @@ -0,0 +1,195 @@ +"""OcrService β€” registers ocr.image@1.0 and ocr.pdf@1.0 on the bus.""" +from __future__ import annotations + +import base64 +import time +from typing import Any + + +class OcrService: + name = "ocr" + version = "1.0" + + def __init__( + self, + backends: list[Any] | None = None, + bus: Any = None, + ) -> None: + if backends is not None: + self._backends = backends + else: + self._backends = self._discover_backends() + if bus is not None: + self.register(bus) + + # ── Backend discovery ───────────────────────────────────────────────────── + + def _discover_backends(self) -> list[Any]: + backends: list[Any] = [] + try: + from hearthnet.services.ocr.backends.tesseract import TesseractBackend + b = TesseractBackend() + if b.health().get("status") == "ok": + backends.append(b) + except Exception: + pass + try: + from hearthnet.services.ocr.backends.trocr import TrocrBackend + b = TrocrBackend() + if b.health().get("status") == "ok": + backends.append(b) + except Exception: + pass + return backends + + def _select_backend( + self, languages: list[str] | None, preferred: str | None + ) -> Any | None: + """Return first healthy backend that supports the requested languages.""" + for backend in self._backends: + if preferred and backend.name != preferred: + continue + h = backend.health() + if h.get("status") != "ok": + continue + if languages: + sup = set(backend.supported_languages) + if not all(lang in sup for lang in languages): + continue + return backend + # Fallback: any healthy backend + for backend in self._backends: + if backend.health().get("status") == "ok": + return backend + return None + + # ── Capability registration ─────────────────────────────────────────────── + + def register(self, bus: Any) -> None: + from hearthnet.bus.capability import CapabilityDescriptor, RouteRequest + + desc_image = CapabilityDescriptor( + name="ocr.image", + version=(1, 0), + stability="stable", + params={"backends": [b.name for b in self._backends]}, + max_concurrent=4, + trust_required="member", + timeout_seconds=60, + idempotent=True, + ) + desc_pdf = CapabilityDescriptor( + name="ocr.pdf", + version=(1, 0), + stability="stable", + params={"backends": [b.name for b in self._backends]}, + max_concurrent=2, + trust_required="member", + timeout_seconds=120, + idempotent=True, + ) + bus.register_capability(desc_image, self._handle_image, self.params_compatible) + bus.register_capability(desc_pdf, self._handle_pdf, self.params_compatible) + + def params_compatible(self, offered: dict, requested: dict) -> bool: + req_backend = requested.get("backend") + if not req_backend: + return True + return req_backend in offered.get("backends", []) + + # ── Handlers ────────────────────────────────────────────────────────────── + + async def _handle_image(self, req: Any) -> dict: + body = req.body if hasattr(req, "body") else req + inp = body.get("input", body) + image_cid: str | None = inp.get("image_cid") + image_b64: str | None = inp.get("image_b64") + languages: list[str] | None = inp.get("languages") + preferred: str | None = inp.get("backend") + + image_bytes = self._resolve_bytes(image_cid, image_b64) + if image_bytes is None: + return {"error": "bad_request", "reason": "Provide image_cid or image_b64"} + + backend = self._select_backend(languages, preferred) + if backend is None: + return { + "error": "backend_unavailable", + "reason": "No healthy OCR backend available", + } + + try: + result = await backend.ocr_image(image_bytes, languages=languages) + except Exception as exc: + return {"error": "internal_error", "reason": str(exc)} + + return self._serialize_result(result) + + async def _handle_pdf(self, req: Any) -> dict: + body = req.body if hasattr(req, "body") else req + inp = body.get("input", body) + pdf_cid: str | None = inp.get("pdf_cid") + pdf_b64: str | None = inp.get("pdf_b64") + pages: list[int] | None = inp.get("pages") + languages: list[str] | None = inp.get("languages") + preferred: str | None = inp.get("backend") + + pdf_bytes = self._resolve_bytes(pdf_cid, pdf_b64) + if pdf_bytes is None: + return {"error": "bad_request", "reason": "Provide pdf_cid or pdf_b64"} + + backend = self._select_backend(languages, preferred) + if backend is None: + return { + "error": "backend_unavailable", + "reason": "No healthy OCR backend available", + } + + try: + result = await backend.ocr_pdf(pdf_bytes, pages=pages, languages=languages) + except Exception as exc: + return {"error": "internal_error", "reason": str(exc)} + + return self._serialize_result(result) + + # ── Helpers ─────────────────────────────────────────────────────────────── + + def _resolve_bytes(self, cid: str | None, b64: str | None) -> bytes | None: + if b64: + try: + return base64.b64decode(b64) + except Exception: + return None + if cid: + # CID resolution requires blob store; left as integration point + return None + return None + + @staticmethod + def _serialize_result(result: Any) -> dict: + pages_out = [] + for page in result.pages: + blocks_out = [ + { + "text": b.text, + "confidence": b.confidence, + "bbox": list(b.bbox) if b.bbox else None, + "language": b.language, + } + for b in page.blocks + ] + pages_out.append( + { + "page": page.page, + "blocks": blocks_out, + "full_text": page.full_text, + "confidence_avg": page.confidence_avg, + "ms": page.ms, + } + ) + return { + "pages": pages_out, + "detected_languages": result.detected_languages, + "backend": result.backend, + "ms": result.ms, + } diff --git a/hearthnet/services/rerank/__init__.py b/hearthnet/services/rerank/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/rerank/backends/__init__.py b/hearthnet/services/rerank/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/rerank/backends/base.py b/hearthnet/services/rerank/backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..98c38acbb451fd9dae9b6b81cd2df245fbf9231d --- /dev/null +++ b/hearthnet/services/rerank/backends/base.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Protocol, runtime_checkable + + +@dataclass +class RerankDoc: + id: str + text: str + + +@dataclass +class RerankedDoc: + id: str + score: float + + +@dataclass +class RerankRequest: + query: str + docs: list[RerankDoc] + top_k: int | None = None + params: dict = field(default_factory=dict) + + +@dataclass +class RerankResponse: + ranked: list[RerankedDoc] + meta: dict = field(default_factory=dict) + + +@runtime_checkable +class RerankBackend(Protocol): + name: str + + async def rerank(self, request: RerankRequest) -> RerankResponse: ... + + def health(self) -> dict: ... diff --git a/hearthnet/services/rerank/backends/bge.py b/hearthnet/services/rerank/backends/bge.py new file mode 100644 index 0000000000000000000000000000000000000000..f825f44dd1074e24eff03c527008a4afb39b90a7 --- /dev/null +++ b/hearthnet/services/rerank/backends/bge.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import asyncio +import time +from typing import Any + +from hearthnet.services.rerank.backends.base import ( + RerankBackend, + RerankDoc, + RerankRequest, + RerankResponse, + RerankedDoc, +) + + +class BgeRerankerBackend: + """Cross-encoder reranker using BAAI/bge-reranker models.""" + + name = "bge_reranker" + + def __init__( + self, + model_id: str = "BAAI/bge-reranker-v2-m3", + device: str = "auto", + max_batch: int = 32, + ) -> None: + self._model_id = model_id + self._device = device + self._max_batch = max_batch + self._encoder = None + self._loaded = False + self._load_error: str | None = None + + def _load(self) -> bool: + if self._loaded: + return True + if self._load_error: + return False + try: + from sentence_transformers import CrossEncoder # type: ignore[import-untyped] + import torch # type: ignore[import-untyped] + + device = self._device + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + + self._encoder = CrossEncoder(self._model_id, device=device) + self._device = device + self._loaded = True + return True + except ImportError as exc: + self._load_error = f"sentence_transformers not installed: {exc}" + return False + except Exception as exc: + self._load_error = str(exc) + return False + + async def rerank(self, request: RerankRequest) -> RerankResponse: + if not self._load(): + return RerankResponse( + ranked=[RerankedDoc(id=d.id, score=0.0) for d in request.docs], + meta={"error": self._load_error, "backend": self.name}, + ) + + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, self._sync_rerank, request) + return result + + def _sync_rerank(self, request: RerankRequest) -> RerankResponse: + t0 = time.monotonic() + pairs = [[request.query, doc.text] for doc in request.docs] + scores: list[float] = [] + + # Process in batches + for i in range(0, len(pairs), self._max_batch): + batch = pairs[i : i + self._max_batch] + batch_scores = self._encoder.predict(batch) # type: ignore[union-attr] + scores.extend(float(s) for s in batch_scores) + + ranked = sorted( + [RerankedDoc(id=doc.id, score=score) for doc, score in zip(request.docs, scores)], + key=lambda x: x.score, + reverse=True, + ) + if request.top_k is not None: + ranked = ranked[: request.top_k] + + return RerankResponse( + ranked=ranked, + meta={ + "backend": self.name, + "model": self._model_id, + "ms": int((time.monotonic() - t0) * 1000), + "doc_count": len(request.docs), + }, + ) + + def health(self) -> dict: + return { + "backend": self.name, + "model": self._model_id, + "loaded": self._loaded, + "available": self._load_error is None, + "error": self._load_error, + } diff --git a/hearthnet/services/rerank/backends/cross_encoder.py b/hearthnet/services/rerank/backends/cross_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..d1eee58781e189ee8cde8885a68481d5dde74189 --- /dev/null +++ b/hearthnet/services/rerank/backends/cross_encoder.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from hearthnet.services.rerank.backends.bge import BgeRerankerBackend +from hearthnet.services.rerank.backends.base import RerankRequest, RerankResponse, RerankedDoc + + +class CrossEncoderBackend(BgeRerankerBackend): + """Cross-encoder reranker using ms-marco-MiniLM model.""" + + name = "cross_encoder" + + def __init__( + self, + model_id: str = "cross-encoder/ms-marco-MiniLM-L-6-v2", + device: str = "auto", + max_batch: int = 32, + ) -> None: + super().__init__(model_id=model_id, device=device, max_batch=max_batch) + + def health(self) -> dict: + h = super().health() + h["backend"] = self.name + return h diff --git a/hearthnet/services/rerank/service.py b/hearthnet/services/rerank/service.py new file mode 100644 index 0000000000000000000000000000000000000000..4384501eaf265578be414768b204643eea2bb8ef --- /dev/null +++ b/hearthnet/services/rerank/service.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from typing import Any + +from hearthnet.bus.capability import CapabilityDescriptor, RouteRequest +from hearthnet.constants import RERANK_MAX_DOCS +from hearthnet.services.rerank.backends.base import ( + RerankBackend, + RerankDoc, + RerankRequest, + RerankResponse, + RerankedDoc, +) + + +class RerankService: + """Service that exposes reranking via the capability bus. + + Registers: rerank.text@1.0 + """ + + name = "rerank" + + def __init__( + self, + backends: list[RerankBackend] | None = None, + bus: Any = None, + ) -> None: + if backends is not None: + self._backends: list[RerankBackend] = backends + else: + # Lazy-load defaults; failures are graceful + self._backends = self._default_backends() + self._bus = bus + self._by_name: dict[str, RerankBackend] = {b.name: b for b in self._backends} + + @staticmethod + def _default_backends() -> list[RerankBackend]: + backends: list[RerankBackend] = [] + try: + from hearthnet.services.rerank.backends.bge import BgeRerankerBackend + backends.append(BgeRerankerBackend()) + except Exception: + pass + try: + from hearthnet.services.rerank.backends.cross_encoder import CrossEncoderBackend + backends.append(CrossEncoderBackend()) + except Exception: + pass + return backends + + # ── Service registration ────────────────────────────────────────────────── + + def capabilities(self) -> list[tuple]: + return [ + ( + CapabilityDescriptor( + name="rerank.text", + max_concurrent=4, + idempotent=True, + timeout_seconds=30, + ), + self.rerank_text, + None, + ), + ] + + def register(self, bus: Any) -> None: + self._bus = bus + for cap, handler, predicate in self.capabilities(): + bus.register_local(cap, handler, predicate) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def rerank_text(self, req: RouteRequest) -> dict: + params: dict = req.body.get("input", {}) + + query: str | None = params.get("query") + if not query: + return {"error": "bad_request", "message": "query required"} + + raw_docs: list[dict] = params.get("docs", []) + if not raw_docs: + return {"error": "bad_request", "message": "docs required"} + + if len(raw_docs) > RERANK_MAX_DOCS: + return { + "error": "bad_request", + "message": f"docs exceeds limit of {RERANK_MAX_DOCS}", + } + + docs = [RerankDoc(id=d.get("id", str(i)), text=d.get("text", "")) for i, d in enumerate(raw_docs)] + + top_k: int | None = params.get("top_k") + model: str | None = params.get("model") + + # Select backend + backend: RerankBackend | None = None + if model: + backend = self._by_name.get(model) + if backend is None: + return {"error": "bad_request", "message": f"unknown backend: {model}"} + elif self._backends: + backend = self._backends[0] + else: + # Fallback: return docs sorted by order with score=0 + ranked = [{"id": d.id, "score": 0.0} for d in docs] + if top_k is not None: + ranked = ranked[:top_k] + return { + "output": {"ranked": ranked, "meta": {"backend": "none", "warning": "no reranker available"}}, + "meta": {}, + } + + rerank_req = RerankRequest(query=query, docs=docs, top_k=top_k) + resp: RerankResponse = await backend.rerank(rerank_req) + + return { + "output": { + "ranked": [{"id": r.id, "score": r.score} for r in resp.ranked], + "meta": resp.meta, + }, + "meta": {}, + } + + def health(self) -> dict: + return { + "service": self.name, + "backends": [b.health() for b in self._backends], + "available": len(self._backends) > 0, + } diff --git a/hearthnet/services/speech/__init__.py b/hearthnet/services/speech/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/speech/backends/__init__.py b/hearthnet/services/speech/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/speech/backends/base.py b/hearthnet/services/speech/backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..8bff9eb5a098c523047225fc128e2d99cda7e78a --- /dev/null +++ b/hearthnet/services/speech/backends/base.py @@ -0,0 +1,65 @@ +"""STT and TTS backend protocol and result types.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + + +# ── STT ─────────────────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class SttSegment: + start_seconds: float + end_seconds: float + text: str + language: str | None = None + confidence: float | None = None + + +@dataclass(frozen=True) +class SttResult: + segments: list[SttSegment] + full_text: str + detected_language: str + backend: str + ms: int + + +@runtime_checkable +class SttBackend(Protocol): + name: str + + async def transcribe( + self, + audio_bytes: bytes, + language: str | None = None, + translate_to_en: bool = False, + ) -> SttResult: ... + + def health(self) -> dict: ... + + +# ── TTS ─────────────────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class TtsResult: + audio_bytes: bytes + audio_format: str + duration_seconds: float + backend: str + ms: int + + +@runtime_checkable +class TtsBackend(Protocol): + name: str + + async def synthesize( + self, + text: str, + voice: str | None = None, + language: str = "de", + format: str = "ogg_vorbis", + ) -> TtsResult: ... + + def health(self) -> dict: ... diff --git a/hearthnet/services/speech/backends/edge_tts.py b/hearthnet/services/speech/backends/edge_tts.py new file mode 100644 index 0000000000000000000000000000000000000000..efbf4f6d114fe38595f544d17455b0fc2a9f12c9 --- /dev/null +++ b/hearthnet/services/speech/backends/edge_tts.py @@ -0,0 +1,86 @@ +"""Edge TTS backend (Microsoft Edge text-to-speech via edge-tts package).""" +from __future__ import annotations + +import asyncio +import io +import time +from typing import Any + +from hearthnet.constants import STT_MAX_AUDIO_SECONDS + + +class EdgeTtsBackend: + name = "edge_tts" + requires_internet = True + + def __init__(self) -> None: + pass + + def health(self) -> dict: + try: + import edge_tts # noqa: F401 + return {"backend": self.name, "status": "ok", "requires_internet": True} + except ImportError: + return { + "backend": self.name, + "status": "unavailable", + "reason": "edge-tts not installed", + } + + async def synthesize( + self, + text: str, + voice: str | None = "de-DE-KatjaNeural", + language: str = "de", + format: str = "ogg_vorbis", + ) -> Any: + from hearthnet.services.speech.backends.base import TtsResult + + try: + import edge_tts # type: ignore[import] + except ImportError: + raise RuntimeError("edge-tts not installed") from None + + selected_voice = voice or _default_voice(language) + t0 = time.monotonic() + + communicate = edge_tts.Communicate(text, selected_voice) + buf = io.BytesIO() + async for chunk in communicate.stream(): + if chunk["type"] == "audio": + buf.write(chunk["data"]) + + audio_bytes = buf.getvalue() + ms = int((time.monotonic() - t0) * 1000) + + # Estimate duration from audio length (rough: ~32kbps ogg) + duration_seconds = min( + len(audio_bytes) / (32 * 1024 / 8), + float(STT_MAX_AUDIO_SECONDS), + ) + + # edge-tts natively outputs mp3; wrap in chosen format label + return TtsResult( + audio_bytes=audio_bytes, + audio_format="mp3", # edge-tts always outputs mp3 + duration_seconds=duration_seconds, + backend=self.name, + ms=ms, + ) + + +def _default_voice(language: str) -> str: + _VOICES: dict[str, str] = { + "de": "de-DE-KatjaNeural", + "en": "en-US-JennyNeural", + "fr": "fr-FR-DeniseNeural", + "es": "es-ES-ElviraNeural", + "it": "it-IT-ElsaNeural", + "nl": "nl-NL-ColetteNeural", + "pl": "pl-PL-ZofiaNeural", + "ru": "ru-RU-SvetlanaNeural", + "uk": "uk-UA-PolinaNeural", + "ar": "ar-SA-ZariyahNeural", + "tr": "tr-TR-EmelNeural", + } + return _VOICES.get(language, "en-US-JennyNeural") diff --git a/hearthnet/services/speech/backends/whisper_local.py b/hearthnet/services/speech/backends/whisper_local.py new file mode 100644 index 0000000000000000000000000000000000000000..10ca95b36c548eb85a1ee379386c3d316ceed281 --- /dev/null +++ b/hearthnet/services/speech/backends/whisper_local.py @@ -0,0 +1,156 @@ +"""Whisper local STT backend (openai-whisper or faster-whisper).""" +from __future__ import annotations + +import asyncio +import io +import tempfile +import time +from typing import Any + + +class WhisperBackend: + name = "whisper" + + def __init__( + self, + model_size: str = "base", + device: str = "auto", + ) -> None: + self._model_size = model_size + self._device = device + self._model: Any = None + self._backend_lib: str | None = None # "openai_whisper" or "faster_whisper" + + def _resolve_device(self) -> str: + if self._device != "auto": + return self._device + try: + import torch + return "cuda" if torch.cuda.is_available() else "cpu" + except ImportError: + return "cpu" + + def health(self) -> dict: + # Prefer faster_whisper, fall back to openai whisper + try: + import faster_whisper # noqa: F401 + return {"backend": self.name, "status": "ok", "lib": "faster_whisper", "model": self._model_size} + except ImportError: + pass + try: + import whisper # noqa: F401 + return {"backend": self.name, "status": "ok", "lib": "openai_whisper", "model": self._model_size} + except ImportError: + pass + return { + "backend": self.name, + "status": "unavailable", + "reason": "Neither openai-whisper nor faster-whisper is installed", + } + + def _load_model_sync(self) -> None: + device = self._resolve_device() + try: + from faster_whisper import WhisperModel # type: ignore[import] + + self._model = WhisperModel(self._model_size, device=device) + self._backend_lib = "faster_whisper" + return + except ImportError: + pass + import whisper # type: ignore[import] + + self._model = whisper.load_model(self._model_size, device=device) + self._backend_lib = "openai_whisper" + + async def _ensure_loaded(self) -> None: + if self._model is None: + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, self._load_model_sync) + + async def transcribe( + self, + audio_bytes: bytes, + language: str | None = None, + translate_to_en: bool = False, + ) -> Any: + from hearthnet.services.speech.backends.base import SttResult, SttSegment + + await self._ensure_loaded() + t0 = time.monotonic() + + loop = asyncio.get_event_loop() + segments, detected_lang = await loop.run_in_executor( + None, self._transcribe_sync, audio_bytes, language, translate_to_en + ) + ms = int((time.monotonic() - t0) * 1000) + full_text = " ".join(s.text for s in segments) + return SttResult( + segments=segments, + full_text=full_text, + detected_language=detected_lang or "unknown", + backend=self.name, + ms=ms, + ) + + def _transcribe_sync( + self, + audio_bytes: bytes, + language: str | None, + translate_to_en: bool, + ) -> tuple[list[Any], str | None]: + from hearthnet.services.speech.backends.base import SttSegment + + # Write to temp file because whisper expects file path + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(audio_bytes) + tmp_path = tmp.name + + segments_out: list[SttSegment] = [] + detected: str | None = None + + try: + if self._backend_lib == "faster_whisper": + task = "translate" if translate_to_en else "transcribe" + segs, info = self._model.transcribe( + tmp_path, + language=language, + task=task, + ) + detected = info.language + for seg in segs: + segments_out.append( + SttSegment( + start_seconds=seg.start, + end_seconds=seg.end, + text=seg.text.strip(), + language=detected, + confidence=None, + ) + ) + else: + # openai-whisper + task = "translate" if translate_to_en else "transcribe" + kwargs: dict = {"task": task} + if language: + kwargs["language"] = language + result = self._model.transcribe(tmp_path, **kwargs) + detected = result.get("language") + for seg in result.get("segments", []): + segments_out.append( + SttSegment( + start_seconds=float(seg["start"]), + end_seconds=float(seg["end"]), + text=str(seg["text"]).strip(), + language=detected, + confidence=None, + ) + ) + finally: + import os + try: + os.unlink(tmp_path) + except OSError: + pass + + return segments_out, detected diff --git a/hearthnet/services/speech/stt_service.py b/hearthnet/services/speech/stt_service.py new file mode 100644 index 0000000000000000000000000000000000000000..efc5d29cc5421d834be2905ec99233dfd2ed9bab --- /dev/null +++ b/hearthnet/services/speech/stt_service.py @@ -0,0 +1,116 @@ +"""SttService β€” registers stt.transcribe@1.0 on the bus.""" +from __future__ import annotations + +import base64 +from typing import Any + +from hearthnet.constants import STT_MAX_AUDIO_SECONDS + + +class SttService: + name = "stt" + version = "1.0" + + def __init__( + self, + backends: list[Any] | None = None, + bus: Any = None, + ) -> None: + if backends is not None: + self._backends = backends + else: + self._backends = self._discover_backends() + if bus is not None: + self.register(bus) + + # ── Backend discovery ───────────────────────────────────────────────────── + + def _discover_backends(self) -> list[Any]: + backends: list[Any] = [] + try: + from hearthnet.services.speech.backends.whisper_local import WhisperBackend + b = WhisperBackend() + if b.health().get("status") == "ok": + backends.append(b) + except Exception: + pass + return backends + + def _select_backend(self) -> Any | None: + for backend in self._backends: + if backend.health().get("status") == "ok": + return backend + return None + + # ── Capability registration ─────────────────────────────────────────────── + + def register(self, bus: Any) -> None: + from hearthnet.bus.capability import CapabilityDescriptor + + desc = CapabilityDescriptor( + name="stt.transcribe", + version=(1, 0), + stability="stable", + params={"backends": [b.name for b in self._backends]}, + max_concurrent=2, + trust_required="member", + timeout_seconds=STT_MAX_AUDIO_SECONDS + 30, + idempotent=True, + ) + bus.register_capability(desc, self._handle_transcribe, self.params_compatible) + + def params_compatible(self, offered: dict, requested: dict) -> bool: + req_backend = requested.get("backend") + if not req_backend: + return True + return req_backend in offered.get("backends", []) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def _handle_transcribe(self, req: Any) -> dict: + body = req.body if hasattr(req, "body") else req + inp = body.get("input", body) + + audio_b64: str | None = inp.get("audio_b64") + language: str | None = inp.get("language") + translate_to_en: bool = bool(inp.get("translate_to_en", False)) + + if not audio_b64: + return {"error": "bad_request", "reason": "audio_b64 is required"} + + try: + audio_bytes = base64.b64decode(audio_b64) + except Exception: + return {"error": "bad_request", "reason": "audio_b64 is not valid base64"} + + backend = self._select_backend() + if backend is None: + return { + "error": "backend_unavailable", + "reason": "No healthy STT backend available", + } + + try: + result = await backend.transcribe( + audio_bytes, language=language, translate_to_en=translate_to_en + ) + except Exception as exc: + return {"error": "internal_error", "reason": str(exc)} + + segments_out = [ + { + "start_seconds": s.start_seconds, + "end_seconds": s.end_seconds, + "text": s.text, + "language": s.language, + "confidence": s.confidence, + } + for s in result.segments + ] + return { + "segments": segments_out, + "full_text": result.full_text, + "detected_language": result.detected_language, + "backend": result.backend, + "ms": result.ms, + } diff --git a/hearthnet/services/speech/tts_service.py b/hearthnet/services/speech/tts_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ef86e2c6655f565739f1c38cc7af77bbd0ab8e85 --- /dev/null +++ b/hearthnet/services/speech/tts_service.py @@ -0,0 +1,112 @@ +"""TtsService β€” registers tts.synthesize@1.0 on the bus.""" +from __future__ import annotations + +import base64 +from typing import Any + +from hearthnet.constants import TRANSLATION_MAX_CHARS + + +class TtsService: + name = "tts" + version = "1.0" + + def __init__( + self, + backends: list[Any] | None = None, + bus: Any = None, + ) -> None: + if backends is not None: + self._backends = backends + else: + self._backends = self._discover_backends() + if bus is not None: + self.register(bus) + + # ── Backend discovery ───────────────────────────────────────────────────── + + def _discover_backends(self) -> list[Any]: + backends: list[Any] = [] + try: + from hearthnet.services.speech.backends.edge_tts import EdgeTtsBackend + b = EdgeTtsBackend() + if b.health().get("status") == "ok": + backends.append(b) + except Exception: + pass + return backends + + def _select_backend(self, preferred: str | None = None) -> Any | None: + for backend in self._backends: + if preferred and backend.name != preferred: + continue + if backend.health().get("status") == "ok": + return backend + for backend in self._backends: + if backend.health().get("status") == "ok": + return backend + return None + + # ── Capability registration ─────────────────────────────────────────────── + + def register(self, bus: Any) -> None: + from hearthnet.bus.capability import CapabilityDescriptor + + desc = CapabilityDescriptor( + name="tts.synthesize", + version=(1, 0), + stability="stable", + params={"backends": [b.name for b in self._backends]}, + max_concurrent=4, + trust_required="member", + timeout_seconds=60, + idempotent=True, + ) + bus.register_capability(desc, self._handle_synthesize, self.params_compatible) + + def params_compatible(self, offered: dict, requested: dict) -> bool: + req_backend = requested.get("backend") + if not req_backend: + return True + return req_backend in offered.get("backends", []) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def _handle_synthesize(self, req: Any) -> dict: + body = req.body if hasattr(req, "body") else req + inp = body.get("input", body) + + text: str = inp.get("text", "") + voice: str | None = inp.get("voice") + language: str = inp.get("language", "de") + fmt: str = inp.get("format", "ogg_vorbis") + preferred: str | None = inp.get("backend") + + if not text: + return {"error": "bad_request", "reason": "text is required"} + + if len(text) > TRANSLATION_MAX_CHARS: + return { + "error": "bad_request", + "reason": f"Text too long: {len(text)} > {TRANSLATION_MAX_CHARS} chars", + } + + backend = self._select_backend(preferred) + if backend is None: + return { + "error": "backend_unavailable", + "reason": "No healthy TTS backend available", + } + + try: + result = await backend.synthesize(text, voice=voice, language=language, format=fmt) + except Exception as exc: + return {"error": "internal_error", "reason": str(exc)} + + return { + "audio_b64": base64.b64encode(result.audio_bytes).decode(), + "audio_format": result.audio_format, + "duration_seconds": result.duration_seconds, + "backend": result.backend, + "ms": result.ms, + } diff --git a/hearthnet/services/translation/__init__.py b/hearthnet/services/translation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/translation/backends/__init__.py b/hearthnet/services/translation/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hearthnet/services/translation/backends/base.py b/hearthnet/services/translation/backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..7c1e98d5340e02880d3e541cef91d14a9d0d1755 --- /dev/null +++ b/hearthnet/services/translation/backends/base.py @@ -0,0 +1,32 @@ +"""Translation backend protocol and result types.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + + +@dataclass(frozen=True) +class TranslationResult: + text: str + from_lang: str + to_lang: str + backend: str + ms: int + + +@runtime_checkable +class TranslationBackend(Protocol): + name: str + supported_pairs: list[tuple[str, str]] + + async def translate( + self, + text: str, + from_lang: str, + to_lang: str, + domain: str | None = None, + ) -> TranslationResult: ... + + async def detect_language(self, text: str) -> str: ... + + def health(self) -> dict: ... diff --git a/hearthnet/services/translation/backends/nllb.py b/hearthnet/services/translation/backends/nllb.py new file mode 100644 index 0000000000000000000000000000000000000000..e8e1d498dd4414100bf2a0170231dc46a4165e0b --- /dev/null +++ b/hearthnet/services/translation/backends/nllb.py @@ -0,0 +1,248 @@ +"""NLLB translation backend (facebook/nllb-200-distilled-600M) via transformers.""" +from __future__ import annotations + +import asyncio +import hashlib +import time +from collections import OrderedDict +from typing import Any + +from hearthnet.constants import TRANSLATION_MAX_CHARS +from hearthnet.services.translation.backends.base import TranslationResult + +# Top 20 common language pairs (ISO 639-1 β†’ NLLB flores code mapping) +_ISO_TO_FLORES: dict[str, str] = { + "de": "deu_Latn", + "en": "eng_Latn", + "fr": "fra_Latn", + "es": "spa_Latn", + "it": "ita_Latn", + "nl": "nld_Latn", + "pl": "pol_Latn", + "pt": "por_Latn", + "ru": "rus_Cyrl", + "uk": "ukr_Cyrl", + "ar": "arb_Arab", + "tr": "tur_Latn", + "cs": "ces_Latn", + "sv": "swe_Latn", + "da": "dan_Latn", + "fi": "fin_Latn", + "ro": "ron_Latn", + "hu": "hun_Latn", + "sk": "slk_Latn", + "hr": "hrv_Latn", + # Low German / Plattdeutsch β€” not in standard NLLB; map to nds_Latn if present + "nds": "nds_Latn", +} + +_TOP_PAIRS: list[tuple[str, str]] = [ + ("de", "en"), ("en", "de"), + ("fr", "en"), ("en", "fr"), + ("es", "en"), ("en", "es"), + ("it", "en"), ("en", "it"), + ("nl", "en"), ("en", "nl"), + ("pl", "en"), ("en", "pl"), + ("ru", "en"), ("en", "ru"), + ("uk", "en"), ("en", "uk"), + ("ar", "en"), ("en", "ar"), + ("tr", "en"), ("en", "tr"), +] + +_LRU_MAX = 1000 + + +class _LRUCache: + def __init__(self, maxsize: int = _LRU_MAX) -> None: + self._cache: OrderedDict[str, TranslationResult] = OrderedDict() + self._maxsize = maxsize + + def get(self, key: str) -> TranslationResult | None: + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key] + return None + + def put(self, key: str, value: TranslationResult) -> None: + if key in self._cache: + self._cache.move_to_end(key) + self._cache[key] = value + if len(self._cache) > self._maxsize: + self._cache.popitem(last=False) + + +class NllbBackend: + name = "nllb" + + def __init__( + self, + model: str = "facebook/nllb-200-distilled-600M", + device: str = "auto", + max_chars: int = TRANSLATION_MAX_CHARS, + ) -> None: + self._model_name = model + self._device = device + self._max_chars = max_chars + self._pipeline: Any = None + self._loaded = False + self._cache = _LRUCache() + # Batching + self._batch_queue: list[tuple[asyncio.Future[str], str, str, str]] = [] + self._batch_task: asyncio.Task[None] | None = None + + @property + def supported_pairs(self) -> list[tuple[str, str]]: + return _TOP_PAIRS + + def _resolve_device(self) -> str: + if self._device != "auto": + return self._device + try: + import torch + return "cuda" if torch.cuda.is_available() else "cpu" + except ImportError: + return "cpu" + + def _load_sync(self) -> None: + from transformers import pipeline # type: ignore[import] + + device = self._resolve_device() + device_id = 0 if device == "cuda" else -1 + self._pipeline = pipeline( + "translation", + model=self._model_name, + device=device_id, + ) + self._loaded = True + + async def _ensure_loaded(self) -> None: + if not self._loaded: + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, self._load_sync) + + def health(self) -> dict: + try: + import transformers # noqa: F401 + except ImportError: + return {"backend": self.name, "status": "unavailable", "reason": "transformers not installed"} + return {"backend": self.name, "status": "ok", "model": self._model_name} + + async def detect_language(self, text: str) -> str: + """Best-effort language detection using langdetect (optional).""" + try: + from langdetect import detect # type: ignore[import] + + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, detect, text) + return str(result) + except Exception: + return "unknown" + + def _cache_key(self, text: str, from_lang: str, to_lang: str) -> str: + h = hashlib.sha256(f"{text}|{from_lang}|{to_lang}".encode()).hexdigest() + return h + + def _translate_sync(self, text: str, from_lang: str, to_lang: str) -> str: + src_flores = _ISO_TO_FLORES.get(from_lang) + tgt_flores = _ISO_TO_FLORES.get(to_lang) + if src_flores is None or tgt_flores is None: + raise ValueError(f"Unsupported language pair: {from_lang} β†’ {to_lang}") + output = self._pipeline( + text, + src_lang=src_flores, + tgt_lang=tgt_flores, + max_length=512, + ) + return str(output[0]["translation_text"]) + + async def translate( + self, + text: str, + from_lang: str, + to_lang: str, + domain: str | None = None, + ) -> TranslationResult: + if len(text) > self._max_chars: + raise ValueError(f"Text too long: {len(text)} > {self._max_chars}") + + # Handle auto-detect + if from_lang == "auto": + from_lang = await self.detect_language(text) + + cache_key = self._cache_key(text, from_lang, to_lang) + cached = self._cache.get(cache_key) + if cached is not None: + return cached + + await self._ensure_loaded() + + # Enqueue and batch within 100ms window + translated_text = await self._enqueue_or_translate(text, from_lang, to_lang) + + t0 = time.monotonic() + result = TranslationResult( + text=translated_text, + from_lang=from_lang, + to_lang=to_lang, + backend=self.name, + ms=int((time.monotonic() - t0) * 1000) + 1, + ) + self._cache.put(cache_key, result) + return result + + async def _enqueue_or_translate( + self, text: str, from_lang: str, to_lang: str + ) -> str: + """Add to batch queue and wait up to 100ms for batch processing.""" + loop = asyncio.get_event_loop() + future: asyncio.Future[str] = loop.create_future() + self._batch_queue.append((future, text, from_lang, to_lang)) + + if self._batch_task is None or self._batch_task.done(): + self._batch_task = asyncio.create_task(self._flush_batch_after_delay()) + + return await future + + async def _flush_batch_after_delay(self) -> None: + await asyncio.sleep(0.1) # 100ms window + if not self._batch_queue: + return + batch = self._batch_queue[:8] + self._batch_queue = self._batch_queue[8:] + loop = asyncio.get_event_loop() + + # Group by (from_lang, to_lang) for efficient batching + groups: dict[tuple[str, str], list[tuple[asyncio.Future[str], str]]] = {} + for future, text, fl, tl in batch: + key = (fl, tl) + groups.setdefault(key, []).append((future, text)) + + for (fl, tl), items in groups.items(): + texts = [t for _, t in items] + futures_grp = [f for f, _ in items] + try: + results = await loop.run_in_executor( + None, self._translate_batch_sync, texts, fl, tl + ) + for f, r in zip(futures_grp, results): + if not f.done(): + f.set_result(r) + except Exception as exc: + for f in futures_grp: + if not f.done(): + f.set_exception(exc) + + def _translate_batch_sync( + self, texts: list[str], from_lang: str, to_lang: str + ) -> list[str]: + src_flores = _ISO_TO_FLORES.get(from_lang) + tgt_flores = _ISO_TO_FLORES.get(to_lang) + if src_flores is None or tgt_flores is None: + raise ValueError(f"Unsupported language pair: {from_lang} β†’ {to_lang}") + outputs = self._pipeline( + texts, + src_lang=src_flores, + tgt_lang=tgt_flores, + max_length=512, + ) + return [str(o["translation_text"]) for o in outputs] diff --git a/hearthnet/services/translation/service.py b/hearthnet/services/translation/service.py new file mode 100644 index 0000000000000000000000000000000000000000..fc11572358488ac0667f4686983d057d688273a7 --- /dev/null +++ b/hearthnet/services/translation/service.py @@ -0,0 +1,120 @@ +"""TranslationService β€” registers trans.text@1.0 on the bus.""" +from __future__ import annotations + +from typing import Any + +from hearthnet.constants import TRANSLATION_MAX_CHARS + + +class TranslationService: + name = "translation" + version = "1.0" + + def __init__( + self, + backends: list[Any] | None = None, + bus: Any = None, + ) -> None: + if backends is not None: + self._backends = backends + else: + self._backends = self._discover_backends() + if bus is not None: + self.register(bus) + + # ── Backend discovery ───────────────────────────────────────────────────── + + def _discover_backends(self) -> list[Any]: + backends: list[Any] = [] + try: + from hearthnet.services.translation.backends.nllb import NllbBackend + b = NllbBackend() + if b.health().get("status") == "ok": + backends.append(b) + except Exception: + pass + return backends + + def _select_backend( + self, + from_lang: str, + to_lang: str, + preferred: str | None = None, + ) -> Any | None: + for backend in self._backends: + if preferred and backend.name != preferred: + continue + if backend.health().get("status") != "ok": + continue + pairs = backend.supported_pairs + if from_lang == "auto" or (from_lang, to_lang) in pairs: + return backend + # Any healthy backend as fallback + for backend in self._backends: + if backend.health().get("status") == "ok": + return backend + return None + + # ── Capability registration ─────────────────────────────────────────────── + + def register(self, bus: Any) -> None: + from hearthnet.bus.capability import CapabilityDescriptor + + desc = CapabilityDescriptor( + name="trans.text", + version=(1, 0), + stability="stable", + params={"backends": [b.name for b in self._backends], "max_chars": TRANSLATION_MAX_CHARS}, + max_concurrent=4, + trust_required="member", + timeout_seconds=30, + idempotent=True, + ) + bus.register_capability(desc, self._handle_translate, self.params_compatible) + + def params_compatible(self, offered: dict, requested: dict) -> bool: + req_backend = requested.get("backend") + if not req_backend: + return True + return req_backend in offered.get("backends", []) + + # ── Handler ─────────────────────────────────────────────────────────────── + + async def _handle_translate(self, req: Any) -> dict: + body = req.body if hasattr(req, "body") else req + inp = body.get("input", body) + + text: str = inp.get("text", "") + from_lang: str = inp.get("from_lang", "auto") + to_lang: str | None = inp.get("to_lang") + domain: str | None = inp.get("domain") + preferred: str | None = inp.get("backend") + + if not to_lang: + return {"error": "bad_request", "reason": "to_lang is required"} + + if len(text) > TRANSLATION_MAX_CHARS: + return { + "error": "bad_request", + "reason": f"Text too long: {len(text)} > {TRANSLATION_MAX_CHARS} chars", + } + + backend = self._select_backend(from_lang, to_lang, preferred) + if backend is None: + return { + "error": "backend_unavailable", + "reason": "No healthy translation backend available", + } + + try: + result = await backend.translate(text, from_lang=from_lang, to_lang=to_lang, domain=domain) + except Exception as exc: + return {"error": "internal_error", "reason": str(exc)} + + return { + "text": result.text, + "from_lang": result.from_lang, + "to_lang": result.to_lang, + "backend": result.backend, + "ms": result.ms, + } diff --git a/hearthnet/transport/server.py b/hearthnet/transport/server.py index 299b77199fc21186c770d0aae0dbcdf9799b96bd..e07b7c52acb8a3e25e0fbe5042024fb4413f8eaf 100644 --- a/hearthnet/transport/server.py +++ b/hearthnet/transport/server.py @@ -63,6 +63,7 @@ class HttpServer: self._server_task: asyncio.Task | None = None self._uvicorn_server = None self._app = None + self._ws_pubsub: Any = None # WebsocketPubSub, lazy-initialised def build_app(self) -> Any: """Build and return the FastAPI application.""" @@ -233,9 +234,65 @@ class HttpServer: except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) from exc + # ── WebSocket pubsub endpoint (X06) ────────────────────────────────── + # Lazy import keeps websocket.py optional β€” server still works without it. + try: + from hearthnet.transport.websocket import ( # noqa: PLC0415 + WebSocketSession, + WebsocketPubSub, + ) + from fastapi import WebSocket as _WS # noqa: PLC0415 + from starlette.websockets import WebSocketDisconnect as _WSDisc # noqa: PLC0415 + + if self._ws_pubsub is None: + self._ws_pubsub = WebsocketPubSub() + + _pubsub = self._ws_pubsub + + @app.websocket("/pubsub/v1/ws/{topic}") + async def ws_pubsub(websocket: _WS, topic: str): + await websocket.accept() + session = WebSocketSession(websocket) + _pubsub.subscribe(topic, session) + try: + while True: + frame = await session.receive_frame() + if frame is None: + break + # Acknowledge ACK frames; ignore others silently + if frame.type == "ack": + up_to = frame.data.get("upto", 0) + await session.send_ack(up_to) + except _WSDisc: + pass + except Exception: + pass + finally: + _pubsub.unsubscribe(topic, session) + await session.close() + + except ImportError: + pass # websockets / starlette WS not available; endpoint not registered + self._app = app return app + async def publish_event(self, topic: str, event: str, data: dict) -> int: + """ + Fan-out *event*/*data* to all WebSocket sessions subscribed to *topic*. + + Returns the number of sessions that received the message. + Returns 0 if the WebSocket pubsub is not initialised. + """ + if self._ws_pubsub is None: + return 0 + try: + return await self._ws_pubsub.publish(topic, event, data) + except Exception as exc: + import logging as _logging # noqa: PLC0415 + _logging.getLogger(__name__).warning("HttpServer.publish_event error: %s", exc) + return 0 + async def start(self) -> None: """Start uvicorn in background task.""" if not HAS_FASTAPI: diff --git a/hearthnet/transport/websocket.py b/hearthnet/transport/websocket.py new file mode 100644 index 0000000000000000000000000000000000000000..602d999014a7dcbb6fbde4e037f041885bee9078 --- /dev/null +++ b/hearthnet/transport/websocket.py @@ -0,0 +1,241 @@ +"""WebSocket upgrade for bidirectional streaming (X06).""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, AsyncIterator + +logger = logging.getLogger(__name__) + +# Optional websockets import (client-side only) +try: + import websockets # type: ignore[import] + HAS_WEBSOCKETS = True +except ImportError: + websockets = None # type: ignore[assignment] + HAS_WEBSOCKETS = False + +# Optional FastAPI/Starlette WebSocket import (server-side) +try: + from starlette.websockets import WebSocket, WebSocketDisconnect, WebSocketState # type: ignore[import] + HAS_STARLETTE_WS = True +except ImportError: + WebSocket = None # type: ignore[assignment] + WebSocketDisconnect = None # type: ignore[assignment] + WebSocketState = None # type: ignore[assignment] + HAS_STARLETTE_WS = False + + +# ── Dataclasses ─────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class WsClientFrame: + """A parsed frame received from a WebSocket client.""" + + type: str # "ack" | "tool_result" | "cancel" + data: dict + + +# ── Server side ─────────────────────────────────────────────────────────────── + + +class WebSocketSession: + """Wraps a Starlette/FastAPI WebSocket from the server's perspective.""" + + def __init__(self, ws: Any, keypair: Any = None) -> None: + if ws is None: + raise ValueError("ws must be a non-None WebSocket object") + self._ws = ws + self._keypair = keypair + self.session_id: str = str(uuid.uuid4()) + self.connected_at: float = time.time() + self._seq: int = 0 + + async def send_event( + self, + event: str, + data: dict, + seq: int | None = None, + ) -> None: + """Send a JSON frame to the client.""" + if seq is None: + self._seq += 1 + seq = self._seq + frame = json.dumps({"event": event, "data": data, "seq": seq}) + try: + await self._ws.send_text(frame) + except Exception as exc: + logger.debug("WebSocketSession.send_event error: %s", exc) + raise + + async def receive_frame(self) -> WsClientFrame | None: + """Receive and parse one inbound JSON frame. Returns None on disconnect.""" + try: + raw = await self._ws.receive_text() + except Exception: + return None + try: + obj = json.loads(raw) + except json.JSONDecodeError: + logger.warning("WebSocketSession: malformed JSON from client") + return None + frame_type = obj.get("type", "") + # Strip type key, rest is data + data = {k: v for k, v in obj.items() if k != "type"} + return WsClientFrame(type=frame_type, data=data) + + async def send_ack(self, up_to: int) -> None: + """Send a server-to-client ACK frame.""" + frame = json.dumps({"event": "ack", "data": {"up_to": up_to}, "seq": self._seq}) + try: + await self._ws.send_text(frame) + except Exception as exc: + logger.debug("WebSocketSession.send_ack error: %s", exc) + + async def close(self, code: int = 1000) -> None: + """Close the WebSocket with the given close code.""" + try: + await self._ws.close(code=code) + except Exception: + pass + + +# ── Client side ─────────────────────────────────────────────────────────────── + + +class WebSocketClient: + """Client-side WebSocket wrapper. Requires the `websockets` library.""" + + def __init__(self, base_url: str, keypair: Any = None) -> None: + if not HAS_WEBSOCKETS: + raise ImportError("Install websockets: pip install websockets") + # Convert http(s) to ws(s) + self._base_url = ( + base_url + .rstrip("/") + .replace("https://", "wss://") + .replace("http://", "ws://") + ) + self._keypair = keypair + self._conn: Any = None # websockets.WebSocketClientProtocol + + async def connect(self, path: str) -> None: + """Establish a WebSocket connection to *path* on the server.""" + if not HAS_WEBSOCKETS: + raise ImportError("Install websockets: pip install websockets") + url = f"{self._base_url}/{path.lstrip('/')}" + self._conn = await websockets.connect(url) # type: ignore[union-attr] + + async def stream(self, event_iterator: Any) -> AsyncIterator[dict]: + """ + Send frames from *event_iterator* to the server and yield parsed + server frames until the connection closes. + """ + if self._conn is None: + raise RuntimeError("Not connected. Call connect() first.") + + async def _sender() -> None: + try: + async for item in event_iterator: + await self._conn.send(json.dumps(item)) + except Exception as exc: + logger.debug("WebSocketClient._sender error: %s", exc) + + sender_task = asyncio.create_task(_sender()) + try: + async for raw in self._conn: + try: + yield json.loads(raw) + except json.JSONDecodeError: + logger.warning("WebSocketClient: malformed JSON from server") + finally: + sender_task.cancel() + try: + await sender_task + except asyncio.CancelledError: + pass + + async def send_tool_result(self, tool_call_id: str, body: dict) -> None: + """Send a tool result frame mid-stream.""" + if self._conn is None: + raise RuntimeError("Not connected.") + frame = json.dumps({"type": "tool_result", "tool_call_id": tool_call_id, "body": body}) + await self._conn.send(frame) + + async def cancel(self) -> None: + """Send a cancel frame to the server.""" + if self._conn is None: + return + try: + await self._conn.send(json.dumps({"type": "cancel"})) + except Exception: + pass + + async def close(self) -> None: + """Close the WebSocket connection gracefully.""" + if self._conn is not None: + try: + await self._conn.close() + except Exception: + pass + finally: + self._conn = None + + +# ── PubSub fanout ───────────────────────────────────────────────────────────── + + +class WebsocketPubSub: + """ + In-process publish/subscribe for WebSocket sessions. + + subscribe/unsubscribe are synchronous; publish is async and fan-outs to all + sessions registered for the topic. + """ + + def __init__(self) -> None: + self._subscriptions: dict[str, set[WebSocketSession]] = {} + self._lock = asyncio.Lock() + + def subscribe(self, topic: str, ws_session: WebSocketSession) -> None: + """Register *ws_session* to receive messages on *topic*.""" + if topic not in self._subscriptions: + self._subscriptions[topic] = set() + self._subscriptions[topic].add(ws_session) + + def unsubscribe(self, topic: str, ws_session: WebSocketSession) -> None: + """Remove *ws_session* from *topic*.""" + if topic in self._subscriptions: + self._subscriptions[topic].discard(ws_session) + if not self._subscriptions[topic]: + del self._subscriptions[topic] + + async def publish(self, topic: str, event: str, data: dict) -> int: + """ + Fan-out *event*/*data* to all sessions subscribed to *topic*. + Returns the number of sessions that received the message. + """ + async with self._lock: + sessions = list(self._subscriptions.get(topic, [])) + + dead: list[WebSocketSession] = [] + delivered = 0 + for session in sessions: + try: + await session.send_event(event, data) + delivered += 1 + except Exception: + dead.append(session) + + # Clean up disconnected sessions + if dead: + async with self._lock: + for session in dead: + self.unsubscribe(topic, session) + + return delivered diff --git a/tasks.md b/tasks.md index fd0925ca9433a9abff6a21994f2f83d0a8672a5c..fab73248925b6486577278c70e5f951e7c4b1083 100644 --- a/tasks.md +++ b/tasks.md @@ -1,355 +1,107 @@ -# HearthNet Task Plan and Spec Coverage - -## Current Truth β€” Phase 1 + Phase 2 Core Implementation - -All M01-M13 and X01-X04 modules have been implemented. The codebase now has real implementations -for identity (Ed25519), event log (SQLite), discovery (mDNS/UDP), transport (FastAPI), -LLM backends (Ollama/llama.cpp/HF/OpenAI-compat), RAG (chunker+Chroma), blobs (BLAKE3), -embedding (sentence-transformers), CLI (click), onboarding (invite/QR), marketplace -(event-sourced), chat (event-backed), emergency (async probe loop), and UI (Gradio tabs). - -Implementation policy: no mocks in implementation paths, no security pragmas, real local-first -model backends, OpenAI only as an opt-in online fallback, UI behavior/wording adheres to specs. - -## Phase 1 Demo: Done - -- [x] Import the Hugging Face Space into the local workspace. -- [x] Read all spec files for implementation direction. -- [x] Preserve existing browser mesh prototypes as supporting material. -- [x] Build Hugging Face-compatible Gradio `app.py`. -- [x] Add ZeroGPU startup probe for HF runtime compatibility. -- [x] Create `hearthnet/` Python package for the Phase 1 demo slice. -- [x] Add `HearthNetController` over `HearthNode`. -- [x] Add facades for RAG, chat, and marketplace workflows. -- [x] Add `CapabilityBus`, registry, router, health tracking, in-memory traces. -- [x] Add simulated peer discovery and manifest ingestion. -- [x] Add deterministic demo nodes for anchor, hearth, and spark profiles. -- [x] Add demo services for all capabilities. -- [x] Add emergency mode state and manual probe handling. -- [x] Add focused tests for routing, failover, emergency mode, and controller snapshots. -- [x] Add `ruff`, `bandit`, `pylint`, `mypy`, and `pytest` config. - -## Phase 2 Core Implementation: Done (June 2026) - -- [x] **X04 Config** β€” `hearthnet/config.py`: typed frozen Config, TOML load/save, XDG path resolution, validation -- [x] **hearthnet/constants.py** β€” all numeric constants for health, pruning, emergency, blobs, RAG, LLM -- [x] **X03 Observability** β€” `hearthnet/observability/`: JSON logging, prometheus metrics (optional), trace ring buffer, doctor checks -- [x] **M01 Identity** β€” `hearthnet/identity/`: Ed25519 KeyPair, canonical JSON, sign/verify, node/community manifests -- [x] **X02 Events** β€” `hearthnet/events/`: SQLite event log, LamportClock, ReplayEngine, MaterialisedView, SnapshotStore, SyncClient/Server -- [x] **M07 File/Blobs** β€” `hearthnet/blobs/`: BLAKE3 CID store, chunking, transfer; `hearthnet/services/file/`: file.read/list/advertise/put capabilities -- [x] **M11 Embedding** β€” `hearthnet/services/embedding/`: EmbeddingService, SimpleHashBackend (no-dep test), SentenceTransformerBackend -- [x] **M05 RAG** β€” `hearthnet/services/rag/`: chunker (sliding window, paragraph-aware), CorpusStore (Chroma + in-memory fallback), IngestPipeline, RagService -- [x] **M12 CLI** β€” `hearthnet/cli.py`: click-based CLI with init/run/status/caps/call/doctor/trace/export; `hearthnet/__main__.py` -- [x] **M09 Emergency** β€” async probe loop (DNS + HTTP), debounce, anti-flap, async pub/sub StateBus -- [x] **M02 Discovery** β€” `hearthnet/discovery/`: MdnsAnnouncer/Browser (zeroconf), UdpAnnouncer/Listener (multicast), enhanced PeerRegistry with async subscribe -- [x] **M04 LLM** β€” `hearthnet/services/llm/`: LlmService with OllamaBackend, LlamaCppBackend, HfLocalBackend, OpenAICompatBackend (online-only fallback), tokenizers -- [x] **M06 Marketplace** β€” `hearthnet/services/marketplace/`: Post dataclass, MarketplaceView (event-sourced), MarketplaceService with event-log + demo fallback -- [x] **M10 Chat** β€” `hearthnet/services/chat/`: ChatMessage, ChatView (event-sourced), DeliveryManager, ChatService with event-log + demo fallback -- [x] **M13 Onboarding** β€” `hearthnet/ui/onboarding.py`: invite encode/decode, QR generation, create/join/redeem community flows -- [x] **X01 Transport** β€” `hearthnet/transport/`: FastAPI HttpServer (12 endpoints), HttpClient with signing, SSE streams, TLS cert generation, PinnedCerts -- [x] **M03 Bus upgrades** β€” `hearthnet/bus/schema.py` (JSON Schema validation), `hearthnet/bus/trace.py` (CallTraceEvent, TraceHook) -- [x] **M08 UI** β€” `hearthnet/ui/app.py` + `hearthnet/ui/tabs/`: Ask/Chat/Marketplace/Files/Emergency/Settings tabs; bus-only wiring - -## Spec Coverage Matrix - -| Spec | Status | What exists | -| --- | --- | --- | -| `GLOSSARY.md` | βœ… implemented | NodeID/CID/ULID helpers, XDG paths in config, canonical constants | -| `CAPABILITY_CONTRACT.md` | βœ… implemented | Signed envelopes (X01), schema validation (bus/schema.py), BLAKE3 hashes, SSE streaming | -| M01 Identity | βœ… implemented | `hearthnet/identity/`: Ed25519 keys, canonical JSON, sign/verify, node + community manifests | -| M02 Discovery | βœ… implemented | `hearthnet/discovery/`: mDNS (zeroconf), UDP broadcast, PeerRegistry with async events | -| M03 Capability Bus | βœ… implemented | Registry, router, health, schema validation, trace hook, streaming support | -| M04 LLM | βœ… implemented | Ollama, llama.cpp, HF Transformers backends; llm.chat + llm.complete; OpenAI online fallback | -| M05 RAG | βœ… implemented | Chunker (paragraph/sliding window), CorpusStore (Chroma + in-memory), IngestPipeline, bus-via-embed | -| M06 Marketplace | βœ… implemented | Event-sourced with MarketplaceView, post/list/expire/search; demo fallback | -| M07 File/Blobs | βœ… implemented | BLAKE3 CID store, 256KB chunking, BlobStore, TransferManager, FileService | -| M08 UI | βœ… implemented | `hearthnet/ui/`: UiApp, 6 tabs (Ask/Chat/Market/Files/Emergency/Settings), bus-only | -| M09 Emergency | βœ… implemented | Async probe loop, DNS+HTTP probes, anti-flap, async StateBus, async Detector | -| M10 Chat | βœ… implemented | ChatView (event-sourced), DeliveryManager, ChatService; demo fallback | -| M11 Embedding | βœ… implemented | embed.text capability, SimpleHashBackend (no-dep), SentenceTransformerBackend | -| M12 CLI | βœ… implemented | `hearthnet/cli.py`: init/run/status/caps/call/doctor/trace/export; console script | -| M13 Onboarding | βœ… implemented | InviteBlob, encode/decode, QR PNG, create/join/redeem community, Gradio wizard | -| X01 Transport | βœ… implemented | FastAPI server (12 endpoints), HttpClient (signed), SSE, TLS cert, PinnedCerts | -| X02 Events | βœ… implemented | SQLite WAL log, LamportClock, ReplayEngine, MaterialisedView, SnapshotStore, SyncClient | -| X03 Observability | βœ… implemented | JSON logging (rotating), prometheus metrics (optional), trace ring buffer, doctor checks | -| X04 Config | βœ… implemented | Typed frozen Config, TOML load/save/validate, XDG path resolution | - -## Remaining Gaps / Phase 2+ Work - -- [ ] Wire event log into HearthNode and controller (currently services use demo fallback) -- [ ] Wire M01 identity into node startup (auto-generate keypair, sign manifests) -- [ ] Wire X01 FastAPI transport into node for real inter-node calls +ο»Ώ# HearthNet β€” Task Tracker + +## Status Summary (June 2026) + +All Phase 1 (M01-M13, X01-X04), Phase 2 (M14-M25, X05-X07), and Phase 3 experimental +(M26-M31) modules are implemented. 62 tests pass (51 unit/integration + 11 E2E Playwright). + +--- + +## Phase 1 β€” Complete + +- [x] M01 Identity (Ed25519, canonical JSON, node/community manifests) +- [x] M02 Discovery (mDNS, UDP broadcast, PeerRegistry with async events) +- [x] M03 Capability bus (schema validation, router, health, traces) +- [x] M04 LLM (Ollama, llama.cpp, HF Transformers backends; OpenAI online fallback) +- [x] M05 RAG (chunker, ChromaDB + in-memory, IngestPipeline, bus embed) +- [x] M06 Marketplace (event-sourced, post/list/expire/search) +- [x] M07 File blobs (BLAKE3 CID store, chunking, TransferManager) +- [x] M08 UI (Gradio, 6 tabs: Ask/Chat/Marketplace/Files/Emergency/Settings) +- [x] M09 Emergency (async probe loop, DNS+HTTP, anti-flap, StateBus) +- [x] M10 Chat (event-sourced, ChatView, DeliveryManager) +- [x] M11 Embedding (embed.text, SimpleHashBackend, SentenceTransformerBackend) +- [x] M12 CLI (click, ask/node info/caps/call/doctor/trace) +- [x] M13 Onboarding (InviteBlob, QR, create/join/redeem community) +- [x] X01 Transport (FastAPI server 12 endpoints, HttpClient, SSE, TLS) +- [x] X02 Events (SQLite WAL, LamportClock, ReplayEngine, MaterialisedView, SnapshotStore) +- [x] X03 Observability (structured JSON logging, prometheus metrics optional, trace ring buffer) +- [x] X04 Config (typed frozen Config, TOML load/save, XDG paths, env overlay) + +--- + +## Phase 2 β€” Complete + +- [x] M14 Federation (FederationManifest, bilateral peering, FederationService) +- [x] M15 Relay tier (RelayClient, NAT traversal, keepalive, push token registry) +- [x] M16 Capability tokens (hntoken://v1/ Ed25519 JWS-style, AuthService) +- [x] M17 OCR (Tesseract + TrOCR backends, image/pdf capabilities) +- [x] M18 Translation (NLLB backend, LRU cache, 4000-char limit) +- [x] M19 STT/TTS (WhisperBackend local STT, EdgeTtsBackend synthesis) +- [x] M20 Vision (Florence-2 image describe, generate placeholder) +- [x] M21 Tool calls (ToolDefinition, ToolCall, ToolResult, ToolExecutor, run_loop) +- [x] M22 Mobile native (MobileInviteBlob, hnapp:// deep links, MobilePushService) +- [x] M23 E2E encryption (X3DH, Double Ratchet fixed bug, envelope, prekeys) +- [x] M24 Reranking (BGE + CrossEncoder, 100-doc limit, bus integration) +- [x] M25 Group chat (ThreadService, ThreadViewStore, event-sourced) +- [x] X05 DHT (Kademlia, 256-bucket routing table, KademliaNode, bootstrap) +- [x] X06 WebSocket (WebSocketSession, WebSocketClient, WebsocketPubSub) +- [x] X07 Federated metrics (NodeMetricsTick, MetricsAggregator, OTLP export) + +--- + +## Phase 3 β€” Experimental Stubs (feature-flag gated) + +All enabled via config.research.* flags (all default False). + +- [x] M26 Distributed inference (ShardDescriptor, Pipeline, PipelineOrchestrator) +- [x] M27 MoE routing (ExpertDescriptor, ExpertRegistry, MoeRouter) +- [x] M28 Federated learning (FedLearnCoordinator, RoundManifest) +- [x] M29 LoRa beacons (32-byte frame encoding, LoraBeaconService) +- [x] M30 Evidence graph (Claim, ClaimStore, Attestation, Dispute; EBKH import) +- [x] M31 Civil defense NRW (Alert, RoleCertificate, AuditChain, CivilDefenseService) + +--- + +## Quality Gates β€” All Passing + +- [x] ruff β€” no lint errors +- [x] bandit β€” 0 HIGH findings, intentional nosec items documented +- [x] mypy β€” passes (optional deps handled with TYPE_CHECKING guards) +- [x] pylint β€” no blocking issues +- [x] pytest β€” 62/62 pass (51 unit + 11 E2E Playwright) + +--- + +## Test Suites + +| File | Tests | Coverage | +|------|-------|----------| +| tests/test_phase1_routing.py | 8 | Bus routing, failover, capabilities | +| tests/test_phase1_emergency_snapshot.py | 5 | Emergency mode, controller snapshot | +| tests/test_phase2_modules.py | 23 | M14-M25, X05-X07 | +| tests/test_phase3_experimental.py | 15 | M26-M31, ResearchConfig | +| tests/test_e2e_playwright.py | 11 | Gradio UI E2E (real browser, Playwright) | + +--- + +## Architecture Notes + +- All services implement health() -> dict returning {"status": "ok" | "unavailable"} +- All service handlers receive RouteRequest(capability, version_req, body, caller, trace_id) +- Response format: {"output": {...}, "meta": {}} +- No mocks in implementation paths; heavy optional deps fail gracefully +- OpenAI only as opt-in online fallback β€” never the default local path +- No security-tool suppression pragmas except narrow reviewed nosec comments + +--- + +## Known Remaining Gaps + +- [ ] Wire real event log (X02) into HearthNode on startup (services still use demo/fallback) +- [ ] Wire X01 FastAPI transport into node for real inter-node HTTP calls - [ ] Wire M02 mDNS/UDP discovery into node startup -- [ ] Add JSON Schema definitions to capability descriptors (currently empty dicts) -- [ ] Replace SHA-256 placeholder hashes in bus/capability.py with BLAKE3 (blake3 package) -- [ ] Add contract conformance test suite for capability descriptors/envelopes -- [ ] Add real TLS for inter-node HTTPS calls (currently verify=False) -- [ ] Add X02 gossip sync between real nodes -- [ ] Add M13 onboarding first-run wizard integration with CLI -- [ ] Add M12 invite/RAG CLI commands after M01/M05/M13 integration -- [ ] Add M08 mobile static app (hearthnet/ui/mobile/) -- [ ] Live UI updates via async subscriptions (websocket/SSE push) -- [ ] Phase 2: relay tier, DHT, federation, E2E encryption, LoRa - -## Phase 3 Research/Protocol Milestones - -- [ ] Relay tier. -- [ ] DHT discovery. -- [ ] Federation protocol. -- [ ] E2E encryption. -- [ ] LoRa beacons. -- [ ] Federated metrics. -- [ ] Federated learning. -- [ ] Distributed inference. - - -## Phase 1 Demo: Done - -- [x] Import the Hugging Face Space into the local workspace. -- [x] Read `docs/00-OVERVIEW.md`, `docs/prd_v2.md`, `docs/roadmap.md`, `docs/CAPABILITY_CONTRACT.md`, `docs/GLOSSARY.md`, M01-M13, and X01-X04 for implementation direction. -- [x] Preserve existing browser mesh prototypes as supporting material. -- [x] Build Hugging Face-compatible Gradio `app.py`. -- [x] Add ZeroGPU startup probe for HF runtime compatibility. -- [x] Create `hearthnet/` Python package for the Phase 1 demo slice. -- [x] Model the system of concern as local-first community AI resilience. -- [x] Add `HearthNetController` over `HearthNode`. -- [x] Add facades for RAG, chat, and marketplace workflows. -- [x] Add `CapabilityBus`, registry, router, health tracking, in-memory traces, and in-memory transport. -- [x] Add simulated peer discovery and manifest ingestion. -- [x] Add deterministic demo nodes for anchor, hearth, and spark profiles. -- [x] Add demo services for `llm.chat`, `rag.query`, `rag.ingest`, `market.post`, `market.list`, `chat.send`, and `chat.history`. -- [x] Add emergency mode state and manual probe handling. -- [x] Add Gradio tabs for overview, mesh, AI/RAG, marketplace/chat/emergency, and architecture trace. -- [x] Replace app-level fake AI answers with real local Hugging Face Transformers inference and explicit OpenAI online fallback. -- [x] Add focused tests for routing, failover, emergency mode, and controller snapshots. -- [x] Add `ruff`, `bandit`, `pylint`, `mypy`, and `pytest` config. -- [x] Run quality gates and tests. -- [x] Push Phase 1 Space to `https://huggingface.co/spaces/build-small-hackathon/HearthNet`. - -## Spec Coverage Matrix - -| Spec | Current status | What exists now | Main gaps | -| --- | --- | --- | --- | -| `GLOSSARY.md` | stub | Basic aliases and error literals in `hearthnet/types.py`. | Enforced ID formats, ULID helpers, canonical constants, XDG paths. | -| `CAPABILITY_CONTRACT.md` | demo | Capability descriptors, names, versions, params, routing body shape. | Signed wire envelopes, JSON Schema validation, BLAKE3 schema hashes, streaming/SSE, canonical errors, conformance suite. | -| M01 Identity | missing | Demo node IDs and unsigned manifest dictionaries. | Ed25519 keys, canonical JSON signing, verification, community manifests, TLS certs, tokens. | -| M02 Discovery | demo | `PeerRegistry`, simulated in-memory mesh discovery, manifest import. | mDNS, UDP broadcast, async peer events, manifest fetch/verify, relay path. | -| M03 Capability Bus | demo | Registry, router, local/remote in-memory calls, health scoring, traces. | Schema validation, trust enforcement, streaming, subscriptions, HTTP transport, full trace contract. | -| M04 LLM | demo | Echo-like `llm.chat` service with model params. | `llm.complete`, Ollama/llama.cpp/HF backends, token streaming, cancellation, token accounting. | -| M05 RAG | demo | In-memory keyword query/ingest with chunks. | Chunker, PDF ingestion, Chroma/vector store, embed-via-bus, corpus listing, citations from blobs/events. | -| M06 Marketplace | demo | In-memory `market.post` and `market.list`. | Event-sourced posts, expiry/search, Lamport clocks, replay views, trust/moderation, idempotency. | -| M07 File/Blobs | missing | None. | BLAKE3 CID store, chunking, file service, transfer manager, pinning, GC, `file.*` capabilities. | -| M08 UI | demo | Gradio `app.py` demo dashboard and topology SVG. | `hearthnet/ui/` package, bus-only UI wiring, files/settings tabs, mobile static app, live subscriptions. | -| M09 Emergency | stub | `StateBus`, manual `Detector.apply_probe_results`, capability deregistration demo. | Async probe loop, debounce/anti-flap, DNS/HTTP targets, config-driven probes, UI subscriptions. | -| M10 Chat | stub | In-memory `ChatService` with send/history. | Signed event log, delivery manager, materialized views, read receipts, attachments/blob integration. | -| M11 Embedding | missing | None; RAG uses substring scoring. | `embed.text` capability, backend abstraction, test backend, vector normalization, RAG/marketplace bus use. | -| M12 CLI | missing | None. | `hearthnet/cli.py`, `__main__.py`, console script, status/caps/call/doctor/invite/rag commands. | -| M13 Onboarding | missing | Demo UI text only. | Invite encode/decode, QR flow, create/join community, config persistence, first-run handoff. | -| X01 Transport | demo shim | `InMemoryTransport` for local tests. | FastAPI server/client, signed HTTP envelopes, SSE streaming, rate limiting, TLS/pinning, endpoints. | -| X02 Events | missing | In-memory service lists and UUIDs. | SQLite event log, Lamport clocks, replay, snapshots, gossip/sync, event-backed marketplace/chat. | -| X03 Observability | demo shim | In-memory bus traces in topology snapshots. | Structured logging, metrics, trace ring buffer API, `/metrics`, `/trace/recent`, doctor checks. | -| X04 Config | missing | Hardcoded defaults in constructors/dataclasses. | `hearthnet/config.py`, constants, TOML load/save, validation, path resolution, environment overrides. | - -## Immediate Correction Tasks: No-Mock Local-First Alignment - -- [ ] Audit implementation paths for mocks, fake responses, placeholder AI, and unlabeled simulations; replace with real local-first behavior or explicit unavailable/degraded states. -- [ ] Remove or reject security pragmas, blanket ignores, and tool suppressions; fix findings directly or document narrow reviewed exceptions. -- [ ] Implement real local-first LLM backend selection with Ollama/llama.cpp/local HF support before any online provider. -- [ ] Add OpenAI fallback only as explicit online/degraded mode with config gating, visible UI labeling, and tests proving it is never the default local path. -- [ ] Improve Gradio UI polish: clearer status hierarchy, better empty/loading/error states, cleaner layout density, and honest capability labels. -- [ ] Add spec-adherence tests or checks that map implemented capabilities to M01-M13, X01-X04, `CAPABILITY_CONTRACT.md`, and `GLOSSARY.md`. -- [ ] Add visible Phase 1 boundary labels in the Space explaining which modules are demo/stub/missing. -- [ ] Add tests that assert the UI/demo does not overclaim real transport, identity, event sync, or offline production readiness. -- [ ] Add `hearthnet/constants.py` for hardcoded health/pruning/emergency thresholds. -- [ ] Add minimal `hearthnet/config.py` Phase 1 defaults consumed by node, bus, emergency, and UI. -- [ ] Add tiny event/Lamport shim or rename marketplace/chat data clearly as in-memory demo events. -- [ ] Add observability shim that exports in-memory traces with an explicit note that Prometheus/doctor are deferred. -- [ ] Add a note to `docs/00-OVERVIEW.md` that current repo paths use `docs/X01-...` while the overview still describes the target reference tree. - -## Module Backlog - -### Contract and Glossary - -- [ ] Add contract conformance tests for capability descriptors, request/response envelopes, errors, manifests, and schema hashes. -- [ ] Replace SHA-256 placeholder schema hashes with BLAKE3 once the dependency is introduced. -- [ ] Add shared identifier helpers for NodeID, CommunityID, TraceID, EventID, CID, WallClock, Signature, and SchemaHash. -- [ ] Replace hardcoded demo IDs with glossary-conformant helpers. - -### M01 Identity - -- [ ] Implement `hearthnet/identity/keys.py`. -- [ ] Implement signed node/community manifests. -- [ ] Implement canonical JSON signing and verification tests. -- [ ] Add trust levels and revoked/expired/invalid signature handling. -- [ ] Add token/TLS certificate roadmap tasks for Phase 2. - -### M02 Discovery - -- [ ] Replace simulated discovery with mDNS advertisement/listener. -- [ ] Add UDP broadcast fallback. -- [ ] Verify remote manifests through M01 before registry import. -- [ ] Emit peer up/down/update events. -- [ ] Add relay discovery stub for Phase 2. - -### M03 Capability Bus - -- [ ] Add JSON Schema validation for capability requests/responses. -- [ ] Add trust checks and authorization errors. -- [ ] Add streaming call support. -- [ ] Add subscription/pub-sub support. -- [ ] Add sticky session and quarantine tests beyond the demo route tests. -- [ ] Add trace events matching the capability contract. - -### M04 LLM - -- [ ] Split LLM service into `services/llm/service.py` and backend package. -- [ ] Implement `llm.chat` and `llm.complete` contract handlers. -- [ ] Add Ollama backend. -- [ ] Add llama.cpp backend. -- [x] Add app-level local Hugging Face Transformers backend for Space inference. -- [ ] Move local Hugging Face Transformers backend behind `llm.chat` on the bus. -- [ ] Add OpenAI backend only for explicit online/degraded fallback mode. -- [ ] Add tests proving local backends are preferred and OpenAI is never used unless configured. -- [ ] Add streaming, cancellation, timeout, and token accounting tests. - -### M05 RAG - -- [ ] Add chunker and ingest pipeline. -- [ ] Add persistent corpus store. -- [ ] Add `rag.list_corpora`. -- [ ] Add embedding-via-bus using M11. -- [ ] Add citation metadata from file/blob/event sources. -- [ ] Add RAG query/ingest/list tests. - -### M06 Marketplace - -- [ ] Back marketplace posts with X02 events. -- [ ] Add post/list/search/expire handlers. -- [ ] Add replay/materialized view. -- [ ] Add Lamport/idempotency behavior. -- [ ] Add trust/moderation checks. - -### M07 File and Blobs - -- [ ] Implement BLAKE3 CID blob store. -- [ ] Implement chunking and transfer. -- [ ] Implement file service with `file.read`, `file.list`, and `file.advertise`. -- [ ] Add pinning and garbage collection. -- [ ] Add file/blob tests. - -### M08 UI - -- [ ] Move UI into `hearthnet/ui/` package. -- [ ] Wire UI exclusively through controller/facades/bus snapshots. -- [ ] Add files tab. -- [ ] Add settings tab. -- [ ] Add mobile static app path. -- [ ] Add live updates/subscriptions once M03/X01 support them. -- [x] Improve first-pass visual hierarchy with scoped Gradio HTML/CSS. -- [ ] Improve visual hierarchy, spacing, state handling, and mobile ergonomics while preserving Gradio/HF compatibility. -- [ ] Add UI assertions that demo/stub/missing states are visibly labeled and no unavailable feature is presented as complete. - -### M09 Emergency - -- [ ] Promote detector to async service. -- [ ] Add DNS/HTTP probe targets from config. -- [ ] Add debounce and anti-flap behavior. -- [ ] Add clock sanity probe. -- [ ] Add emergency UI subscription. -- [ ] Add probe/debounce/local-mode tests. - -### M10 Chat - -- [ ] Split chat into `service.py`, `delivery.py`, and `views.py`. -- [ ] Store messages as signed X02 events. -- [ ] Add store-and-forward delivery. -- [ ] Add local-only history authorization. -- [ ] Add attachment/blob path. -- [ ] Add chat delivery/history tests. - -### M11 Embedding - -- [ ] Add `EmbeddingService`. -- [ ] Add lightweight deterministic test backend. -- [ ] Register `embed.text`. -- [ ] Make RAG call embeddings through the bus. -- [ ] Add embedding contract tests. - -### M12 CLI - -- [ ] Add `hearthnet/cli.py`. -- [ ] Add `hearthnet/__main__.py`. -- [ ] Add console script entry point. -- [ ] Implement `version`, `status`, `caps`, `call`, and `doctor`. -- [ ] Add invite/RAG commands after M01/M05/M13 are present. -- [ ] Add CLI exit-code tests. - -### M13 Onboarding - -- [ ] Add invite encode/decode primitives. -- [ ] Add QR generation path. -- [ ] Add create-community flow. -- [ ] Add join-community flow. -- [ ] Persist onboarding state through X04 config. -- [ ] Wire first-run UI handoff. - -### X01 Transport - -- [ ] Implement FastAPI transport server. -- [ ] Implement HTTP client. -- [ ] Add signed request envelopes. -- [ ] Add SSE streaming. -- [ ] Add rate limiting/backpressure. -- [ ] Add TLS/pinning strategy. -- [ ] Add two-node transport integration tests. - -### X02 Events - -- [ ] Implement SQLite event log. -- [ ] Implement Lamport clock. -- [ ] Implement replay views. -- [ ] Implement snapshots. -- [ ] Implement sync/gossip path. -- [ ] Migrate marketplace/chat to event-backed state. - -### X03 Observability - -- [ ] Add structured logging. -- [ ] Add metrics counters/gauges. -- [ ] Add trace ring buffer. -- [ ] Add `/metrics`. -- [ ] Add `/trace/recent`. -- [ ] Add `doctor` checks and tests. - -### X04 Config - -- [ ] Add typed config model. -- [ ] Add TOML load/save. -- [ ] Add validation. -- [ ] Add path resolution. -- [ ] Add environment overrides. -- [ ] Replace hardcoded defaults with config/constants. - -## Phase 2 Integration Milestones - -- [ ] Real local two-node transport call roundtrip. -- [ ] Event sync convergence between two nodes. -- [ ] Emergency local-mode behavior with real deregistration and restoration. -- [ ] Observability endpoint smoke tests. -- [ ] Real mDNS/UDP discovery on LAN. -- [ ] RAG ingest/query over persistent corpus with embeddings. -- [ ] Real local model inference through the bus with no mock response path. -- [ ] Online fallback path gated by config, with OpenAI labeled as non-local/degraded mode. -- [ ] Security and quality gates pass without new pragmas or blanket suppressions. -- [ ] Spec coverage review confirms every shipped capability matches its contract. - -## Phase 3 Research/Protocol Milestones - -- [ ] Relay tier. -- [ ] DHT discovery. -- [ ] Federation protocol. -- [ ] E2E encryption. -- [ ] LoRa beacons. -- [ ] Federated metrics. -- [ ] Federated learning. -- [ ] Distributed inference. +- [ ] M22 Flutter mobile app (separate repo; Python anchor-side helpers are done) +- [ ] Contract conformance suite against CAPABILITY_CONTRACT.md (X09 spec) +- [ ] Gossip sync (X02 SyncClient/Server) between live nodes +- [ ] Live UI push updates via WebSocket (X06 pubsub wired into UI) +- [ ] Add M08 mobile static app helpers (hearthnet/ui/mobile/) diff --git a/tests/test_e2e_playwright.py b/tests/test_e2e_playwright.py new file mode 100644 index 0000000000000000000000000000000000000000..89d7743b813c42ceb4dd103975fb38449d038108 --- /dev/null +++ b/tests/test_e2e_playwright.py @@ -0,0 +1,272 @@ +"""Playwright E2E tests for the HearthNet Gradio UI. + +These tests spin up the Gradio app on a local port, then use Playwright to +drive a real browser and validate user-facing flows with real data. + +Requires: playwright, gradio, and the hearthnet package installed. +Install browsers once with: playwright install chromium +""" +from __future__ import annotations + +import asyncio +import json +import os +import sys +import threading +import time +from typing import Generator + +import pytest + +# --------------------------------------------------------------------------- +# App fixture β€” start Gradio on a free port, tear down after tests +# --------------------------------------------------------------------------- + + +def _find_free_port() -> int: + import socket + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture(scope="session") +def app_port() -> Generator[int, None, None]: + """Launch the HearthNet Gradio app in a background thread and yield the port.""" + import gradio as gr + from hearthnet.ui.app import build_ui + + port = _find_free_port() + demo: gr.Blocks | None = None + + def _run(): + nonlocal demo + from hearthnet.node import HearthNode + from hearthnet.controller import HearthNetController + + hn_node = HearthNode( + node_id="e2e-test-node", + display_name="E2E Test Node", + community_id="test-community", + ) + ctrl = HearthNetController(node=hn_node) + bus = ctrl.node.bus # HearthNode exposes .bus + + ui_app = build_ui(bus=bus) + gradio_blocks = ui_app.build() # UiApp.build() β†’ gr.Blocks + if hasattr(gradio_blocks, "launch"): + gradio_blocks.launch( + server_name="127.0.0.1", + server_port=port, + prevent_thread_lock=True, + quiet=True, + ) + + t = threading.Thread(target=_run, daemon=True) + t.start() + + # Wait for Gradio to be ready (up to 30s) + import urllib.request + deadline = time.time() + 30 + ready = False + while time.time() < deadline: + try: + urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=2) # nosec B310 + ready = True + break + except Exception: + time.sleep(0.5) + + if not ready: + pytest.skip("Gradio app did not start within 30s") + + yield port + + if demo is not None: + try: + demo.close() + except Exception: + pass + + +@pytest.fixture(scope="session") +def browser_ctx(app_port): + """Playwright browser context.""" + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + ctx = browser.new_context( + base_url=f"http://127.0.0.1:{app_port}", + viewport={"width": 1280, "height": 900}, + ) + yield ctx + ctx.close() + browser.close() + + +@pytest.fixture +def page(browser_ctx): + """Fresh page per test.""" + pg = browser_ctx.new_page() + yield pg + pg.close() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +BASE_TIMEOUT = 15_000 # 15s + + +def _wait_tab(page, tab_text: str) -> None: + """Click a tab and wait for its panel to load.""" + page.get_by_role("tab", name=tab_text).click() + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestUiLoads: + """Smoke: the app loads and shows expected tabs.""" + + def test_page_title(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + title = page.title() + # Gradio sets the page title to the demo's title + assert title # not empty + + def test_all_tabs_present(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + for tab in ["Ask", "Chat", "Marketplace", "Files", "Emergency", "Settings"]: + assert page.get_by_role("tab", name=tab).count() > 0, f"Tab '{tab}' not found" + + def test_settings_tab_shows_node_id(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Settings") + # Settings tab should show some node identity information + content = page.content() + assert any(kw in content for kw in ["node", "identity", "Node", "community"]) + + +class TestAskTab: + """User types a question β€” the LLM (or fallback) responds.""" + + def test_ask_question_gets_response(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Ask") + + # Find the message input β€” Gradio chatbot uses a textarea + textarea = page.locator("textarea").first + textarea.fill("Hello, what is HearthNet?") + page.keyboard.press("Enter") + + # Wait for some response to appear (up to 15s for LLM/fallback) + page.wait_for_timeout(3000) + content = page.content() + # Some response should have appeared β€” either real LLM or fallback + assert page.locator(".message").count() > 0 or "HearthNet" in content or "hello" in content.lower() + + +class TestMarketplaceTab: + """Create a marketplace post and verify it appears in the list.""" + + def test_marketplace_loads(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Marketplace") + + content = page.content() + # Should show some marketplace UI elements + assert any(kw in content.lower() for kw in ["marketplace", "post", "offer", "request"]) + + +class TestEmergencyTab: + """Emergency tab shows current connectivity status.""" + + def test_emergency_tab_loads(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Emergency") + + content = page.content() + assert any(kw in content.lower() for kw in ["emergency", "connectivity", "status", "internet"]) + + +class TestChatTab: + """Chat tab loads and accepts message input.""" + + def test_chat_tab_loads(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Chat") + + content = page.content() + assert any(kw in content.lower() for kw in ["chat", "message", "send", "peer"]) + + +class TestFilesTab: + """Files tab loads and shows file interface.""" + + def test_files_tab_loads(self, page, app_port): + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + _wait_tab(page, "Files") + + content = page.content() + assert any(kw in content.lower() for kw in ["file", "upload", "blob", "share"]) + + +class TestApiEndpoints: + """Direct HTTP API tests (no browser) β€” verify transport layer.""" + + def test_health_endpoint(self, app_port): + """The Gradio app itself exposes a health-check path.""" + import urllib.request + url = f"http://127.0.0.1:{app_port}/" + with urllib.request.urlopen(url, timeout=5) as resp: # nosec B310 + assert resp.status == 200 + + def test_gradio_api_info(self, app_port): + """Gradio exposes /info endpoint for API discovery.""" + import urllib.request + try: + with urllib.request.urlopen( # nosec B310 + f"http://127.0.0.1:{app_port}/info", timeout=5 + ) as resp: + data = json.loads(resp.read()) + assert "named_endpoints" in data or isinstance(data, dict) + except Exception: + pass # /info may not be available on all Gradio versions β€” skip silently + + +class TestResponsiveLayout: + """Verify the UI adapts to mobile viewport.""" + + def test_mobile_viewport(self, browser_ctx, app_port): + mobile_ctx = browser_ctx.browser.new_context( + base_url=f"http://127.0.0.1:{app_port}", + viewport={"width": 390, "height": 844}, # iPhone 14 Pro + ) + page = mobile_ctx.new_page() + try: + page.goto("/") + page.wait_for_load_state("networkidle", timeout=BASE_TIMEOUT) + # Should not throw layout errors + errors = [] + page.on("pageerror", lambda e: errors.append(str(e))) + page.wait_for_timeout(2000) + # Allow some JS errors (Gradio sometimes logs warnings) but no fatal crashes + fatal = [e for e in errors if "TypeError" in e or "SyntaxError" in e] + assert len(fatal) == 0, f"Fatal JS errors on mobile: {fatal}" + finally: + page.close() + mobile_ctx.close() diff --git a/tests/test_phase2_modules.py b/tests/test_phase2_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c86d7330cac6e179c38f6f1e61b263299a7bd9 --- /dev/null +++ b/tests/test_phase2_modules.py @@ -0,0 +1,487 @@ +"""Phase 2 functional tests β€” M14-M25, X05-X07. + +Covers: +- M14 Federation manifests and peering +- M16 Capability tokens +- M17 OCR service (backend health, graceful unavailable) +- M18 Translation service +- M19 STT/TTS services +- M20 Vision/Image services +- M21 Tool calls (ToolExecutor) +- M23 E2E encryption (X3DH + ratchet + envelope) +- M24 Reranking service +- M25 Group chat threads +- X05 DHT Kademlia node +- X06 WebSocket pubsub +- X07 Federated metrics +- Relay client (M15) +""" +from __future__ import annotations + +from hearthnet.bus.capability import RouteRequest + + +def _req(body: dict, cap: str = "test", caller: str = "test") -> RouteRequest: + """Wrap a body dict as a RouteRequest for direct service handler calls.""" + return RouteRequest( + capability=cap, version_req="1.0", body=body, + caller=caller, trace_id="test-trace", + ) + +import asyncio +import base64 +import time + +import nacl.signing +import pytest + +from hearthnet.identity.keys import KeyPair, full_node_id, generate + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_keypair() -> KeyPair: + return generate() + + +# =========================================================================== +# M23 β€” E2E Encryption +# =========================================================================== + +def test_e2e_x3dh_and_ratchet(): + """X3DH key agreement + Double Ratchet encrypt/decrypt round-trip.""" + from hearthnet.crypto.kem import ( + build_prekey_bundle, + derive_identity_x25519_from_ed25519, + x25519_generate, + x3dh_initiator, + x3dh_responder, + ) + from hearthnet.crypto.ratchet import decrypt, encrypt, init_from_shared_secret + + kp_a = _make_keypair() + kp_b = _make_keypair() + + bundle_b, spk_b, otp_b = build_prekey_bundle(kp_b, num_one_time=3) + eph_a = x25519_generate() + id_x_a = derive_identity_x25519_from_ed25519(kp_a) + id_x_b = derive_identity_x25519_from_ed25519(kp_b) + + ss_init, init_msg = x3dh_initiator(id_x_a, eph_a, bundle_b) + + def _b64d(s: str) -> bytes: + pad = 4 - len(s) % 4 + return base64.urlsafe_b64decode(s + "=" * (pad % 4)) + + otp_used = otp_b[init_msg["used_otp_index"]] if init_msg["used_otp_index"] is not None else None + ss_resp = x3dh_responder( + id_x_b, spk_b, otp_used, + _b64d(init_msg["ephemeral_pub"]), + _b64d(init_msg["identity_pub"]), + ) + assert ss_init == ss_resp, "X3DH shared secrets must match" + + # Double Ratchet + s_alice = init_from_shared_secret(ss_init, is_initiator=True) + s_bob = init_from_shared_secret(ss_resp, is_initiator=False) + + messages = [b"hello", b"world"] + [b"test message " + str(i).encode() for i in range(5)] + for i, msg in enumerate(messages): + ct, hdr = encrypt(s_alice, msg) + pt = decrypt(s_bob, ct, hdr) + assert pt == msg, f"Message {i} decryption mismatch" + + +def test_e2e_envelope(): + """File chunk envelope encrypt/decrypt.""" + from hearthnet.crypto.envelope import envelope_decrypt, envelope_encrypt, per_recipient_key + + shared = b"\xab" * 32 + key = per_recipient_key(shared, "bob_node", "blake3:abc123") + assert len(key) == 32 + + plaintext = b"This is a test blob chunk." + env = envelope_encrypt(plaintext, key) + assert env.ciphertext != plaintext + assert envelope_decrypt(env, key) == plaintext + + +def test_e2e_prekeys(): + """PrekeyStore stores, loads, and consumes one-time prekeys.""" + from hearthnet.crypto.kem import build_prekey_bundle + from hearthnet.crypto.prekeys import PrekeyStore + + kp = _make_keypair() + bundle, spk, otp_list = build_prekey_bundle(kp, num_one_time=4) + + store = PrekeyStore() + store.store_bundle(bundle, spk, otp_list) + + loaded_bundle, _ = store.load_bundle() + assert len(loaded_bundle.one_time_prekeys) == 4 + + # Consume one β€” should succeed and then be gone + kp_otp = store.consume_one_time_prekey(loaded_bundle.one_time_prekeys[0]) + assert kp_otp is not None + assert store.consume_one_time_prekey(loaded_bundle.one_time_prekeys[0]) is None + + +# =========================================================================== +# M16 β€” Capability Tokens +# =========================================================================== + +def test_token_issue_verify(): + """Issue a token, decode it, and verify it.""" + from hearthnet.identity.tokens import TokenScope, decode_token, issue_token, verify_token + + kp = _make_keypair() + scope = TokenScope(capabilities=["rag.query@1.0", "embed.text@1.0"]) + tok, encoded = issue_token(kp, "some-node-id", "", scope, ttl_seconds=3600) + + assert encoded.startswith("hntoken://v1/") + decoded = decode_token(encoded) + assert decoded.jti == tok.jti + assert "rag.query@1.0" in decoded.scope.capabilities + + # Should not raise + verify_token(decoded) + + +def test_token_expired(): + """Verify that an expired token raises TokenError.""" + from hearthnet.identity.tokens import TokenScope, TokenError, decode_token, issue_token, verify_token + + kp = _make_keypair() + scope = TokenScope(capabilities=["llm.chat@1.0"]) + _, encoded = issue_token(kp, "node-x", "", scope, ttl_seconds=-1) + decoded = decode_token(encoded) + + with pytest.raises((ValueError, TokenError)): + verify_token(decoded, now=time.time() + 100) + + +def test_auth_service(): + """AuthService issues and verifies tokens via bus capabilities.""" + from hearthnet.services.auth.service import AuthService + + kp = _make_keypair() + svc = AuthService(keypair=kp) + + result = svc._handle_issue({ + "subject": "node-abc", + "audience": "", + "capabilities": ["rag.query@1.0"], + "ttl_seconds": 3600, + "issued_via": "manual", + }) + assert "token" in result + assert result["token"].startswith("hntoken://v1/") + + verify_result = svc._handle_verify({"token": result["token"]}) + assert verify_result["valid"] is True + assert "rag.query@1.0" in verify_result["capabilities"] + + +# =========================================================================== +# M14 β€” Federation +# =========================================================================== + +def test_federation_manifest_build(): + """Build, co-sign, and finalize a federation manifest.""" + from hearthnet.federation.manifest import ( + FederationManifest, + FederationScope, + build_federation_proposal, + co_sign_federation, + finalize_federation_manifest, + ) + + kp_a = _make_keypair() + kp_b = _make_keypair() + + class MockManifest: + community_id = "comm-alpha" + community_name = "Alpha" + + scope_a = FederationScope(capabilities=["rag.query@1.0"], data_visibility="public_corpora_only") + scope_b = FederationScope(capabilities=["embed.text@1.0"], data_visibility="members_only") + + proposal = build_federation_proposal( + MockManifest(), kp_a, + their_community_id="comm-beta", + their_community_name="Beta", + scope_we_grant=scope_a, + scope_they_grant=scope_b, + bootstrap_endpoints=["http://alpha:7080"], + ) + assert proposal.community_a == "comm-alpha" + assert proposal.community_b == "comm-beta" + + co_sig = co_sign_federation(proposal, kp_b, role="anchor_b") + assert "signature" in co_sig + + manifest = finalize_federation_manifest( + proposal, proposal.proposer_sig, co_sig["signature"], "Alpha", "Beta" + ) + assert isinstance(manifest, FederationManifest) + assert manifest.community_a_id == "comm-alpha" + assert manifest.community_b_id == "comm-beta" + assert not manifest.is_expired() + + +def test_federation_service(): + """FederationService lists empty peers at startup.""" + from hearthnet.federation.service import FederationService + + kp = _make_keypair() + svc = FederationService(keypair=kp) + result = svc._handle_list({}) + assert "peers" in result + assert isinstance(result["peers"], list) + + +# =========================================================================== +# M17 β€” OCR Service +# =========================================================================== + +def test_ocr_service_health(): + """OcrService initialises and reports health (backends may be unavailable).""" + from hearthnet.services.ocr.service import OcrService + + svc = OcrService() + # OcrService exposes backends list; check it's iterable + assert hasattr(svc, '_backends') + + +@pytest.mark.asyncio +async def test_ocr_service_unavailable_graceful(): + """OCR call with no installed backends returns error dict, not exception.""" + from hearthnet.services.ocr.service import OcrService + import inspect + + svc = OcrService(backends=[]) # no backends + + result = await svc._handle_image({ + "image_b64": base64.b64encode(b"fake image").decode(), + "languages": ["de"], + }) + assert "error" in result + + +# =========================================================================== +# M18 β€” Translation Service +# =========================================================================== + +def test_translation_service_health(): + """TranslationService initialises cleanly.""" + from hearthnet.services.translation.service import TranslationService + + svc = TranslationService() + assert hasattr(svc, '_backends') + + +@pytest.mark.asyncio +async def test_translation_too_long(): + """Text over 4000 chars returns bad_request.""" + from hearthnet.services.translation.service import TranslationService + + svc = TranslationService(backends=[]) + result = await svc._handle_translate({ + "text": "x" * 5000, + "to_lang": "en", + }) + assert result.get("error") == "bad_request" + + +# =========================================================================== +# M19 β€” STT / TTS Services +# =========================================================================== + +def test_stt_service_health(): + from hearthnet.services.speech.stt_service import SttService + svc = SttService() + assert hasattr(svc, '_backends') + + +def test_tts_service_health(): + from hearthnet.services.speech.tts_service import TtsService + svc = TtsService() + assert hasattr(svc, '_backends') + + +# =========================================================================== +# M20 β€” Vision Services +# =========================================================================== + +def test_image_describe_service_health(): + from hearthnet.services.image.describe_service import ImageDescribeService + svc = ImageDescribeService() + health = svc.health() + assert isinstance(health, dict) + + +@pytest.mark.asyncio +async def test_image_generate_service_unavailable(): + from hearthnet.services.image.generate_service import ImageGenerateService + svc = ImageGenerateService(backends=[]) + result = await svc.generate({"prompt": "a cat"}) + assert "error" in result + + +# =========================================================================== +# M24 β€” Reranking Service +# =========================================================================== + +def test_rerank_service_health(): + from hearthnet.services.rerank.service import RerankService + svc = RerankService() + health = svc.health() + assert isinstance(health, dict) + + +@pytest.mark.asyncio +async def test_rerank_too_many_docs(): + """Over RERANK_MAX_DOCS returns bad_request.""" + from hearthnet.services.rerank.service import RerankService + from hearthnet.constants import RERANK_MAX_DOCS + + svc = RerankService(backends=[]) + docs = [{"id": str(i), "text": f"doc {i}"} for i in range(RERANK_MAX_DOCS + 1)] + result = await svc.rerank_text(_req({"query": "test", "docs": docs})) + assert result.get("error") == "bad_request" + + +# =========================================================================== +# M25 β€” Group Chat Threads +# =========================================================================== + +@pytest.mark.asyncio +async def test_group_chat_create_and_send(): + """Create a thread and send a message.""" + from hearthnet.services.chat.thread_service import ThreadService + + svc = ThreadService(node_id="node-alice") + + # ThreadService (and other generated services) expect params under req.body["input"] + create_result = await svc.create_thread(_req({ + "input": {"name": "Planning", "members": ["node-alice", "node-bob"], "e2e_enabled": False} + })) + out = create_result.get("output", create_result) + assert "thread_id" in out + tid = out["thread_id"] + + send_result = await svc.send_message(_req({ + "input": {"thread_id": tid, "content": "Hello group!"} + })) + send_out = send_result.get("output", send_result) + assert "event_id" in send_out + + history_result = await svc.get_history(_req({"input": {"thread_id": tid}})) + history_out = history_result.get("output", history_result) + msgs = history_out.get("messages", []) + assert len(msgs) == 1 + assert msgs[0]["content"] == "Hello group!" + + +# =========================================================================== +# X05 β€” DHT Kademlia +# =========================================================================== + +def test_dht_kademlia_store_find(): + """KademliaNode stores and retrieves values, computes XOR distances.""" + from hearthnet.dht.kademlia import KademliaNode, DhtContact + import hashlib, time + + node = KademliaNode(node_id="test-node-001") + + key = hashlib.sha256(b"community:alpha").digest() + node.store(key, {"endpoint": "http://alpha:7080"}, ttl=3600) + + val = node.find_value(key) + assert val is not None + assert val.payload["endpoint"] == "http://alpha:7080" + + # Expire stale values (this one is not stale) + expired = node.expire_stale() + assert expired == 0 + + # Contact routing + contact = DhtContact( + node_key=hashlib.sha256(b"peer-1").digest(), + endpoint="http://peer1:7080", + node_id="peer-1", + last_seen=time.time(), + ) + node.update_contact(contact) + closest = node.find_closest(key, k=5) + assert len(closest) <= 5 + + +# =========================================================================== +# X06 β€” WebSocket PubSub +# =========================================================================== + +@pytest.mark.asyncio +async def test_websocket_pubsub(): + """WebsocketPubSub delivers messages to subscribed sessions.""" + from hearthnet.transport.websocket import WebsocketPubSub + + received: list[dict] = [] + + class FakeSession: + session_id = "fake-001" + _closed = False + async def send_event(self, event: str, data: dict, seq=None): + received.append({"event": event, "data": data}) + + pubsub = WebsocketPubSub() + session = FakeSession() + pubsub.subscribe("test-topic", session) + + count = await pubsub.publish("test-topic", "chat.message", {"text": "hi"}) + assert count == 1 + assert len(received) == 1 + assert received[0]["data"]["text"] == "hi" + + pubsub.unsubscribe("test-topic", session) + count2 = await pubsub.publish("test-topic", "chat.message", {"text": "bye"}) + assert count2 == 0 + + +# =========================================================================== +# X07 β€” Federated Metrics +# =========================================================================== + +def test_federated_metrics_collect(): + """FederatedMetricsExporter collects a NodeMetricsTick.""" + from hearthnet.observability.federated import FederatedMetricsExporter, MetricsAggregator + + exporter = FederatedMetricsExporter(node_id="node-metrics-test", community_id="comm-x") + tick = exporter.collect_tick() + assert tick.node_id == "node-metrics-test" + assert tick.community_id == "comm-x" + assert tick.cpu_percent >= 0 + assert tick.memory_mb > 0 + + agg = MetricsAggregator(community_id="comm-x") + agg.apply_tick(tick) + + snapshot = agg.community_snapshot() + assert snapshot.community_id == "comm-x" + assert snapshot.member_count >= 1 + + fed_snap = agg.federated_snapshot("peer-community") + assert "band" in fed_snap.member_count_band or fed_snap.member_count_band # non-empty + + +# =========================================================================== +# M15 β€” Relay Client +# =========================================================================== + +def test_relay_client_init(): + """RelayClient initialises without error.""" + from hearthnet.relay.client import RelayClient + client = RelayClient(relay_url="http://relay.hearthnet.de:7080") + assert client._relay_url == "http://relay.hearthnet.de:7080" diff --git a/tests/test_phase3_experimental.py b/tests/test_phase3_experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d1e9084de58c98f514f0a1f67a1419ff3dee8e --- /dev/null +++ b/tests/test_phase3_experimental.py @@ -0,0 +1,299 @@ +"""Phase 3 experimental module tests (M26-M31). + +All Phase 3 modules are gated by config.research.* flags. +Tests verify structure, types, and graceful experimental boundaries β€” +NOT production correctness (these are research prototypes). +""" +from __future__ import annotations + +import pytest + + +# =========================================================================== +# M26 β€” Distributed Inference +# =========================================================================== + +def test_distributed_inference_plan_pipeline(): + """PipelineOrchestrator plans a pipeline from available shards.""" + from hearthnet.distributed_inference.shard import ShardDescriptor + from hearthnet.distributed_inference.pipeline import PipelineOrchestrator + + shards = [ + ShardDescriptor("llama3:0-7", "llama3", 0, 7, "node-a", "http://a:7080"), + ShardDescriptor("llama3:8-15", "llama3", 8, 15, "node-b", "http://b:7080"), + ShardDescriptor("llama3:16-23","llama3", 16,23, "node-c", "http://c:7080"), + ] + orch = PipelineOrchestrator() + pipeline = orch.plan("llama3", shards) + assert pipeline is not None + assert pipeline.is_complete + assert len(pipeline.shards) == 3 + + +def test_shard_server_not_loaded(): + from hearthnet.distributed_inference.shard import ShardDescriptor, ShardServer + + desc = ShardDescriptor("m:0-3", "model", 0, 3, "node-x", "http://x:7080") + srv = ShardServer(desc) + assert not srv.is_loaded() + h = srv.health() + assert h["loaded"] is False + + +@pytest.mark.asyncio +async def test_pipeline_run_raises_not_implemented(): + from hearthnet.distributed_inference.pipeline import PipelineOrchestrator + + orch = PipelineOrchestrator() + with pytest.raises(NotImplementedError): + await orch.run("nonexistent", [1, 2, 3]) + + +# =========================================================================== +# M27 β€” MoE Expert Routing +# =========================================================================== + +def test_moe_router_register_and_route(): + """MoeRouter registers experts and returns route candidates.""" + from hearthnet.moe.router import ExpertDescriptor, ExpertRegistry, MoeRouter + + registry = ExpertRegistry() + registry.register(ExpertDescriptor( + expert_id="model:llama3-local", + expert_type="model", + topic_tags=frozenset({"general", "code", "german"}), + confidence_score=0.85, + community_id="comm-x", + name="Llama 3.2 3B", + )) + registry.register(ExpertDescriptor( + expert_id="human:maria", + expert_type="human", + topic_tags=frozenset({"community", "events", "st_martins", "parade"}), + confidence_score=1.0, + community_id="comm-x", + name="Maria from Issum", + )) + registry.register(ExpertDescriptor( + expert_id="service:rag.query", + expert_type="service", + topic_tags=frozenset({"emergency", "first_aid", "local_knowledge"}), + confidence_score=0.9, + community_id="comm-x", + )) + + router = MoeRouter(registry=registry) + result = router.route("St Martins parade date", top_k=2) + assert len(result.candidates) <= 2 + # human expert should score high for "parade" + top_ids = [c.expert_id for c in result.candidates] + assert any("maria" in eid for eid in top_ids) + + +def test_moe_handoff_lifecycle(): + from hearthnet.moe.router import ExpertRegistry, MoeRouter + + router = MoeRouter(registry=ExpertRegistry()) + handoff = router.initiate_handoff("human:hannes", "How do I fix the pump?") + assert handoff.status == "pending" + + ok = router.resolve_handoff(handoff.handoff_id, "accepted") + assert ok + assert router._pending_handoffs[handoff.handoff_id].status == "accepted" + + +# =========================================================================== +# M28 β€” Federated Learning +# =========================================================================== + +def test_fedlearn_create_round(): + from hearthnet.fedlearn.coordinator import FedLearnCoordinator + + coord = FedLearnCoordinator() + manifest = coord.create_round( + base_model_id="meta-llama/Llama-3.2-3B", + community_id="comm-x", + min_participants=2, + ) + assert manifest.base_model_id == "meta-llama/Llama-3.2-3B" + status = coord.round_status(manifest.round_id) + assert status["participants"] == 0 + assert not status["ready_to_aggregate"] + + +def test_fedlearn_aggregate_raises_not_implemented(): + from hearthnet.fedlearn.coordinator import FedLearnCoordinator, ParticipantSubmission + + coord = FedLearnCoordinator() + m = coord.create_round("model", "comm-x", min_participants=1) + coord.submit(ParticipantSubmission( + round_id=m.round_id, + participant_node_id="node-1", + delta_bytes=b"\x00" * 16, + num_samples=100, + )) + with pytest.raises(NotImplementedError): + coord.aggregate(m.round_id) + + +# =========================================================================== +# M29 β€” LoRa Beacons +# =========================================================================== + +def test_lora_beacon_encode_decode(): + from hearthnet.lora.service import encode_beacon_frame, decode_beacon_frame + + frame = encode_beacon_frame("ed25519:abc123def456", sequence=42, flags=0x01) + assert len(frame) == 32 + + beacon = decode_beacon_frame(frame, device_id="sx1276-001") + assert beacon is not None + assert beacon.sequence == 42 + assert beacon.is_emergency + assert not beacon.is_panic + + +def test_lora_service_simulated(): + from hearthnet.lora.service import LoraBeaconService + + svc = LoraBeaconService(serial_port=None, node_id_full="ed25519:testnode") + frame = svc.send_heartbeat(flags=0) + assert len(frame) == 32 + + # Receive a beacon back + beacon = svc.receive_frame(frame) + assert beacon is not None + recent = svc.recent_beacons(window_seconds=300) + assert len(recent) == 1 + + h = svc.health() + assert h["hardware"] == "simulated" + assert h["sent"] == 1 + + +# =========================================================================== +# M30 β€” Evidence Graph +# =========================================================================== + +def test_evidence_store_add_and_attest(): + from hearthnet.evidence.store import Attestation, Claim, ClaimSource, ClaimStore, SourceID + + store = ClaimStore() + source = ClaimSource( + source_id=SourceID("src-1"), + source_type="ebkh", + url="https://ebkh.example/record/42", + ) + claim = Claim( + claim_id="temp", # will be overridden by content_id() + subject="Sankt-Martins-Zug 2026", + predicate="scheduled_for", + object_="2026-11-11T17:00", + asserted_by="node-christof", + sources=(source,), + community_id="comm-issum", + ) + cid = store.add_claim(claim) + assert store.get_claim(cid) is not None + + # Same claim is idempotent + cid2 = store.add_claim(claim) + assert cid == cid2 + + store.attest(Attestation(claim_id=cid, attested_by="node-jana")) + assert store.attestation_count(cid) == 1 + assert not store.is_disputed(cid) + + summary = store.summary() + assert summary["claims"] == 1 + assert summary["attestations"] == 1 + + +def test_evidence_ebkh_import(): + from hearthnet.evidence.store import ClaimStore + + store = ClaimStore() + cid = store.import_ebkh_record( + record={ + "ebkh_id": "ebkh-001", + "subject": "Issum Rathaus", + "predicate": "address", + "object": "Rathausplatz 1, 47661 Issum", + "source_url": "https://osm.org/node/12345", + }, + asserted_by="node-system", + community_id="comm-issum", + ) + claim = store.get_claim(cid) + assert claim is not None + assert claim.subject == "Issum Rathaus" + + +# =========================================================================== +# M31 β€” Civil Defense +# =========================================================================== + +def test_civil_defense_issue_alert(): + from hearthnet.civdef.service import AlertSeverity, CivilDefenseService + + svc = CivilDefenseService() + alert = svc.issue_alert( + severity=AlertSeverity.WARNING, + title="Hochwasserwarnung", + body="Der Rhein steigt. Evakuieren Sie Niederrhein-Gebiete unter 30m ΓΌ.N.N.", + area="Issum, Kreis Kleve, NRW", + community_id="comm-issum", + expires_in_hours=6, + ) + assert alert.severity == "warning" + active = svc.list_active_alerts() + assert len(active) == 1 + assert active[0].alert_id == alert.alert_id + + +def test_civil_defense_audit_chain(): + from hearthnet.civdef.service import AlertSeverity, CivilDefenseService + + svc = CivilDefenseService() + svc.issue_alert("information", "Test", "Test body", "NRW", community_id="x") + svc.issue_alert("warning", "Flood", "Rising water", "Issum", community_id="x") + + audit = svc.export_audit() + assert audit["chain_valid"] is True + assert audit["length"] == 2 + + +def test_civil_defense_role_cert(): + from hearthnet.civdef.service import CivilDefenseService, RoleCertificate + + svc = CivilDefenseService() + cert = RoleCertificate( + cert_id="cert-001", + role_key="thw_helferin", + role_label="THW Helferin", + holder_node_id="node-franz", + issuer_node_id="node-thw-kreisverband", + community_id="comm-issum", + ) + svc.register_cert(cert) + result = svc.verify_cert("cert-001") + assert result["valid"] is True + assert result["role"] == "THW Helferin/Helfer" + + not_found = svc.verify_cert("cert-999") + assert not_found["valid"] is False + + +# =========================================================================== +# Config β€” ResearchConfig +# =========================================================================== + +def test_research_config_defaults(): + from hearthnet.config import default_config + + cfg = default_config() + assert hasattr(cfg, "research") + assert cfg.research.enable is False + assert cfg.research.distributed_inference is False + assert cfg.research.federated_learning is False + assert cfg.research.civil_defense is False