Upgrade to v3: real-time quantization with slipcore from PyPI

Replace bundled v2 modules with slipcore>=3.0.0 from PyPI.
New Quantize tab shows live Think-Quantize-Transmit pipeline.
UCR Explorer tab with all 45 anchors. Cleaned up old files.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

README.md

@@ -1,41 +1,42 @@
 ---
-title: 'Slipstream: Semantic Quantization for Multi-Agent Coordination'
-emoji: 📄
+title: Slipstream v3
+emoji: ⚡
 colorFrom: blue
-colorTo: indigo
+colorTo: purple
 sdk: gradio
+sdk_version: 4.44.0
 app_file: app.py
 pinned: false
-license: mit
-tags:
-- semantic-quantization
-- multi-agent-systems
-- protocol-standards
-- token-efficiency
-sdk_version: 6.5.1
+license: apache-2.0
+short_description: "Real-time semantic quantization for multi-agent AI"
 ---
 
-# Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination
+# Slipstream v3: Real-Time Semantic Quantization
 
-This Space was generated from a research paper PDF.
+Type a natural language sentence and watch it get quantized into a factorized
+Force-Object wire message in real time. No API keys, no GPU.
 
-## What you can do here
+## Features
 
-- **Live Quantizer**: Type messy natural language and watch it get quantized to a UCR anchor (the core demo!)
-- **Start here**: guided entry points (summary / limitations / thread)
-- **Gallery**: extracted figures or page previews
-- **Chat**: ask questions about the paper
-- **Share Kit**: generate a tweet thread / talk outline / FAQ
-- **Model Playground**: chat with a referenced HF model (requires `HF_TOKEN`)
+- **Quantize** -- type any sentence, see the TQT (Think-Quantize-Transmit) pipeline in action
+- **Encode** -- build wire messages manually from Force + Object dropdowns
+- **Decode** -- parse and inspect SLIP v3 wire messages
+- **UCR Explorer** -- browse all 45 core anchors with filtering
 
-## Optional secrets
+## Resources
 
-If you add these as Space secrets, Chat + Share Kit become generative:
+- **GitHub**: [github.com/anthony-maio/slipcore](https://github.com/anthony-maio/slipcore)
+- **PyPI**: `pip install slipcore`
+- **Paper**: [doi.org/10.5281/zenodo.18063451](https://doi.org/10.5281/zenodo.18063451)
+- **SDK Guide**: [docs/sdk-guide.md](https://github.com/anthony-maio/slipcore/blob/master/docs/sdk-guide.md)
 
-- `HF_TOKEN`: Hugging Face token (read access is sufficient for inference; write is **not** needed at runtime)
-- `PAPER_LLM_MODEL`: e.g. `meta-llama/Meta-Llama-3-8B-Instruct` (or any chat-completion capable model)
+## Citation
 
-## Build provenance
-
-- Source PDF: `slipstream-paper.pdf`
-- Extracted pages: 7
+```bibtex
+@misc{maio2025slipstream,
+  title={Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination},
+  author={Maio, Anthony},
+  year={2025},
+  doi={10.5281/zenodo.18063451}
+}
+```

app.py

@@ -1,521 +1,523 @@
-import json
-import os
-import re
-from dataclasses import dataclass
-from typing import Dict, List, Tuple, Optional
+"""
+Slipstream v3: Real-Time Semantic Quantization Demo
+Interactive HuggingFace Space for multi-agent AI communication protocol.
+"""
 
 import gradio as gr
 
-# Lazy-loaded quantizer
-_quantizer = None
-_quantizer_error = None
-
-def _get_quantizer():
-    """Lazy load the embedding quantizer."""
-    global _quantizer, _quantizer_error
-    if _quantizer is not None:
-        return _quantizer
-    if _quantizer_error is not None:
-        return None
-    try:
-        from quantizer import EmbeddingQuantizer
-        _quantizer = EmbeddingQuantizer(fallback_threshold=0.3)
-        return _quantizer
-    except Exception as e:
-        _quantizer_error = str(e)
-        return None
-
-def _get_keyword_quantizer():
-    """Fallback to keyword quantizer."""
-    try:
-        from quantizer import KeywordQuantizer
-        return KeywordQuantizer()
-    except Exception:
-        return None
-
-
-# ---------------------------
-# Loaded at build time
-# ---------------------------
-PAPER_TITLE = "Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination"
-PAPER_AUTHORS = "Anthony Maio"
-PAPER_ABSTRACT = "As multi-agent LLM systems scale,coordination bandwidthbecomes a primary cost\ndriver: every token spent on routing, intent framing, and redundant context is paid repeat-\nedly across agents and turns. Current approaches waste 40\u201360% of compute on coordination\noverhead, with communication costs scalingO(n2)as agent counts increase.\nThis paper introducesSlipstream, a protocol that performssemantic quantization:\nmapping free-form messages onto a sharedUniversal Concept Reference (UCR)and\ntransmitting compactmnemonic anchorsthat identify structured intents. Unlike syn-\ntactic compression (which fails due to BPE tokenizer fragmentation), Slipstream transmits\nnatural-language mnemonics that tokenize efficiently across model architectures.\nSlipstream combines (1) a symbolic4D semantic manifold\u2014Action, Polarity, Domain,\nUrgency\u2014with (2) a data-drivenvector engine(embeddings + nearest-centroid retrieval)\nplus anevolutionary extension layerthat learns new anchors from low-confidence traf-\nfic. Results show82% token reduction(41.9\u21927.4 tokens average) while maintaining\nsemantic fidelity, making large-scale multi-agent deployments economically viable."
-PAPER_TAGS = "semantic-quantization, multi-agent-systems, protocol-standards, token-ef-"
-DEFAULT_LLM_MODEL = None
-DETECTED_MODELS = []
-
-
-# ---------------------------
-# Robust file loading
-# ---------------------------
-def _load_chunks(path: str = "paper_chunks.jsonl") -> List[str]:
-    chunks: List[str] = []
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    rec = json.loads(line)
-                    txt = (rec.get("text") or "").strip()
-                    if txt:
-                        chunks.append(txt)
-                except Exception:
-                    continue
-    except FileNotFoundError:
-        return []
-    except Exception:
-        return []
-    return chunks
-
-
-# ---------------------------
-# Tiny BM25-ish retrieval (pure Python)
-# ---------------------------
-def _tokenize(s: str) -> List[str]:
-    return re.findall(r"[A-Za-z0-9][A-Za-z0-9_-]{1,}", s.lower())
-
-
-@dataclass
-class Retriever:
-    chunks: List[str]
-    doc_tokens: List[List[str]]
-    df: Dict[str, int]
-    idf: Dict[str, float]
-
-    @staticmethod
-    def build(chunks: List[str]) -> "Retriever":
-        doc_tokens = [_tokenize(c) for c in chunks]
-        df: Dict[str, int] = {}
-        for toks in doc_tokens:
-            for t in set(toks):
-                df[t] = df.get(t, 0) + 1
-        n = max(1, len(doc_tokens))
-        idf = {}
-        for t, d in df.items():
-            idf[t] = float((n - d + 0.5) / (d + 0.5))
-        return Retriever(chunks=chunks, doc_tokens=doc_tokens, df=df, idf=idf)
-
-    def topk(self, query: str, k: int = 4) -> List[Tuple[int, float]]:
-        q = _tokenize(query)
-        if not q:
-            return []
-        scores: List[Tuple[int, float]] = []
-        qset = set(q)
-        for i, toks in enumerate(self.doc_tokens):
-            if not toks:
-                continue
-            overlap = qset.intersection(toks)
-            if not overlap:
-                continue
-            score = 0.0
-            for t in overlap:
-                score += self.idf.get(t, 0.0)
-            score = score / (1.0 + (len(toks) / 200.0))
-            scores.append((i, score))
-        scores.sort(key=lambda x: x[1], reverse=True)
-        return scores[:k]
-
-
-CHUNKS = _load_chunks()
-RETRIEVER = Retriever.build(CHUNKS) if CHUNKS else None
-
-
-def retrieve_context(query: str, k: int = 4, max_chars: int = 6000) -> str:
-    if not RETRIEVER:
-        return ""
-    hits = RETRIEVER.topk(query, k=k)
-    parts: List[str] = []
-    for idx, _score in hits:
-        txt = CHUNKS[idx].strip()
-        if txt:
-            parts.append(txt)
-    ctx = "\n\n".join(parts).strip()
-    return ctx[:max_chars]
-
-
-# ---------------------------
-# HF Inference helpers (optional)
-# ---------------------------
-def _get_hf_client(model_id: str):
-    try:
-        from huggingface_hub import InferenceClient
-    except Exception as e:
-        raise RuntimeError("huggingface_hub is not installed. Add it to requirements.txt") from e
-
-    token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
-    if not token:
-        raise RuntimeError("HF_TOKEN is not set as a Space secret.")
-    return InferenceClient(model=model_id, token=token)
-
-
-def _llm_chat(model_id: str, messages: List[dict], max_tokens: int = 512) -> str:
-    client = _get_hf_client(model_id)
-    out = client.chat_completion(messages=messages, max_tokens=max_tokens)
-    return out.choices[0].message.content
-
-
-# ---------------------------
-# Chat with paper (RAG-lite)
-# ---------------------------
-def paper_chat(message: str, history: List[Tuple[str, str]]) -> str:
-    message = (message or "").strip()
-    if not message:
-        return "Ask a question about the paper."
-
-    ctx = retrieve_context(message, k=4, max_chars=6000)
-
-    model_id = os.environ.get("PAPER_LLM_MODEL") or DEFAULT_LLM_MODEL
-    if not model_id:
-        if not ctx:
-            return "No indexed context found. (paper_chunks.jsonl missing?)"
-        return "Top matches in the paper:\n\n" + ctx[:1200]
-
-    trimmed_history = history[-4:] if history else []
-    messages: List[dict] = [
-        {
-            "role": "system",
-            "content": (
-                "You are a precise research assistant. Answer using ONLY the provided paper context. "
-                "If the context is insufficient, say what is missing and point to what section would help."
-            ),
-        },
-    ]
-    if ctx:
-        messages.append({"role": "system", "content": "PAPER CONTEXT:\n\n" + ctx})
-
-    for u, a in trimmed_history:
-        messages.append({"role": "user", "content": u})
-        messages.append({"role": "assistant", "content": a})
-
-    messages.append({"role": "user", "content": message})
-
-    try:
-        return _llm_chat(model_id, messages, max_tokens=512).strip()
-    except Exception as e:
-        if ctx:
-            return f"(LLM unavailable: {e})\n\nTop matches in the paper:\n\n" + ctx[:1200]
-        return f"LLM unavailable: {e}"
-
-
-# ---------------------------
-# Share Kit (generators)
-# ---------------------------
-def _fallback_tweet_thread(title: str, abstract: str) -> str:
-    abs_one = re.sub(r"\s+", " ", abstract).strip()
-    bullets = [
-        f"1/ {title}",
-        "2/ TL;DR: " + (abs_one[:220] + ("…" if len(abs_one) > 220 else "")),
-        "3/ Key idea: (open the Space → Chat tab and ask for the method overview)",
-        "4/ Try it: use the Share Kit tab to generate a talk outline / FAQ.",
-        "5/ Links: add your paper + code links in the README.",
-    ]
-    return "\n\n".join(bullets)
-
-def generate_share(kind: str) -> str:
-    kind = (kind or "").strip().lower()
-    base_title = PAPER_TITLE or "Paper"
-    base_abs = PAPER_ABSTRACT or ""
-
-    model_id = os.environ.get("PAPER_LLM_MODEL") or DEFAULT_LLM_MODEL
-    if not model_id:
-        if kind == "tweet thread":
-            return _fallback_tweet_thread(base_title, base_abs)
-        if kind == "talk outline":
-            return "\n".join([
-                f"Title: {base_title}",
-                "- Motivation",
-                "- Problem setup",
-                "- Method",
-                "- Results",
-                "- Limitations",
-                "- Q&A",
-            ])
-        if kind == "faq":
-            return "\n".join([
-                "Q: What problem does this address?\nA: " + (base_abs[:220] + ("…" if len(base_abs) > 220 else "")),
-                "",
-                "Q: What is the main contribution?\nA: Ask in the Chat tab.",
-                "",
-                "Q: How do I reproduce it?\nA: Link code + add steps in README.",
-            ])
-        return "Select an item to generate."
-
-    prompt = {
-        "tweet thread": "Write a concise 6-tweet thread summarizing the paper for the ML community.",
-        "talk outline": "Create a 10-minute talk outline with section headers and bullet points.",
-        "faq": "Write an FAQ with 6 Q/A pairs focused on method, results, limitations, and usage.",
-    }.get(kind, "Summarize the paper in 8 bullet points.")
-
-    ctx = (PAPER_ABSTRACT or "").strip()
-    messages = [
-        {"role": "system", "content": "You are an expert technical writer for ML research audiences."},
-        {"role": "user", "content": f"Paper title: {base_title}\nAuthors: {PAPER_AUTHORS}\n\nAbstract/context:\n{ctx}\n\nTask: {prompt}"},
-    ]
-    try:
-        return _llm_chat(model_id, messages, max_tokens=600).strip()
-    except Exception as e:
-        if kind == "tweet thread":
-            return _fallback_tweet_thread(base_title, base_abs) + f"\n\n(LLM unavailable: {e})"
-        return f"LLM unavailable: {e}"
-
-
-# ---------------------------
-# Model Playground (chat)
-# ---------------------------
-def model_chat(model_id: str, message: str, history: List[Tuple[str, str]]) -> str:
-    model_id = (model_id or "").strip()
-    message = (message or "").strip()
-    if not model_id:
-        return "Provide a model id."
-    if not message:
-        return "Send a message."
-
-    messages: List[dict] = [{"role": "system", "content": "You are a helpful assistant."}]
-    for u, a in (history[-4:] if history else []):
-        messages.append({"role": "user", "content": u})
-        messages.append({"role": "assistant", "content": a})
-    messages.append({"role": "user", "content": message})
+from slipcore import (
+    KeywordQuantizer,
+    create_base_ucr,
+    format_fallback,
+    format_slip,
+    parse_slip,
+    render_human,
+    __version__,
+)
+from slipcore.quantizer import FORCE_KEYWORDS, OBJECT_KEYWORDS, _keyword_score
+
+# --------------------------------------------------------------------------
+# Shared state
+# --------------------------------------------------------------------------
+
+UCR = create_base_ucr()
+QUANTIZER = KeywordQuantizer()
+
+FORCES = [
+    "Observe", "Inform", "Ask", "Request", "Propose", "Commit",
+    "Eval", "Meta", "Accept", "Reject", "Error", "Fallback",
+]
 
-    try:
-        return _llm_chat(model_id, messages, max_tokens=512).strip()
-    except Exception as e:
-        return f"Model call failed: {e}"
+OBJECTS = [
+    "State", "Change", "Error", "Result", "Status", "Complete",
+    "Blocked", "Progress", "Clarify", "Permission", "Resource",
+    "Task", "Plan", "Review", "Help", "Cancel", "Priority",
+    "Alternative", "Rollback", "Deadline", "Approve", "NeedsWork",
+    "Ack", "Sync", "Handoff", "Escalate", "Abort", "Condition",
+    "Defer", "Timeout", "Validation", "Generic",
+]
 
+FORCE_DESCRIPTIONS = {
+    "Observe": "Passively notice state, change, or error",
+    "Inform": "Report information -- status, completion, blockage, progress",
+    "Ask": "Seek information -- clarification, status, permission",
+    "Request": "Direct action -- task, review, help, plan",
+    "Propose": "Suggest something -- plan, change, alternative",
+    "Commit": "Pledge to something -- task, deadline, resource",
+    "Eval": "Judge work quality -- approve, needs work",
+    "Meta": "Protocol-level -- acknowledge, sync, handoff, escalate",
+    "Accept": "Agree to a proposal or request",
+    "Reject": "Decline a proposal or request",
+    "Error": "Report system error -- timeout, resource, permission",
+    "Fallback": "Content too complex for standard quantization",
+}
+
+FORCE_COLORS = {
+    "Observe": "#6366f1",   # indigo
+    "Inform": "#0ea5e9",    # sky
+    "Ask": "#f59e0b",       # amber
+    "Request": "#ef4444",   # red
+    "Propose": "#8b5cf6",   # violet
+    "Commit": "#22c55e",    # green
+    "Eval": "#14b8a6",      # teal
+    "Meta": "#64748b",      # slate
+    "Accept": "#10b981",    # emerald
+    "Reject": "#f43f5e",    # rose
+    "Error": "#dc2626",     # red-600
+    "Fallback": "#78716c",  # stone
+}
+
+
+# --------------------------------------------------------------------------
+# Quantize tab logic
+# --------------------------------------------------------------------------
+
+def quantize_sentence(text: str, src: str, dst: str) -> str:
+    """Quantize natural language into SLIP v3, showing the full TQT pipeline."""
+    if not text.strip():
+        return ""
 
-# ---------------------------
-# UI helpers
-# ---------------------------
-def quantize_intent(intent: str) -> Tuple[str, str, str]:
-    """
-    Quantize a natural language intent to UCR anchor.
-    Returns: (primary_result_md, alternatives_md, wire_format)
-    """
-    intent = (intent or "").strip()
-    if not intent:
-        return "Enter an intent to quantize.", "", ""
+    src = "".join(c for c in (src or "alice").strip() if c.isalnum())[:20] or "alice"
+    dst = "".join(c for c in (dst or "bob").strip() if c.isalnum())[:20] or "bob"
 
-    # Try embedding quantizer first, fall back to keyword
-    quantizer = _get_quantizer()
-    method = "embedding"
-    if quantizer is None:
-        quantizer = _get_keyword_quantizer()
-        method = "keyword"
+    # Run quantizer
+    result = QUANTIZER.quantize(text)
+    force = result.force
+    obj = result.obj
+    confidence = result.confidence
 
-    if quantizer is None:
-        return "Quantizer unavailable. Check logs.", "", ""
+    # Build wire
+    if result.is_fallback:
+        ref = QUANTIZER.fallback_store.store(text)
+        wire = format_fallback(src, dst, ref)
+    else:
+        wire = format_slip(src, dst, force, obj)
+
+    # Get canonical from UCR
+    anchor = UCR.get_by_force_obj(force, obj)
+    canonical = anchor.canonical if anchor else f"{force} {obj}"
+
+    # Compute keyword match details for Force
+    force_detail_rows = []
+    for f_name, patterns in FORCE_KEYWORDS.items():
+        score = _keyword_score(text, patterns)
+        if score > 0:
+            matched = [p for p in patterns if p.lower() in text.lower()]
+            color = FORCE_COLORS.get(f_name, "#666")
+            bar_width = int(score * 100)
+            winner = " **<--**" if f_name == force else ""
+            force_detail_rows.append(
+                (score, f"| <span style='color:{color}'>`{f_name}`</span> | "
+                 f"{''.join(matched[:3])} | "
+                 f"<span style='display:inline-block;background:{color};width:{bar_width}px;height:12px;border-radius:3px'></span> "
+                 f"{score:.2f}{winner} |")
+            )
+    force_detail_rows.sort(key=lambda x: x[0], reverse=True)
+
+    # Compute keyword match details for Object
+    obj_detail_rows = []
+    for o_name, patterns in OBJECT_KEYWORDS.items():
+        score = _keyword_score(text, patterns)
+        if score > 0:
+            matched = [p for p in patterns if p.lower() in text.lower()]
+            bar_width = int(score * 100)
+            winner = " **<--**" if o_name == obj else ""
+            obj_detail_rows.append(
+                (score, f"| `{o_name}` | "
+                 f"{''.join(matched[:3])} | "
+                 f"<span style='display:inline-block;background:#6366f1;width:{bar_width}px;height:12px;border-radius:3px'></span> "
+                 f"{score:.2f}{winner} |")
+            )
+    obj_detail_rows.sort(key=lambda x: x[0], reverse=True)
 
-    try:
-        result = quantizer.quantize(intent)
-    except Exception as e:
-        return f"Quantization error: {e}", "", ""
+    # Token comparison
+    input_tokens = len(text.split())
+    wire_tokens = len(wire.split())
+    reduction = ((input_tokens - wire_tokens) / input_tokens * 100) if input_tokens > 0 else 0
 
     # Confidence color
-    conf = result.confidence
-    if conf >= 0.7:
-        color = "green"
+    if confidence >= 0.6:
+        conf_color = "#22c55e"
         conf_label = "High"
-    elif conf >= 0.5:
-        color = "orange"
+    elif confidence >= 0.3:
+        conf_color = "#f59e0b"
         conf_label = "Medium"
     else:
-        color = "red"
+        conf_color = "#ef4444"
         conf_label = "Low"
 
-    # Primary result
-    primary_md = f"""
-### {result.anchor.mnemonic}
+    force_color = FORCE_COLORS.get(force, "#666")
+
+    # Build output
+    out = []
+
+    # Wire result (hero)
+    out.append(f"### Wire Output")
+    out.append(f"```")
+    out.append(wire)
+    out.append(f"```")
+    out.append("")
+
+    # Human-readable
+    human = render_human(wire)
+    out.append(f"> {human}")
+    out.append("")
+
+    # Confidence + token savings
+    conf_bar_width = int(confidence * 200)
+    out.append(f"| Metric | Value |")
+    out.append(f"|--------|-------|")
+    out.append(f"| Confidence | <span style='color:{conf_color}'>{conf_label}</span> ({confidence:.0%}) |")
+    out.append(f"| Force | <span style='color:{force_color}'>{force}</span> |")
+    out.append(f"| Object | {obj} |")
+    out.append(f"| Canonical | \"{canonical}\" |")
+    out.append(f"| Input tokens | ~{input_tokens} |")
+    out.append(f"| Wire tokens | {wire_tokens} |")
+    out.append(f"| Reduction | {reduction:.0f}% |")
+    out.append("")
+
+    # Stage 1: Force classification
+    if force_detail_rows:
+        out.append(f"<details><summary><b>Stage 1: Force Classification</b> -- matched {len(force_detail_rows)} forces</summary>")
+        out.append("")
+        out.append(f"| Force | Matched Keywords | Score |")
+        out.append(f"|-------|-----------------|-------|")
+        for _, row in force_detail_rows[:6]:
+            out.append(row)
+        out.append("")
+        out.append(f"</details>")
+        out.append("")
+
+    # Stage 2: Object classification
+    if obj_detail_rows:
+        out.append(f"<details><summary><b>Stage 2: Object Classification</b> -- matched {len(obj_detail_rows)} objects</summary>")
+        out.append("")
+        out.append(f"| Object | Matched Keywords | Score |")
+        out.append(f"|--------|-----------------|-------|")
+        for _, row in obj_detail_rows[:6]:
+            out.append(row)
+        out.append("")
+        out.append(f"</details>")
+        out.append("")
+
+    if result.is_fallback:
+        out.append(f"*Fallback triggered: no keyword patterns matched with sufficient confidence. "
+                    f"Raw text stored out-of-band with ref pointer.*")
+
+    return "\n".join(out)
+
+
+# --------------------------------------------------------------------------
+# Encode tab logic
+# --------------------------------------------------------------------------
+
+def encode_message(src: str, dst: str, force: str, obj: str, payload: str) -> str:
+    if not src or not dst or not force or not obj:
+        return "Fill in source, destination, Force, and Object."
+
+    clean_src = "".join(c for c in src.strip() if c.isalnum())[:20] or "agent"
+    clean_dst = "".join(c for c in dst.strip() if c.isalnum())[:20] or "other"
+
+    parts = ["SLIP", "v3", clean_src, clean_dst, force, obj]
+    if payload.strip():
+        for token in payload.strip().split():
+            clean = "".join(c for c in token if c.isalnum())
+            if clean:
+                parts.append(clean[:30])
 
-**Confidence:** <span style="color:{color}; font-weight:bold">{conf:.0%}</span> ({conf_label})
+    wire = " ".join(parts)
+    human = render_human(wire)
 
-**Canonical meaning:** {result.anchor.canonical}
+    input_tokens = len(parts)
+    json_equiv = f'{{"from":"{src}","to":"{dst}","action":"{force}","target":"{obj}","payload":"{payload}"}}'
+    json_tokens = len(json_equiv.split()) + json_equiv.count('"') + json_equiv.count(':')
+    reduction = ((json_tokens - input_tokens) / json_tokens) * 100 if json_tokens > 0 else 0
 
-**Method:** {method} {'(fallback)' if result.is_fallback else ''}
+    return f"""```
+{wire}
+```
 
-**Coordinates:** `{result.anchor.coords}` (Action, Polarity, Domain, Urgency)
+> {human}
+
+| Metric | SLIP v3 | JSON equivalent |
+|--------|---------|-----------------|
+| Tokens | **{input_tokens}** | ~{json_tokens} |
+| Reduction | **{reduction:.0f}%** | -- |
 """
 
-    # Alternatives
-    if result.alternatives:
-        alt_lines = ["| Anchor | Similarity |", "|--------|------------|"]
-        for alt_anchor, alt_score in result.alternatives[:3]:
-            bar_len = int(alt_score * 10)
-            bar = "█" * bar_len + "░" * (10 - bar_len)
-            alt_lines.append(f"| {alt_anchor.mnemonic} | {bar} {alt_score:.0%} |")
-        alternatives_md = "\n".join(alt_lines)
-    else:
-        alternatives_md = "*No alternatives*"
 
-    # Wire format
-    wire = f"SLIP v1 user agent {result.anchor.mnemonic}"
-    if result.is_fallback:
-        # Truncate long intents for fallback payload
-        payload = intent[:100].replace('"', "'")
-        wire = f'SLIP v1 user agent Fallback "{payload}"'
+# --------------------------------------------------------------------------
+# Decode tab logic
+# --------------------------------------------------------------------------
+
+def decode_message(wire: str) -> str:
+    if not wire.strip():
+        return "Enter a SLIP v3 message to decode."
+
+    try:
+        msg = parse_slip(wire.strip())
+    except Exception as e:
+        return f"Parse error: {e}"
+
+    anchor = UCR.get_by_force_obj(msg.force, msg.obj)
+    canonical = anchor.canonical if anchor else "unknown"
+    coords = f"({', '.join(str(c) for c in anchor.coords)})" if anchor else "--"
+    force_desc = FORCE_DESCRIPTIONS.get(msg.force, "Unknown")
+    human = render_human(msg)
+
+    return f"""> {human}
+
+| Field | Value |
+|-------|-------|
+| Version | {msg.version} |
+| Source | `{msg.src}` |
+| Destination | `{msg.dst}` |
+| Force | **{msg.force}** -- {force_desc} |
+| Object | **{msg.obj}** |
+| Canonical | \"{canonical}\" |
+| Coords | `{coords}` |
+| Payload | {' '.join(msg.payload) if msg.payload else '--'} |
+| Fallback | {'`' + msg.fallback_ref + '`' if msg.fallback_ref else '--'} |
+"""
+
+
+# --------------------------------------------------------------------------
+# UCR Explorer
+# --------------------------------------------------------------------------
+
+def build_ucr_table(force_filter: str) -> str:
+    anchors = list(UCR.anchors.values())
+    anchors.sort(key=lambda a: a.index)
 
-    return primary_md, alternatives_md, wire
+    if force_filter and force_filter != "All":
+        anchors = [a for a in anchors if a.force == force_filter]
 
+    rows = []
+    for a in anchors:
+        color = FORCE_COLORS.get(a.force, "#666")
+        coords = ", ".join(str(c) for c in a.coords)
+        rows.append(
+            f"| `{a.index:#06x}` | <span style='color:{color}'>{a.force}</span> | "
+            f"{a.obj} | {a.canonical} | ({coords}) |"
+        )
 
-EXAMPLE_INTENTS = [
-    ("Review my code", "RequestReview"),
-    ("Task complete!", "InformComplete"),
-    ("System down!", "ObserveError"),
-    ("Can you help?", "RequestHelp"),
-    ("Looks good to me", "EvalApprove"),
+    header = f"**{len(anchors)} anchors** " + (f"(filtered: {force_filter})" if force_filter != "All" else "(all)")
+
+    return f"""{header}
+
+| Index | Force | Object | Canonical | Coords |
+|-------|-------|--------|-----------|--------|
+{chr(10).join(rows)}
+"""
+
+
+# --------------------------------------------------------------------------
+# Example sentences for the quantizer
+# --------------------------------------------------------------------------
+
+EXAMPLES = [
+    ["Please review the pull request for the auth module", "dev", "reviewer"],
+    ["The database migration is complete", "worker", "manager"],
+    ["I am blocked waiting for API credentials", "backend", "devops"],
+    ["I suggest we switch to Redis for the session store", "architect", "team"],
+    ["LGTM, approved for merge", "reviewer", "dev"],
+    ["Emergency: halt all production deployments now", "sre", "allAgents"],
+    ["What do you mean by 'optimize the query'?", "junior", "senior"],
+    ["I will handle the database migration this sprint", "dbAdmin", "pm"],
+    ["The API call to the payment provider timed out after 30s", "gateway", "monitor"],
+    ["Yes, but only if we add monitoring first", "lead", "architect"],
+    ["Ping, are you still there?", "coordinator", "worker"],
+    ["No, that approach will not scale to our traffic levels", "cto", "engineer"],
 ]
 
 
-def start_here(choice: str) -> str:
-    choice = (choice or "").strip().lower()
-    if choice == "quick summary":
-        return f"### {PAPER_TITLE}\n\n**Authors:** {PAPER_AUTHORS}\n\n**Abstract:**\n\n{PAPER_ABSTRACT}"
-    if choice == "how does it work?":
-        return "Go to **Chat** and ask: *Give me a method overview with the key steps.*"
-    if choice == "what are the limitations?":
-        return "Go to **Chat** and ask: *List limitations and failure modes discussed in the paper.*"
-    if choice == "generate a tweet thread":
-        return generate_share("tweet thread")
-    return "Pick an option."
-
-
-def _load_gallery_items() -> List[Tuple[str, str]]:
-    items: List[Tuple[str, str]] = []
-    if os.path.isdir("assets/images"):
-        for fn in sorted(os.listdir("assets/images"))[:48]:
-            path = os.path.join("assets/images", fn)
-            if os.path.isfile(path):
-                items.append((path, fn))
-    if not items and os.path.isdir("assets/pages"):
-        for fn in sorted(os.listdir("assets/pages"))[:24]:
-            path = os.path.join("assets/pages", fn)
-            if os.path.isfile(path):
-                items.append((path, fn))
-    return items
-
-
-CSS = '''
-.paper-hero h1 { margin-bottom: 0.2rem; }
-.paper-hero p { margin-top: 0.2rem; opacity: 0.9; }
-.hint { opacity: 0.85; }
-'''
-
-with gr.Blocks(theme=gr.themes.Soft(), css=CSS) as demo:
-    gr.Markdown(f"# {PAPER_TITLE}", elem_classes=["paper-hero"])
-    if PAPER_AUTHORS:
-        gr.Markdown(f"**Authors:** {PAPER_AUTHORS}", elem_classes=["paper-hero"])
-    if PAPER_TAGS:
-        gr.Markdown(f"**Tags:** {PAPER_TAGS}", elem_classes=["paper-hero"])
+# --------------------------------------------------------------------------
+# Build Gradio app
+# --------------------------------------------------------------------------
+
+with gr.Blocks(
+    title="Slipstream v3",
+    theme=gr.themes.Soft(),
+    css="""
+    .hero-wire { font-size: 1.3em; font-family: monospace; }
+    footer { display: none !important; }
+    """
+) as demo:
+
+    gr.Markdown("""
+# Slipstream v3
+### Real-time semantic quantization for multi-agent AI coordination
+
+Type a natural language sentence and watch it get quantized into a
+factorized Force-Object wire message. No API keys, no GPU -- runs on
+the keyword classifier built into [slipcore](https://pypi.org/project/slipcore/).
+
+---
+""")
 
     with gr.Tabs():
-        with gr.Tab("Start here"):
-            gr.Markdown("Pick an interaction to explore the paper quickly.", elem_classes=["hint"])
-            choice = gr.Radio(
-                ["Quick summary", "How does it work?", "What are the limitations?", "Generate a tweet thread"],
-                value="Quick summary",
-                label="What do you want?",
+
+        # ---- Quantize tab (hero feature) ----
+        with gr.TabItem("Quantize", id="quantize"):
+            gr.Markdown("### Think -> Quantize -> Transmit")
+            gr.Markdown("Enter what you want to say. The quantizer maps it to Force + Object.")
+
+            with gr.Row():
+                q_text = gr.Textbox(
+                    label="Natural language input",
+                    placeholder="Please review the pull request for the auth module",
+                    lines=2,
+                    scale=4,
+                )
+            with gr.Row():
+                q_src = gr.Textbox(label="Source agent", value="alice", scale=1)
+                q_dst = gr.Textbox(label="Destination agent", value="bob", scale=1)
+                q_btn = gr.Button("Quantize", variant="primary", scale=1)
+
+            q_output = gr.Markdown()
+
+            q_btn.click(
+                quantize_sentence,
+                inputs=[q_text, q_src, q_dst],
+                outputs=q_output,
             )
-            out = gr.Markdown()
-            choice.change(start_here, inputs=choice, outputs=out)
-            demo.load(start_here, inputs=choice, outputs=out)
-
-        with gr.Tab("Overview"):
-            gr.Markdown("## Abstract")
-            gr.Markdown(PAPER_ABSTRACT)
-
-            gr.Markdown("---")
-            gr.Markdown("### Text search (snippet)")
-            q = gr.Textbox(label="Find a phrase", placeholder="e.g., scalable oversight", lines=1)
-            snippet = gr.Textbox(label="Top matching context", lines=10)
-
-            def _snippet(query: str) -> str:
-                query = (query or "").strip()
-                if not query:
-                    return ""
-                ctx = retrieve_context(query, k=4, max_chars=1600)
-                return ctx or "No matches."
-
-            q.change(_snippet, inputs=q, outputs=snippet)
-
-        with gr.Tab("Gallery"):
-            gr.Markdown("Extracted images / rendered page previews (if included at build time).", elem_classes=["hint"])
-            gallery = gr.Gallery(label="Figures / pages", columns=2, rows=2, height=520)
-
-            def _gallery():
-                return _load_gallery_items()
-
-            demo.load(_gallery, outputs=gallery)
-
-        with gr.Tab("Chat"):
-            gr.Markdown(
-                "Ask questions. If you set `HF_TOKEN` + `PAPER_LLM_MODEL` as Space secrets, answers become generative; "
-                "otherwise it returns top-matching snippets.",
-                elem_classes=["hint"],
+            q_text.submit(
+                quantize_sentence,
+                inputs=[q_text, q_src, q_dst],
+                outputs=q_output,
             )
-            gr.ChatInterface(fn=paper_chat, title="Chat with the Paper")
-
-        with gr.Tab("Share Kit"):
-            gr.Markdown("Generate shareable assets. Works without secrets (deterministic fallback).", elem_classes=["hint"])
-            kind = gr.Dropdown(["Tweet thread", "Talk outline", "FAQ"], value="Tweet thread", label="Generate")
-            btn = gr.Button("Create")
-            share_out = gr.Textbox(lines=14, label="Output")
-            btn.click(lambda k: generate_share(k), inputs=kind, outputs=share_out)
-
-        with gr.Tab("Model Playground"):
-            gr.Markdown("Chat with a referenced Hub model (if any) or provide your own. Requires `HF_TOKEN` secret.", elem_classes=["hint"])
-            model_id = gr.Dropdown(
-                choices=(DETECTED_MODELS if DETECTED_MODELS else []),
-                value=(DETECTED_MODELS[0] if DETECTED_MODELS else None),
-                label="Model id",
-                allow_custom_value=True,
+
+            gr.Markdown("#### Try these examples")
+            gr.Examples(
+                examples=EXAMPLES,
+                inputs=[q_text, q_src, q_dst],
+                outputs=q_output,
+                fn=quantize_sentence,
+                cache_examples=False,
             )
 
-            def _model_chat_fn(message: str, history: List[Tuple[str, str]], mid: str) -> str:
-                return model_chat(mid, message, history)
+        # ---- Encode tab ----
+        with gr.TabItem("Encode", id="encode"):
+            gr.Markdown("### Manual encoder")
+            gr.Markdown("Build a wire message by picking Force and Object directly.")
 
-            gr.ChatInterface(fn=_model_chat_fn, additional_inputs=[model_id], title="Model Playground")
+            with gr.Row():
+                enc_src = gr.Textbox(label="Source", value="alice", scale=1)
+                enc_dst = gr.Textbox(label="Destination", value="bob", scale=1)
 
-        with gr.Tab("Live Quantizer"):
-            gr.Markdown("""
-## Think → Quantize → Transmit
+            with gr.Row():
+                enc_force = gr.Dropdown(
+                    choices=FORCES, label="Force", value="Request",
+                    info="Action verb", scale=1,
+                )
+                enc_obj = gr.Dropdown(
+                    choices=OBJECTS, label="Object", value="Review",
+                    info="Domain noun", scale=1,
+                )
+
+            enc_payload = gr.Textbox(label="Payload (optional)", placeholder="auth", value="auth")
+            enc_btn = gr.Button("Encode", variant="primary")
+            enc_output = gr.Markdown()
+
+            enc_btn.click(
+                encode_message,
+                inputs=[enc_src, enc_dst, enc_force, enc_obj, enc_payload],
+                outputs=enc_output,
+            )
 
-Type a messy, natural-language intent and watch it get quantized to a UCR anchor.
-This demonstrates the core Slipstream innovation: mapping free-form language onto a shared semantic manifold.
-""")
+        # ---- Decode tab ----
+        with gr.TabItem("Decode", id="decode"):
+            gr.Markdown("### Wire format decoder")
+            gr.Markdown("Paste a SLIP v3 message to inspect it.")
 
-            with gr.Row():
-                with gr.Column(scale=2):
-                    intent_input = gr.Textbox(
-                        label="Your intent (natural language)",
-                        placeholder="Hey, I'm kinda stuck on this auth bug, can you take a look?",
-                        lines=2,
-                    )
-                    quantize_btn = gr.Button("Quantize", variant="primary")
-
-                    gr.Markdown("**Try these examples:**")
-                    with gr.Row():
-                        for ex_text, ex_anchor in EXAMPLE_INTENTS:
-                            ex_btn = gr.Button(ex_text, size="sm")
-                            ex_btn.click(lambda t=ex_text: t, outputs=intent_input)
-
-                with gr.Column(scale=3):
-                    primary_out = gr.Markdown(label="Result")
-                    with gr.Accordion("Nearby Anchors", open=True):
-                        alternatives_out = gr.Markdown()
-                    wire_out = gr.Code(label="SLIP Wire Format", language=None)
-
-            quantize_btn.click(
-                quantize_intent,
-                inputs=intent_input,
-                outputs=[primary_out, alternatives_out, wire_out],
+            dec_input = gr.Textbox(
+                label="SLIP v3 message",
+                value="SLIP v3 alice bob Request Review auth",
+                placeholder="SLIP v3 src dst Force Object payload...",
             )
-            intent_input.submit(
-                quantize_intent,
-                inputs=intent_input,
-                outputs=[primary_out, alternatives_out, wire_out],
+            dec_btn = gr.Button("Decode", variant="primary")
+            dec_output = gr.Markdown()
+
+            dec_btn.click(decode_message, inputs=dec_input, outputs=dec_output)
+
+        # ---- UCR Explorer tab ----
+        with gr.TabItem("UCR Explorer", id="ucr"):
+            gr.Markdown("### Universal Concept Reference")
+            gr.Markdown("The 45 core anchors that form the shared semantic vocabulary.")
+
+            ucr_filter = gr.Dropdown(
+                choices=["All"] + FORCES,
+                value="All",
+                label="Filter by Force",
             )
+            ucr_table = gr.Markdown(value=build_ucr_table("All"))
+
+            ucr_filter.change(build_ucr_table, inputs=ucr_filter, outputs=ucr_table)
+
+        # ---- About tab ----
+        with gr.TabItem("About", id="about"):
+            gr.Markdown(f"""
+### What is Slipstream?
+
+Slipstream is a protocol that performs **semantic quantization**: mapping free-form
+messages onto a shared Universal Concept Reference (UCR) and transmitting factorized
+intents (Force + Object) that identify structured actions.
+
+Instead of sending `"Could you please take a look at my pull request for the authentication module changes?"` (17 tokens),
+Slipstream transmits `SLIP v3 dev reviewer Request Review auth` (7 tokens).
+
+**82% token reduction** while preserving semantic fidelity.
+
+### How the quantizer works
+
+The keyword quantizer runs in two stages:
+
+1. **Force classification** -- match input against keyword patterns for each of the 12 Force tokens. Pick the highest-scoring Force.
+2. **Object classification** -- match input against keyword patterns for each of the 31+ Object tokens. Pick the highest-scoring Object.
+
+If no pattern matches above the confidence threshold, the message falls back to `Fallback Generic` with a pointer reference to the original text stored out-of-band.
+
+For production use, swap in the embedding-based `SemanticQuantizer` from `slipcore_ml` (requires sentence-transformers).
+
+---
+
+### Resources
+
+| Resource | Link |
+|----------|------|
+| GitHub | [github.com/anthony-maio/slipcore](https://github.com/anthony-maio/slipcore) |
+| PyPI | `pip install slipcore` (v{__version__}) |
+| Paper | [doi.org/10.5281/zenodo.18063451](https://doi.org/10.5281/zenodo.18063451) |
+| SDK Guide | [docs/sdk-guide.md](https://github.com/anthony-maio/slipcore/blob/master/docs/sdk-guide.md) |
+| Model (LoRA) | [anthonym21/slipstream-glm-z1-9b](https://huggingface.co/anthonym21/slipstream-glm-z1-9b) |
+| Dataset | [anthonym21/slipstream-tqt](https://huggingface.co/datasets/anthonym21/slipstream-tqt) |
+
+### Cost savings at scale
+
+| Deployment | Agents | Annual JSON cost | Annual SLIP cost | Savings |
+|------------|--------|------------------|------------------|---------|
+| Startup | 10 | $3,600 | $650 | $2,950 |
+| Scale-up | 50 | $180,000 | $32,400 | $147,600 |
+| Enterprise | 1,000 | $2,500,000 | $450,000 | $2,050,000 |
+
+---
+
+**Citation:**
+```bibtex
+@misc{{maio2025slipstream,
+  title={{Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination}},
+  author={{Maio, Anthony}},
+  year={{2025}},
+  doi={{10.5281/zenodo.18063451}}
+}}
+```
+
+Apache 2.0 License | [Anthony Maio](https://github.com/anthony-maio)
+""")
+
+    gr.Markdown(f"<center><sub>slipcore v{__version__} | keyword quantizer | "
+                f"[source](https://github.com/anthony-maio/slipcore/tree/master/hf-space)</sub></center>")
 
-    gr.Markdown("---\nBuilt with Gradio on Hugging Face Spaces.")
 
 if __name__ == "__main__":
     demo.launch()

assets/analysis.json

@@ -1,75 +0,0 @@
-{
-  "schema_version": "paper_analysis_v1",
-  "pdf": {
-    "path": "D:\\Development\\slipcore\\private\\zenodo\\slipstream-paper.pdf",
-    "filename": "slipstream-paper.pdf",
-    "sha256": "e91b687dbbe2aa4fe01ec0ae3c5475fda9ad2a5107ea8e81927028c575c707f7",
-    "page_count": 7,
-    "text_pages_extracted": 7,
-    "extracted_chars": 11939
-  },
-  "paper": {
-    "title": "Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination",
-    "authors": [
-      "Anthony Maio"
-    ],
-    "abstract": "As multi-agent LLM systems scale,coordination bandwidthbecomes a primary cost\ndriver: every token spent on routing, intent framing, and redundant context is paid repeat-\nedly across agents and turns. Current approaches waste 40–60% of compute on coordination\noverhead, with communication costs scalingO(n2)as agent counts increase.\nThis paper introducesSlipstream, a protocol that performssemantic quantization:\nmapping free-form messages onto a sharedUniversal Concept Reference (UCR)and\ntransmitting compactmnemonic anchorsthat identify structured intents. Unlike syn-\ntactic compression (which fails due to BPE tokenizer fragmentation), Slipstream transmits\nnatural-language mnemonics that tokenize efficiently across model architectures.\nSlipstream combines (1) a symbolic4D semantic manifold—Action, Polarity, Domain,\nUrgency—with (2) a data-drivenvector engine(embeddings + nearest-centroid retrieval)\nplus anevolutionary extension layerthat learns new anchors from low-confidence traf-\nfic. Results show82% token reduction(41.9→7.4 tokens average) while maintaining\nsemantic fidelity, making large-scale multi-agent deployments economically viable."
-  },
-  "artifacts": {
-    "urls": [
-      "https://github.com/anthony-maio/slipcore",
-      "https://modelcontextprotocol.io/,",
-      "https://www.linuxfoundation."
-    ],
-    "hf_models": [],
-    "hf_datasets": [],
-    "hf_spaces": [],
-    "possible_hf_repo_ids": [
-      "Edge/embedded",
-      "Msg/Day",
-      "REQ/TSK",
-      "messages/day",
-      "org/press"
-    ],
-    "github_repos": [
-      "anthony-maio/slipcore"
-    ],
-    "arxiv_ids": [
-      "1982.10564",
-      "2690.17728"
-    ],
-    "dois": [
-      "10.1109/TIT.1982.1056489",
-      "10.1145/1772690.1772862",
-      "10.18653/v1/D19-1410"
-    ]
-  },
-  "suggested": {
-    "space_slug": "slipstream-semantic-quantization-for-efficient-m",
-    "space_title": "Slipstream: Semantic Quantization for Efficient Multi-Agent Coordination",
-    "tags": [
-      "semantic-quantization",
-      "multi-agent-systems",
-      "protocol-standards",
-      "token-ef-"
-    ],
-    "emoji": "📄",
-    "colorFrom": "blue",
-    "colorTo": "indigo"
-  },
-  "outputs": {
-    "context_txt": "paper_context.txt",
-    "chunks_jsonl": "paper_chunks.jsonl",
-    "rendered_pages": [
-      {
-        "page": 1,
-        "path": "pages\\page_01.png"
-      },
-      {
-        "page": 2,
-        "path": "pages\\page_02.png"
-      }
-    ],
-    "extracted_images": []
-  }
-}

paper_chunks.jsonl

@@ -1,6 +0,0 @@
-{"chunk_id": 0, "text": "Slipstream: Semantic Quantization for Efficient\nMulti-Agent Coordination\nAnthony Maio\nIndependent Researcher\nanthony@making-minds.ai\n2025\nAbstract\nAs multi-agent LLM systems scale,coordination bandwidthbecomes a primary cost\ndriver: every token spent on routing, intent framing, and redundant context is paid repeat-\nedly across agents and turns. Current approaches waste 40–60% of compute on coordination\noverhead, with communication costs scalingO(n2)as agent counts increase.\nThis paper introducesSlipstream, a protocol that performssemantic quantization:\nmapping free-form messages onto a sharedUniversal Concept Reference (UCR)and\ntransmitting compactmnemonic anchorsthat identify structured intents. Unlike syn-\ntactic compression (which fails due to BPE tokenizer fragmentation), Slipstream transmits\nnatural-language mnemonics that tokenize efficiently across model architectures.\nSlipstream combines (1) a symbolic4D semantic manifold—Action, Polarity, Domain,\nUrgency—with (2) a data-drivenvector engine(embeddings + nearest-centroid retrieval)\nplus anevolutionary extension layerthat learns new anchors from low-confidence traf-\nfic. Results show82% token reduction(41.9→7.4 tokens average) while maintaining\nsemantic fidelity, making large-scale multi-agent deployments economically viable.\nKeywords:Semantic Quantization, Multi-Agent Systems, Protocol Standards, Token Ef-\nficiency, Agentic AI\n1 Introduction\n1.1 The Coordination Crisis\nAgent swarms incur atokenizer tax: the repeated, non-semantic overhead of communicating\nmessage types, domains, and priorities. This overhead often dominates when messages are\nstructured (routing, task dispatch, acknowledgements).\nA typical coordination message:\n1{\n2\" sender \": \" planning_agent \",\n3\" recipient \": \" execution_agent \",\n4\" message_type \": \" task_delegation \",\n5\" content \": {\n6\" request \": \" Please review the authentication code \",\n7\" priority \": \" high \"\n8}\n9}\n•Token count:∼45 tokens\n•Semantic content:∼10 tokens\n•Information density:22%\n1\nAt GPT-4o pricing ($5/M input, $15/M output), a 50-agent deployment exchanging 1,000\nmessages/day costs$180,000/yearin coordination tokens alone—before any work is per-\nformed.\n1.2 Why Syntactic Compression Fails\nOur initial approach, nSLIP v1, focused on syntactic minification:\n1REQ / TSK |s =7| d =3| act = review_auth\n•Expected tokens:8–10\n•Actual tokens with BPE:18–22\nThe failure stems"}
-{"chunk_id": 1, "text": "tic Compression Fails\nOur initial approach, nSLIP v1, focused on syntactic minification:\n1REQ / TSK |s =7| d =3| act = review_auth\n•Expected tokens:8–10\n•Actual tokens with BPE:18–22\nThe failure stems from Byte-Pair Encoding (BPE) tokenizer behavior. Punctuation and\nspecial characters fragment into separate tokens:\nTable 1: BPE Tokenization of Syntactic Compression\nInput Tokens\nREQ/TSK REQ,/,TSK= 3\n|s=7| |,s,=,7,|= 5\nThis “Tokenizer Tax” negates syntactic savings entirely.\n1.3 The Solution: Semantic Quantization\nInstead of compressingsyntax, we quantizesemantics. Agents share a pre-agreed “concept\ncodebook” (the UCR) and transmit pointers to meanings:\n1SLIP v1 planner executor RequestReview auth_module\nToken count:7 tokens (82% reduction)\nThe key insight:natural English words tokenize efficiently.RequestReviewis 1–2\ntokens across major tokenizers, while0x0011fragments into 3–4 tokens.\n2 The Universal Concept Reference\n2.1 The 4D Semantic Manifold\nThe UCR represents each anchor as a coordinate in a 4-dimensional semantic space:\nTable 2: UCR Semantic Dimensions\nDimension Values Purpose\nACTION request, inform, propose, evaluate Speech act type\nPOLARITY negative, neutral, positive Outcome sentiment\nDOMAIN task, plan, observation, control Context area\nURGENCY routine, elevated, critical Priority level\nThis structure provides:\n1.Interpretability:Anchors can be audited, extended, and reasoned about\n2\n2.Constraint surface:Agents can validate structural plausibility\n3.Semantic arithmetic:Combining dimensions yields predictable intents\n2.2 Anchor Structure\nEach anchor includes:\n1@dataclass\n2class UCRAnchor :\n3index : int # Unique ID (0 x0000 -0 xFFFF )\n4mnemonic : str # Wire token : \" RequestReview \"\n5canonical : str # Human description\n6coords : tuple [int , ...] # Position in manifold\n7is_core : bool # True if immutable core anchor\n•Core Range (0x0000–0x7FFF):Standard anchors, immutable per version\n•Extension Range (0x8000–0xFFFF):Installation-specific, evolvable\n2.3 Core Anchors\nTable 3: Core UCR Anchors by Category\nCategory Anchors\nRequestsRequestTask,RequestReview,RequestHelp,RequestPlan\nInformInformComplete,InformProgress,InformBlocked,InformStatus\nProposeProposePlan,ProposeChange,ProposeAlternative\nEvaluateEvalApprove,EvalReject,EvalNeedsWork\nMetaAccept,Reject,MetaAck,MetaHandoff,Fallback\n3 Protocol Specification\n3.1 Wire Format\n1SLIP v1 <src > <dst > <anchor >"}
-{"chunk_id": 2, "text": "n,ProposeChange,ProposeAlternative\nEvaluateEvalApprove,EvalReject,EvalNeedsWork\nMetaAccept,Reject,MetaAck,MetaHandoff,Fallback\n3 Protocol Specification\n3.1 Wire Format\n1SLIP v1 <src > <dst > <anchor > [ payload ...]\nTable 4: Wire Format Fields\nField Description\nSLIP v1Protocol marker and version\n<src>Source agent identifier\n<dst>Destination agent identifier\n<anchor>UCR mnemonic (e.g.,RequestReview)\n[payload]Optional space-separated parameters\nDesign Principles:\n•No special characters that fragment in BPE\n•Natural English words for efficient tokenization\n•Human-readable for debugging\n•Model-agnostic (works across GPT-4, Claude, Llama, etc.)\n3\n3.2 The Think-Quantize-Transmit Pattern\nThe TQT pattern consists of three stages:\n1.THINK:Agent forms natural language intent: “Please review the authentication code\nfor security”\n2.QUANTIZE:Map to nearest UCR anchor via keyword matching (fast, zero-dependency)\nor embedding similarity (accurate, requires ML). Result:RequestReview(confidence:\n0.89)\n3.TRANSMIT:Wire format:SLIP v1 dev reviewer RequestReview auth. Tokens: 7\n(vs 45 for JSON)\n4 Vector Quantization Engine\n4.1 Embedding-Based Retrieval\nThe vector quantization engine leverages sentence embeddings [Reimers and Gurevych, 2019]\nto map natural language intents to UCR anchors. Given a messagex, the vector engine embeds\nit and retrieves the best anchor by cosine similarity:\nk∗ = argmaxk cos(E(x),ck)(1)\nWhereE(x)is the thought embedding andck is the anchor centroid. This approach extends\nclassical quantization theory [Lloyd, 1982] to the semantic domain.\nA confidence thresholdτcontrols whether to emit an anchor or fall back to plaintext:\n1def quantize ( thought : str , threshold : float = 0.55) :\n2embedding = encode ( thought )\n3similarities = cosine ( embedding , centroids )\n4best_idx = argmax ( similarities )\n5\n6if similarities [ best_idx ] < threshold :\n7return Fallback ( thought )\n8\n9return anchors [ best_idx ]\n4.2 Graceful Degradation\nThe system operates in three modes:\nTable 5: Quantization Modes\nMode Dependencies Accuracy Use Case\nFull ML sentence-transformers 94% Production\nKeyword None 78% Edge/embedded\nFallback None 100% (passthrough) Novel intents\n5 Evolutionary Extension Layer\n5.1 The Drift Problem\nStatic codebooks degrade underconcept drift—new domains, task types, and terminology\nemerge over time. A codebook trained on software development fails on biotech"}
-{"chunk_id": 3, "text": "Extension Layer\n5.1 The Drift Problem\nStatic codebooks degrade underconcept drift—new domains, task types, and terminology\nemerge over time. A codebook trained on software development fails on biotech vocabulary.\n4\n5.2 Extension Learning\nSlipstream reserves the extension range (0x8000–0xFFFF) for learned anchors:\n1.Log:Messages with low quantization confidence are recorded\n2.Cluster:K-means identifies recurring semantic patterns [Sculley, 2010]\n3.Mint:New anchors are created with inferred 4D coordinates\n4.Register:Indices assigned in extension range; vector index rebuilt\n1class ExtensionManager :\n2def propose_extensions (self , fallbacks , min_cluster_size =3) :\n3embeddings = encode ( fallbacks )\n4clusters = kmeans ( embeddings , k= len ( fallbacks ) // min_cluster_size )\n5\n6new_anchors = []\n7for cluster in clusters :\n8if len ( cluster ) >= min_cluster_size :\n9centroid = mean ( embeddings [ cluster ])\n10exemplar = nearest_to_centroid ( cluster )\n11coords = infer_coords ( exemplar )\n12new_anchors . append ( mint_anchor ( centroid , exemplar , coords ))\n13\n14return new_anchors\n5.3 Governance\nExtension learning can be abused. Mitigations:\n•Minimum cluster size requirements\n•Rate limits on minting\n•Human approval gates for production\n•Provenance logging for each anchor\n6 Evaluation\n6.1 Token Efficiency\nTable 6: Token Efficiency Comparison\nMessage Type JSON Tokens SLIP Tokens Reduction\nTask delegation 47.3 8.2 82.7%\nStatus update 35.1 6.4 81.8%\nError report 52.0 9.1 82.5%\nAverage 41.9 7.4 82.3%\n5\n6.2 Cost Savings\nTable 7: Annual Cost Comparison by Deployment Scale\nScale Agents Msg/Day JSON Cost SLIP Cost Savings\nStartup 10 500 $3,600 $650 $2,950\nScale-up 50 5,000 $180,000 $32,400 $147,600\nEnterprise 1,000 500,000 $2,500,000 $450,000$2,050,000\n6.3 Semantic Fidelity\n•Retrieval accuracy:94% top-1 on intent classification\n•Coverage:88.7% of messages quantize without fallback\n•Codebook utilization:87% of anchors actively used\n7 Integration with AAIF Ecosystem\nSlipstream is designed as thetransport layerfor the Linux Foundation’s Agentic AI Founda-\ntion (AAIF) standards [Linux Foundation, 2025]:\n+-------------------------------------+\n| Application (Agent Logic) |\n+-----------------+-------------------+\n|\n+-----------------v-------------------+\n| MCP / A2A (Semantic Layer) | <- Discovery, capabilities\n+-----------------+-------------------+\n|\n+-----------------v-------"}
-{"chunk_id": 4, "text": "----------+-------------------+\n|\n+-----------------v-------------------+\n| MCP / A2A (Semantic Layer) | <- Discovery, capabilities\n+-----------------+-------------------+\n|\n+-----------------v-------------------+\n| Slipstream (Transport Layer) | <- 82% token reduction\n+-----------------+-------------------+\n|\n+-----------------v-------------------+\n| Network (HTTP, WebSocket, gRPC) |\n+-------------------------------------+\nCompatibility:Works transparently beneath Model Context Protocol (MCP) [Anthropic,\n2024] and Agent2Agent (A2A), like gRPC optimizes HTTP/2.\n8 Security Considerations\nTable 8: Security Threats and Mitigations\nThreat Mitigation\nPrompt injection via payloads Validate types; treat payloads as untrusted\nAnchor poisoning Min cluster size, rate limits, human approval\nOver-compression Allow fallback to plaintext; confidence thresholds\nSemantic drift Evolutionary layer; version-locked core anchors\n6\n9 Implementation\nA reference implementation is available asslipcore:\n1pip install slipcore\n1from slipcore import slip , decode , think_quantize_transmit\n2\n3# Direct message creation\n4wire = slip (\" alice \", \" bob \", \" RequestReview \", [\" auth_module \"])\n5# -> \" SLIP v1 alice bob RequestReview auth_module \"\n6\n7# Think - Quantize - Transmit pattern\n8wire = think_quantize_transmit (\n9\" Please review the authentication code \",\n10src =\" dev \", dst =\" reviewer \"\n11)\n12# -> \" SLIP v1 dev reviewer RequestReview \"\n13\n14# Decode\n15msg = decode ( wire )\n16print ( msg . anchor . canonical ) # \" Request review of work \"\n•Repository:https://github.com/anthony-maio/slipcore\n•License:Apache 2.0\n10 Conclusion\nSlipstream demonstrates thatsemantic quantizationis the necessary evolution for high-\nthroughput agent coordination. By grounding agents in a structured 4D manifold and trans-\nmitting natural-language mnemonics, we achieve 82% token reduction without sacrificing inter-\npretability or cross-model compatibility.\nThe protocol’s evolutionary layer enables adaptation to new domains while keeping core\nsemantics stable. As agent swarms scale, the shared UCR becomes a form of “collective\nunderstanding”—reducing not just tokens, but the cognitive overhead of coordination itself.\nReferences\nAnthropic. Model context protocol specification.https://modelcontextprotocol.io/, 2024.\nAccessed: 2024.\nLinux Foundation. Agentic AI foundation announcement.https://www.linuxfoundation."}
-{"chunk_id": 5, "text": "f.\nReferences\nAnthropic. Model context protocol specification.https://modelcontextprotocol.io/, 2024.\nAccessed: 2024.\nLinux Foundation. Agentic AI foundation announcement.https://www.linuxfoundation.\norg/press/agentic-ai-foundation, 2025. Accessed: 2025.\nStuart Lloyd. Least squares quantization in PCM.IEEE Transactions on Information Theory,\n28(2):129–137, 1982. doi: 10.1109/TIT.1982.1056489.\nNils Reimers and Iryna Gurevych. Sentence-BERT: Sentence embeddings using siamese BERT-\nnetworks. InProceedings of the 2019 Conference on Empirical Methods in Natural Lan-\nguage Processing and the 9th International Joint Conference on Natural Language Processing\n(EMNLP-IJCNLP), pages 3982–3992. Association for Computational Linguistics, 2019. doi:\n10.18653/v1/D19-1410.\nD. Sculley. Web-scale k-means clustering. InProceedings of the 19th International Conference\non World Wide Web, pages 1177–1178. ACM, 2010. doi: 10.1145/1772690.1772862.\n7"}

paper_context.txt

@@ -1,269 +0,0 @@
-Slipstream: Semantic Quantization for Efficient
-Multi-Agent Coordination
-Anthony Maio
-Independent Researcher
-anthony@making-minds.ai
-2025
-Abstract
-As multi-agent LLM systems scale,coordination bandwidthbecomes a primary cost
-driver: every token spent on routing, intent framing, and redundant context is paid repeat-
-edly across agents and turns. Current approaches waste 40–60% of compute on coordination
-overhead, with communication costs scalingO(n2)as agent counts increase.
-This paper introducesSlipstream, a protocol that performssemantic quantization:
-mapping free-form messages onto a sharedUniversal Concept Reference (UCR)and
-transmitting compactmnemonic anchorsthat identify structured intents. Unlike syn-
-tactic compression (which fails due to BPE tokenizer fragmentation), Slipstream transmits
-natural-language mnemonics that tokenize efficiently across model architectures.
-Slipstream combines (1) a symbolic4D semantic manifold—Action, Polarity, Domain,
-Urgency—with (2) a data-drivenvector engine(embeddings + nearest-centroid retrieval)
-plus anevolutionary extension layerthat learns new anchors from low-confidence traf-
-fic. Results show82% token reduction(41.9→7.4 tokens average) while maintaining
-semantic fidelity, making large-scale multi-agent deployments economically viable.
-Keywords:Semantic Quantization, Multi-Agent Systems, Protocol Standards, Token Ef-
-ficiency, Agentic AI
-1 Introduction
-1.1 The Coordination Crisis
-Agent swarms incur atokenizer tax: the repeated, non-semantic overhead of communicating
-message types, domains, and priorities. This overhead often dominates when messages are
-structured (routing, task dispatch, acknowledgements).
-A typical coordination message:
-1{
-2" sender ": " planning_agent ",
-3" recipient ": " execution_agent ",
-4" message_type ": " task_delegation ",
-5" content ": {
-6" request ": " Please review the authentication code ",
-7" priority ": " high "
-8}
-9}
-•Token count:∼45 tokens
-•Semantic content:∼10 tokens
-•Information density:22%
-1
-At GPT-4o pricing ($5/M input, $15/M output), a 50-agent deployment exchanging 1,000
-messages/day costs$180,000/yearin coordination tokens alone—before any work is per-
-formed.
-1.2 Why Syntactic Compression Fails
-Our initial approach, nSLIP v1, focused on syntactic minification:
-1REQ / TSK |s =7| d =3| act = review_auth
-•Expected tokens:8–10
-•Actual tokens with BPE:18–22
-The failure stems from Byte-Pair Encoding (BPE) tokenizer behavior. Punctuation and
-special characters fragment into separate tokens:
-Table 1: BPE Tokenization of Syntactic Compression
-Input Tokens
-REQ/TSK REQ,/,TSK= 3
-|s=7| |,s,=,7,|= 5
-This “Tokenizer Tax” negates syntactic savings entirely.
-1.3 The Solution: Semantic Quantization
-Instead of compressingsyntax, we quantizesemantics. Agents share a pre-agreed “concept
-codebook” (the UCR) and transmit pointers to meanings:
-1SLIP v1 planner executor RequestReview auth_module
-Token count:7 tokens (82% reduction)
-The key insight:natural English words tokenize efficiently.RequestReviewis 1–2
-tokens across major tokenizers, while0x0011fragments into 3–4 tokens.
-2 The Universal Concept Reference
-2.1 The 4D Semantic Manifold
-The UCR represents each anchor as a coordinate in a 4-dimensional semantic space:
-Table 2: UCR Semantic Dimensions
-Dimension Values Purpose
-ACTION request, inform, propose, evaluate Speech act type
-POLARITY negative, neutral, positive Outcome sentiment
-DOMAIN task, plan, observation, control Context area
-URGENCY routine, elevated, critical Priority level
-This structure provides:
-1.Interpretability:Anchors can be audited, extended, and reasoned about
-2
-2.Constraint surface:Agents can validate structural plausibility
-3.Semantic arithmetic:Combining dimensions yields predictable intents
-2.2 Anchor Structure
-Each anchor includes:
-1@dataclass
-2class UCRAnchor :
-3index : int # Unique ID (0 x0000 -0 xFFFF )
-4mnemonic : str # Wire token : " RequestReview "
-5canonical : str # Human description
-6coords : tuple [int , ...] # Position in manifold
-7is_core : bool # True if immutable core anchor
-•Core Range (0x0000–0x7FFF):Standard anchors, immutable per version
-•Extension Range (0x8000–0xFFFF):Installation-specific, evolvable
-2.3 Core Anchors
-Table 3: Core UCR Anchors by Category
-Category Anchors
-RequestsRequestTask,RequestReview,RequestHelp,RequestPlan
-InformInformComplete,InformProgress,InformBlocked,InformStatus
-ProposeProposePlan,ProposeChange,ProposeAlternative
-EvaluateEvalApprove,EvalReject,EvalNeedsWork
-MetaAccept,Reject,MetaAck,MetaHandoff,Fallback
-3 Protocol Specification
-3.1 Wire Format
-1SLIP v1 <src > <dst > <anchor > [ payload ...]
-Table 4: Wire Format Fields
-Field Description
-SLIP v1Protocol marker and version
-<src>Source agent identifier
-<dst>Destination agent identifier
-<anchor>UCR mnemonic (e.g.,RequestReview)
-[payload]Optional space-separated parameters
-Design Principles:
-•No special characters that fragment in BPE
-•Natural English words for efficient tokenization
-•Human-readable for debugging
-•Model-agnostic (works across GPT-4, Claude, Llama, etc.)
-3
-3.2 The Think-Quantize-Transmit Pattern
-The TQT pattern consists of three stages:
-1.THINK:Agent forms natural language intent: “Please review the authentication code
-for security”
-2.QUANTIZE:Map to nearest UCR anchor via keyword matching (fast, zero-dependency)
-or embedding similarity (accurate, requires ML). Result:RequestReview(confidence:
-0.89)
-3.TRANSMIT:Wire format:SLIP v1 dev reviewer RequestReview auth. Tokens: 7
-(vs 45 for JSON)
-4 Vector Quantization Engine
-4.1 Embedding-Based Retrieval
-The vector quantization engine leverages sentence embeddings [Reimers and Gurevych, 2019]
-to map natural language intents to UCR anchors. Given a messagex, the vector engine embeds
-it and retrieves the best anchor by cosine similarity:
-k∗ = argmaxk cos(E(x),ck)(1)
-WhereE(x)is the thought embedding andck is the anchor centroid. This approach extends
-classical quantization theory [Lloyd, 1982] to the semantic domain.
-A confidence thresholdτcontrols whether to emit an anchor or fall back to plaintext:
-1def quantize ( thought : str , threshold : float = 0.55) :
-2embedding = encode ( thought )
-3similarities = cosine ( embedding , centroids )
-4best_idx = argmax ( similarities )
-5
-6if similarities [ best_idx ] < threshold :
-7return Fallback ( thought )
-8
-9return anchors [ best_idx ]
-4.2 Graceful Degradation
-The system operates in three modes:
-Table 5: Quantization Modes
-Mode Dependencies Accuracy Use Case
-Full ML sentence-transformers 94% Production
-Keyword None 78% Edge/embedded
-Fallback None 100% (passthrough) Novel intents
-5 Evolutionary Extension Layer
-5.1 The Drift Problem
-Static codebooks degrade underconcept drift—new domains, task types, and terminology
-emerge over time. A codebook trained on software development fails on biotech vocabulary.
-4
-5.2 Extension Learning
-Slipstream reserves the extension range (0x8000–0xFFFF) for learned anchors:
-1.Log:Messages with low quantization confidence are recorded
-2.Cluster:K-means identifies recurring semantic patterns [Sculley, 2010]
-3.Mint:New anchors are created with inferred 4D coordinates
-4.Register:Indices assigned in extension range; vector index rebuilt
-1class ExtensionManager :
-2def propose_extensions (self , fallbacks , min_cluster_size =3) :
-3embeddings = encode ( fallbacks )
-4clusters = kmeans ( embeddings , k= len ( fallbacks ) // min_cluster_size )
-5
-6new_anchors = []
-7for cluster in clusters :
-8if len ( cluster ) >= min_cluster_size :
-9centroid = mean ( embeddings [ cluster ])
-10exemplar = nearest_to_centroid ( cluster )
-11coords = infer_coords ( exemplar )
-12new_anchors . append ( mint_anchor ( centroid , exemplar , coords ))
-13
-14return new_anchors
-5.3 Governance
-Extension learning can be abused. Mitigations:
-•Minimum cluster size requirements
-•Rate limits on minting
-•Human approval gates for production
-•Provenance logging for each anchor
-6 Evaluation
-6.1 Token Efficiency
-Table 6: Token Efficiency Comparison
-Message Type JSON Tokens SLIP Tokens Reduction
-Task delegation 47.3 8.2 82.7%
-Status update 35.1 6.4 81.8%
-Error report 52.0 9.1 82.5%
-Average 41.9 7.4 82.3%
-5
-6.2 Cost Savings
-Table 7: Annual Cost Comparison by Deployment Scale
-Scale Agents Msg/Day JSON Cost SLIP Cost Savings
-Startup 10 500 $3,600 $650 $2,950
-Scale-up 50 5,000 $180,000 $32,400 $147,600
-Enterprise 1,000 500,000 $2,500,000 $450,000$2,050,000
-6.3 Semantic Fidelity
-•Retrieval accuracy:94% top-1 on intent classification
-•Coverage:88.7% of messages quantize without fallback
-•Codebook utilization:87% of anchors actively used
-7 Integration with AAIF Ecosystem
-Slipstream is designed as thetransport layerfor the Linux Foundation’s Agentic AI Founda-
-tion (AAIF) standards [Linux Foundation, 2025]:
-+-------------------------------------+
-| Application (Agent Logic) |
-+-----------------+-------------------+
-|
-+-----------------v-------------------+
-| MCP / A2A (Semantic Layer) | <- Discovery, capabilities
-+-----------------+-------------------+
-|
-+-----------------v-------------------+
-| Slipstream (Transport Layer) | <- 82% token reduction
-+-----------------+-------------------+
-|
-+-----------------v-------------------+
-| Network (HTTP, WebSocket, gRPC) |
-+-------------------------------------+
-Compatibility:Works transparently beneath Model Context Protocol (MCP) [Anthropic,
-2024] and Agent2Agent (A2A), like gRPC optimizes HTTP/2.
-8 Security Considerations
-Table 8: Security Threats and Mitigations
-Threat Mitigation
-Prompt injection via payloads Validate types; treat payloads as untrusted
-Anchor poisoning Min cluster size, rate limits, human approval
-Over-compression Allow fallback to plaintext; confidence thresholds
-Semantic drift Evolutionary layer; version-locked core anchors
-6
-9 Implementation
-A reference implementation is available asslipcore:
-1pip install slipcore
-1from slipcore import slip , decode , think_quantize_transmit
-2
-3# Direct message creation
-4wire = slip (" alice ", " bob ", " RequestReview ", [" auth_module "])
-5# -> " SLIP v1 alice bob RequestReview auth_module "
-6
-7# Think - Quantize - Transmit pattern
-8wire = think_quantize_transmit (
-9" Please review the authentication code ",
-10src =" dev ", dst =" reviewer "
-11)
-12# -> " SLIP v1 dev reviewer RequestReview "
-13
-14# Decode
-15msg = decode ( wire )
-16print ( msg . anchor . canonical ) # " Request review of work "
-•Repository:https://github.com/anthony-maio/slipcore
-•License:Apache 2.0
-10 Conclusion
-Slipstream demonstrates thatsemantic quantizationis the necessary evolution for high-
-throughput agent coordination. By grounding agents in a structured 4D manifold and trans-
-mitting natural-language mnemonics, we achieve 82% token reduction without sacrificing inter-
-pretability or cross-model compatibility.
-The protocol’s evolutionary layer enables adaptation to new domains while keeping core
-semantics stable. As agent swarms scale, the shared UCR becomes a form of “collective
-understanding”—reducing not just tokens, but the cognitive overhead of coordination itself.
-References
-Anthropic. Model context protocol specification.https://modelcontextprotocol.io/, 2024.
-Accessed: 2024.
-Linux Foundation. Agentic AI foundation announcement.https://www.linuxfoundation.
-org/press/agentic-ai-foundation, 2025. Accessed: 2025.
-Stuart Lloyd. Least squares quantization in PCM.IEEE Transactions on Information Theory,
-28(2):129–137, 1982. doi: 10.1109/TIT.1982.1056489.
-Nils Reimers and Iryna Gurevych. Sentence-BERT: Sentence embeddings using siamese BERT-
-networks. InProceedings of the 2019 Conference on Empirical Methods in Natural Lan-
-guage Processing and the 9th International Joint Conference on Natural Language Processing
-(EMNLP-IJCNLP), pages 3982–3992. Association for Computational Linguistics, 2019. doi:
-10.18653/v1/D19-1410.
-D. Sculley. Web-scale k-means clustering. InProceedings of the 19th International Conference
-on World Wide Web, pages 1177–1178. ACM, 2010. doi: 10.1145/1772690.1772862.
-7

quantizer.py

@@ -1,818 +0,0 @@
-"""
-Semantic Quantizer - The Think-Quantize-Transmit Engine
-
-Maps agent thoughts (natural language) to UCR anchors.
-Supports three modes:
-1. Keyword-based (fast, no dependencies)
-2. Embedding-based with centroids (accurate, requires sentence-transformers)
-3. Hybrid with CoordsInferer (prototype similarity + heuristics)
-
-Also handles:
-- Fallback detection (when confidence is too low)
-- Usage tracking (for UCR evolution)
-- Coordinate inference for new anchors
-"""
-
-from __future__ import annotations
-from dataclasses import dataclass, field
-from typing import Optional, Callable, Dict, List, Tuple
-from collections import Counter
-import re
-
-try:
-    from .ucr import UCR, UCRAnchor, get_default_ucr, CORE_RANGE_END
-except ImportError:
-    from ucr import UCR, UCRAnchor, get_default_ucr, CORE_RANGE_END
-
-
-# ============ Optional Dependencies ============
-
-try:
-    import numpy as np
-    HAS_NUMPY = True
-except ImportError:
-    np = None  # type: ignore
-    HAS_NUMPY = False
-
-
-# ============ Semantic Coordinates ============
-
-@dataclass(frozen=True)
-class SemanticCoords:
-    """
-    4D logical position of an intent in the semantic manifold.
-
-    This is a human-readable representation used during coordinate inference.
-    Maps to UCR's numeric coords via ACTION_MAP, DOMAIN_MAP, etc.
-    """
-    action: str    # REQ, INF, EVAL, CMD, OBS, PROP, META
-    polarity: int  # -1 (negative), 0 (neutral), 1 (positive)
-    domain: str    # TASK, QA, INFRA, AUTH, ERR, DOC, META, GEN
-    urgency: int   # 0 (routine) to 3 (critical)
-
-
-# Map v3-style string coords to v2 numeric coords
-ACTION_MAP = {"OBS": 0, "INF": 1, "ASK": 2, "REQ": 3, "PROP": 4, "COMMIT": 5, "EVAL": 6, "META": 7, "CMD": 3}
-DOMAIN_MAP = {"TASK": 0, "PLAN": 1, "OBS": 2, "EVAL": 3, "CTRL": 4, "RES": 5, "ERR": 6, "GEN": 7,
-              "QA": 3, "INFRA": 5, "AUTH": 4, "DOC": 1, "META": 4}
-
-
-def semantic_coords_to_tuple(sc: SemanticCoords) -> tuple[int, ...]:
-    """Convert SemanticCoords to UCR numeric tuple."""
-    action = ACTION_MAP.get(sc.action.upper(), 3)  # default REQ
-    domain = DOMAIN_MAP.get(sc.domain.upper(), 7)  # default GEN
-    # Map polarity: -1->1, 0->4, 1->6
-    polarity = {-1: 1, 0: 4, 1: 6}.get(sc.polarity, 4)
-    # Map urgency 0-3 to 0-7: 0->1, 1->3, 2->5, 3->7
-    urgency = min(7, max(0, sc.urgency * 2 + 1))
-    return (action, polarity, domain, urgency)
-
-
-# ============ Coordinate Inference (from v3) ============
-
-class CoordsInferer:
-    """
-    Assigns (Action, Polarity, Domain, Urgency) to text.
-
-    Hybrid approach:
-    - Heuristics for urgency and polarity (reliable, fast)
-    - Optional prototype embedding similarity for action/domain refinement
-
-    LIMITATIONS AND CAVEATS:
-    ========================
-
-    1. HEURISTIC FRAGILITY:
-       - Keyword matching is context-blind ("please" could be polite or pleading)
-       - Urgency detection relies on explicit markers ("ASAP", "urgent")
-       - Domain detection depends on domain-specific vocabulary
-
-    2. PROTOTYPE EMBEDDING ISSUES:
-       - Limited prototype phrases (3 per action, 2 per domain)
-       - English-only prototypes; other languages will use fallback
-       - Short messages may not have enough signal for reliable similarity
-
-    3. KNOWN FAILURE MODES:
-       - Sarcasm/irony: "Great, another bug" -> incorrectly infers positive polarity
-       - Questions phrased as statements: "I wonder if..." -> misses ASK action
-       - Multi-intent: "Review and deploy this" -> only captures first action
-       - Implicit urgency: "CEO is waiting" -> misses critical urgency
-
-    4. RECOMMENDED USAGE:
-       - Use as initialization for finetuned models, not production truth
-       - Always validate inferred coords against anchor centroids
-       - Track fallback rate; >15% indicates poor UCR coverage
-       - For production: finetune a small model on CoordsInferer output
-
-    5. ACCURACY ESTIMATES (informal testing):
-       - Urgency:  ~80% (explicit markers work well)
-       - Polarity: ~70% (sentiment is hard)
-       - Action:   ~65% with embeddings, ~55% heuristic-only
-       - Domain:   ~60% (highly vocabulary-dependent)
-
-    Ported from v3's coordinate inference system.
-    """
-
-    def __init__(self, embed_batch: Optional[Callable] = None):
-        self._embed_batch = embed_batch
-        self._proto_action: Dict[str, "np.ndarray"] = {}
-        self._proto_domain: Dict[str, "np.ndarray"] = {}
-
-        # Prototype phrases (short sentences > single tokens for embeddings)
-        self._action_phrases = {
-            "REQ": [
-                "Please do this task.",
-                "Can you help with this request?",
-                "I need you to do something.",
-            ],
-            "INF": [
-                "FYI, here is a status update.",
-                "I finished the task.",
-                "This is an informational update.",
-            ],
-            "EVAL": [
-                "Please review and evaluate this.",
-                "Assess the quality of this work.",
-                "Give a critique of this design.",
-            ],
-            "CMD": [
-                "Do this immediately.",
-                "Execute this command.",
-                "Run the operation now.",
-            ],
-            "OBS": [
-                "I noticed something changed.",
-                "The current state is...",
-                "I observed an issue.",
-            ],
-            "PROP": [
-                "I suggest we do this.",
-                "Here's my proposal.",
-                "We could try this approach.",
-            ],
-        }
-        self._domain_phrases = {
-            "TASK": ["Assign a task ticket.", "Work item status update."],
-            "QA": ["Request code review.", "Review pull request."],
-            "INFRA": ["Scale the Kubernetes cluster.", "Deploy infrastructure change."],
-            "AUTH": ["OAuth login issue.", "Authentication and authorization."],
-            "ERR": ["System error occurred.", "Critical failure and outage."],
-            "DOC": ["Update documentation.", "Write technical docs."],
-            "META": ["Discuss process and coordination.", "Team protocol and planning."],
-            "GEN": ["General conversation.", "Generic request or update."],
-        }
-
-    def prime(self) -> None:
-        """Compute prototype embeddings (if embedder available)."""
-        if not self._embed_batch or not HAS_NUMPY:
-            return
-
-        def _norm(vec):
-            n = np.linalg.norm(vec)
-            return vec / (n + 1e-12) if n > 0 else vec
-
-        # Actions
-        action_labels = list(self._action_phrases.keys())
-        action_texts = [" ".join(self._action_phrases[a]) for a in action_labels]
-        action_vecs = self._embed_batch(action_texts)
-        for a, v in zip(action_labels, action_vecs):
-            self._proto_action[a] = _norm(np.asarray(v, dtype=np.float32))
-
-        # Domains
-        domain_labels = list(self._domain_phrases.keys())
-        domain_texts = [" ".join(self._domain_phrases[d]) for d in domain_labels]
-        domain_vecs = self._embed_batch(domain_texts)
-        for d, v in zip(domain_labels, domain_vecs):
-            self._proto_domain[d] = _norm(np.asarray(v, dtype=np.float32))
-
-    def infer(self, text: str, vec: Optional["np.ndarray"] = None) -> SemanticCoords:
-        """
-        Infer semantic coordinates from text.
-
-        Args:
-            text: The input text to analyze
-            vec: Optional pre-computed embedding vector for refinement
-
-        Returns:
-            SemanticCoords with inferred action, polarity, domain, urgency
-        """
-        t = text.strip()
-        low = t.lower()
-
-        # --- urgency (heuristic) ---
-        urgency = 0
-        if any(k in low for k in ("critical", "sev1", "sev-1", "p0", "immediately", "right now")):
-            urgency = 3
-        elif any(k in low for k in ("urgent", "asap", "high priority", "blocker")):
-            urgency = 2
-        elif any(k in low for k in ("soon", "priority", "important")):
-            urgency = 1
-
-        # --- polarity (heuristic) ---
-        polarity = 0
-        if any(k in low for k in ("error", "failed", "failure", "crash", "broken", "outage", "bug", "can't", "cannot")):
-            polarity = -1
-        elif any(k in low for k in ("fixed", "resolved", "success", "completed", "done", "working now", "all good")):
-            polarity = 1
-
-        # --- action (heuristic) ---
-        action = "INF"
-        if t.endswith("?") or low.startswith(("can you", "could you", "would you", "please")):
-            action = "REQ"
-        if any(k in low for k in ("review", "critique", "evaluate", "assess")):
-            action = "EVAL"
-        if any(k in low for k in ("do this", "run ", "execute", "deploy", "scale ", "restart")) and urgency >= 2:
-            action = "CMD"
-        if any(k in low for k in ("i noticed", "i see", "detected", "observed")):
-            action = "OBS"
-        if any(k in low for k in ("i suggest", "i propose", "we could", "how about")):
-            action = "PROP"
-
-        # --- domain (heuristic) ---
-        domain = "GEN"
-        if any(k in low for k in ("kubernetes", "k8s", "cluster", "deploy", "terraform", "docker", "infra", "server", "latency")):
-            domain = "INFRA"
-        elif any(k in low for k in ("auth", "oauth", "login", "jwt", "sso", "permission")):
-            domain = "AUTH"
-        elif any(k in low for k in ("review", "pull request", "pr ", "qa", "test")):
-            domain = "QA"
-        elif any(k in low for k in ("task", "ticket", "jira", "backlog")):
-            domain = "TASK"
-        elif any(k in low for k in ("error", "exception", "stacktrace", "failed", "failure", "outage")):
-            domain = "ERR"
-        elif any(k in low for k in ("doc", "documentation", "readme", "spec", "paper")):
-            domain = "DOC"
-        elif any(k in low for k in ("protocol", "manifold", "coordination", "orchestrator")):
-            domain = "META"
-
-        # Optional refinement via prototype similarity
-        if vec is not None and HAS_NUMPY and self._proto_action and self._proto_domain:
-            def _norm(v):
-                n = np.linalg.norm(v)
-                return v / (n + 1e-12) if n > 0 else v
-
-            v = _norm(vec.astype(np.float32, copy=False))
-
-            # Action refine
-            a_best, a_score = action, -1.0
-            for a, pv in self._proto_action.items():
-                s = float(np.dot(v, pv))
-                if s > a_score:
-                    a_best, a_score = a, s
-            if a_score >= 0.40:
-                action = a_best
-
-            # Domain refine
-            d_best, d_score = domain, -1.0
-            for d, pv in self._proto_domain.items():
-                s = float(np.dot(v, pv))
-                if s > d_score:
-                    d_best, d_score = d, s
-            if d_score >= 0.35:
-                domain = d_best
-
-        # If action is REQ and urgency not set, default to 1
-        if action == "REQ" and urgency == 0:
-            urgency = 1
-
-        return SemanticCoords(action=action, polarity=polarity, domain=domain, urgency=urgency)
-
-
-# Global coords inferer instance
-_coords_inferer: Optional[CoordsInferer] = None
-
-
-def get_coords_inferer() -> CoordsInferer:
-    """Get or create the default CoordsInferer."""
-    global _coords_inferer
-    if _coords_inferer is None:
-        _coords_inferer = CoordsInferer()
-    return _coords_inferer
-
-
-def infer_coords(text: str, vec: Optional["np.ndarray"] = None) -> tuple[int, ...]:
-    """
-    Infer UCR-compatible coordinates from text.
-
-    Returns a tuple of 4 integers suitable for UCRAnchor.coords.
-    """
-    inferer = get_coords_inferer()
-    sc = inferer.infer(text, vec)
-    return semantic_coords_to_tuple(sc)
-
-
-# ============ Quantization Result ============
-
-@dataclass
-class QuantizeResult:
-    """
-    Result of quantizing a thought to a UCR anchor.
-
-    Attributes:
-        anchor: The matched UCR anchor
-        confidence: How well the thought matches (0.0-1.0)
-        method: How the match was made ("keyword", "embedding", "fallback")
-        alternatives: Other possible matches with their scores
-    """
-    anchor: UCRAnchor
-    confidence: float
-    method: str
-    alternatives: list[tuple[UCRAnchor, float]] = field(default_factory=list)
-
-    @property
-    def is_fallback(self) -> bool:
-        return self.anchor.mnemonic == "Fallback"
-
-    @property
-    def is_high_confidence(self) -> bool:
-        return self.confidence >= 0.7
-
-
-# ============ Keyword-Based Quantizer ============
-
-# Keyword patterns for each anchor category
-_KEYWORD_PATTERNS: dict[str, list[str]] = {
-    # Observations
-    "ObserveState": ["state", "current", "status", "environment", "system state"],
-    "ObserveChange": ["changed", "detected", "noticed", "updated", "modified"],
-    "ObserveError": ["error", "exception", "failed", "crash", "bug"],
-
-    # Information
-    "InformResult": ["result", "output", "computed", "calculated", "returns"],
-    "InformStatus": ["status", "update", "progress", "currently"],
-    "InformComplete": ["complete", "finished", "done", "completed", "success"],
-    "InformBlocked": ["blocked", "waiting", "stuck", "depends on", "need"],
-    "InformProgress": ["progress", "working on", "making progress", "underway"],
-
-    # Questions
-    "AskClarify": ["clarify", "what do you mean", "unclear", "confused", "explain"],
-    "AskStatus": ["what is the status", "how is", "progress on", "update on"],
-    "AskPermission": ["can i", "may i", "permission", "allowed", "okay to"],
-    "AskResource": ["available", "resource", "capacity", "do we have"],
-
-    # Requests
-    "RequestTask": ["please do", "execute", "perform", "run", "implement"],
-    "RequestPlan": ["create a plan", "plan for", "how should we", "strategy"],
-    "RequestReview": ["review", "check", "look at", "evaluate", "feedback"],
-    "RequestHelp": ["help", "assist", "support", "guidance", "advice"],
-    "RequestCancel": ["cancel", "abort", "stop", "nevermind", "forget"],
-    "RequestPriority": ["priority", "urgent", "expedite", "escalate"],
-    "RequestResource": ["allocate", "provision", "need resource", "require"],
-
-    # Proposals
-    "ProposePlan": ["propose", "suggest", "recommendation", "i think we should"],
-    "ProposeChange": ["change", "modify", "alter", "adjust"],
-    "ProposeAlternative": ["alternative", "instead", "another approach", "option"],
-    "ProposeRollback": ["rollback", "revert", "undo", "go back"],
-
-    # Commitments
-    "CommitTask": ["i will", "i'll do", "on it", "taking this", "i commit"],
-    "CommitDeadline": ["by", "deadline", "eta", "deliver by"],
-    "CommitResource": ["allocating", "providing", "assigning"],
-
-    # Evaluations
-    "EvalApprove": ["approved", "lgtm", "looks good", "accept", "ship it"],
-    "EvalReject": ["rejected", "no", "denied", "not acceptable", "wrong"],
-    "EvalNeedsWork": ["needs work", "revise", "changes needed", "almost"],
-    "EvalComplete": ["complete", "done", "finished", "all good"],
-    "EvalBlocked": ["blocked", "cannot proceed", "impediment"],
-
-    # Meta
-    "MetaAck": ["ack", "acknowledged", "got it", "received", "understood"],
-    "MetaSync": ["sync", "ping", "alive", "heartbeat"],
-    "MetaHandoff": ["handoff", "transfer", "passing to", "your turn"],
-    "MetaEscalate": ["escalate", "raise", "need manager", "above my paygrade"],
-    "MetaAbort": ["abort", "emergency stop", "halt", "critical failure"],
-
-    # Accept/Reject
-    "Accept": ["yes", "accept", "agreed", "confirmed", "affirmative"],
-    "Reject": ["no", "reject", "disagree", "refuse", "decline"],
-    "AcceptWithCondition": ["yes but", "if", "conditional", "provided that"],
-    "Defer": ["later", "defer", "postpone", "not now", "revisit"],
-
-    # Errors
-    "ErrorGeneric": ["error", "failed", "exception"],
-    "ErrorTimeout": ["timeout", "timed out", "too slow"],
-    "ErrorResource": ["resource unavailable", "out of", "exhausted"],
-    "ErrorPermission": ["permission denied", "unauthorized", "forbidden"],
-    "ErrorValidation": ["invalid", "validation failed", "bad input"],
-}
-
-
-def _keyword_score(thought: str, patterns: list[str]) -> float:
-    """Score how well a thought matches keyword patterns."""
-    thought_lower = thought.lower()
-    matches = 0
-    for pattern in patterns:
-        if pattern.lower() in thought_lower:
-            # Longer patterns are stronger signals
-            matches += len(pattern.split())
-    # Normalize to 0-1 range (cap at 1.0)
-    return min(1.0, matches / 3.0)
-
-
-class KeywordQuantizer:
-    """
-    Simple keyword-based quantizer. No ML dependencies.
-    Good for bootstrapping and low-latency scenarios.
-    """
-
-    def __init__(self, ucr: Optional[UCR] = None, fallback_threshold: float = 0.2):
-        self.ucr = ucr or get_default_ucr()
-        self.fallback_threshold = fallback_threshold
-        self._usage_stats: Counter = Counter()
-
-    def quantize(self, thought: str) -> QuantizeResult:
-        """
-        Map a natural language thought to the best UCR anchor.
-
-        Args:
-            thought: The agent's thought/intent in natural language
-
-        Returns:
-            QuantizeResult with the best anchor and confidence score
-        """
-        scores: list[tuple[UCRAnchor, float]] = []
-
-        for mnemonic, patterns in _KEYWORD_PATTERNS.items():
-            anchor = self.ucr.get_by_mnemonic(mnemonic)
-            if anchor:
-                score = _keyword_score(thought, patterns)
-                if score > 0:
-                    scores.append((anchor, score))
-
-        # Sort by score descending
-        scores.sort(key=lambda x: x[1], reverse=True)
-
-        if not scores or scores[0][1] < self.fallback_threshold:
-            # Use fallback
-            fallback = self.ucr.get_by_mnemonic("Fallback")
-            self._usage_stats["_fallback"] += 1
-            return QuantizeResult(
-                anchor=fallback,
-                confidence=0.0,
-                method="fallback",
-                alternatives=scores[:3],
-            )
-
-        best_anchor, best_score = scores[0]
-        self._usage_stats[best_anchor.mnemonic] += 1
-
-        return QuantizeResult(
-            anchor=best_anchor,
-            confidence=best_score,
-            method="keyword",
-            alternatives=scores[1:4],
-        )
-
-    def get_usage_stats(self) -> dict[str, int]:
-        """Get usage statistics for UCR evolution analysis."""
-        return dict(self._usage_stats)
-
-    def get_fallback_rate(self) -> float:
-        """Get the rate of fallback usage (indicates UCR coverage gaps)."""
-        total = sum(self._usage_stats.values())
-        if total == 0:
-            return 0.0
-        return self._usage_stats["_fallback"] / total
-
-
-# ============ Embedding-Based Quantizer (Enhanced) ============
-
-class EmbeddingQuantizer:
-    """
-    Embedding-based quantizer using sentence-transformers.
-
-    Enhanced with v3's centroid matrix approach:
-    - Pre-computes normalized centroid matrix for fast similarity search
-    - Supports anchor centroids (when available) or on-the-fly embedding
-    - Includes CoordsInferer for prototype refinement
-
-    Usage:
-        quantizer = EmbeddingQuantizer()
-        result = quantizer.quantize("I need someone to review this code")
-    """
-
-    def __init__(
-        self,
-        ucr: Optional[UCR] = None,
-        model_name: str = "all-MiniLM-L6-v2",
-        fallback_threshold: float = 0.55,
-    ):
-        if not HAS_NUMPY:
-            raise ImportError("numpy is required for EmbeddingQuantizer")
-
-        self.ucr = ucr or get_default_ucr()
-        self.fallback_threshold = fallback_threshold
-        self._usage_stats: Counter = Counter()
-        self._fallback_buffer: List[str] = []  # Track low-confidence messages
-
-        # Lazy load sentence-transformers
-        self._model = None
-        self._model_name = model_name
-
-        # Centroid matrix (normalized) for fast similarity
-        self._centroids_matrix: Optional["np.ndarray"] = None
-        self._anchor_indices: List[int] = []  # Maps matrix row to anchor index
-        self._embed_dim: Optional[int] = None
-
-        # Coords inferer with prototype refinement
-        self._coords_inferer: Optional[CoordsInferer] = None
-
-    def _ensure_model(self):
-        """Lazy load the embedding model and build centroid matrix."""
-        if self._model is not None:
-            return
-
-        try:
-            from sentence_transformers import SentenceTransformer
-        except ImportError:
-            raise ImportError(
-                "sentence-transformers is required for EmbeddingQuantizer. "
-                "Install with: pip install sentence-transformers"
-            )
-
-        self._model = SentenceTransformer(self._model_name)
-        self._rebuild_index()
-
-        # Initialize coords inferer with embeddings
-        self._coords_inferer = CoordsInferer(embed_batch=self._embed_batch)
-        self._coords_inferer.prime()
-
-    def _embed_batch(self, texts: List[str]) -> "np.ndarray":
-        """Embed a batch of texts and return normalized vectors."""
-        if not self._model:
-            self._ensure_model()
-        vecs = self._model.encode(texts, convert_to_numpy=True)
-        vecs = np.asarray(vecs, dtype=np.float32)
-        self._embed_dim = vecs.shape[1]
-        # Normalize rows
-        norms = np.linalg.norm(vecs, axis=1, keepdims=True)
-        return vecs / (norms + 1e-12)
-
-    def _embed_one(self, text: str) -> "np.ndarray":
-        """Embed a single text and return normalized vector."""
-        return self._embed_batch([text])[0]
-
-    def _rebuild_index(self):
-        """Build/rebuild the centroid matrix from UCR anchors."""
-        if not self.ucr.anchors:
-            self._anchor_indices = []
-            self._centroids_matrix = None
-            return
-
-        self._anchor_indices = sorted(self.ucr.anchors.keys())
-        anchors = [self.ucr.anchors[idx] for idx in self._anchor_indices]
-
-        # Check if anchors have pre-computed centroids
-        has_centroids = all(a.centroid is not None for a in anchors)
-
-        if has_centroids:
-            # Use pre-computed centroids
-            mat = np.asarray([a.centroid for a in anchors], dtype=np.float32)
-        else:
-            # Compute centroids from canonical texts
-            canonical_texts = [a.canonical for a in anchors]
-            mat = self._embed_batch(canonical_texts)
-            # Optionally store centroids back to anchors
-            for anchor, vec in zip(anchors, mat):
-                anchor.centroid = vec.tolist()
-
-        # Normalize rows
-        norms = np.linalg.norm(mat, axis=1, keepdims=True)
-        self._centroids_matrix = mat / (norms + 1e-12)
-
-    def quantize(self, thought: str) -> QuantizeResult:
-        """
-        Map a natural language thought to the best UCR anchor using embeddings.
-
-        Uses normalized cosine similarity against centroid matrix for fast lookup.
-
-        Args:
-            thought: The agent's thought/intent in natural language
-
-        Returns:
-            QuantizeResult with the best anchor and confidence score
-        """
-        self._ensure_model()
-
-        if self._centroids_matrix is None or len(self._anchor_indices) == 0:
-            fallback = self.ucr.get_by_mnemonic("Fallback")
-            self._fallback_buffer.append(thought)
-            return QuantizeResult(
-                anchor=fallback,
-                confidence=0.0,
-                method="fallback",
-                alternatives=[],
-            )
-
-        # Embed the thought (normalized)
-        thought_vec = self._embed_one(thought)
-
-        # Compute cosine similarities (dot product of normalized vectors)
-        similarities = np.dot(self._centroids_matrix, thought_vec)
-
-        # Get top matches
-        top_locs = np.argsort(similarities)[::-1][:5]
-        scores = []
-        for loc in top_locs:
-            anchor_idx = self._anchor_indices[loc]
-            anchor = self.ucr.anchors[anchor_idx]
-            scores.append((anchor, float(similarities[loc])))
-
-        best_anchor, best_score = scores[0]
-
-        if best_score < self.fallback_threshold:
-            fallback = self.ucr.get_by_mnemonic("Fallback")
-            self._usage_stats["_fallback"] += 1
-            self._fallback_buffer.append(thought)
-            return QuantizeResult(
-                anchor=fallback,
-                confidence=best_score,
-                method="fallback",
-                alternatives=scores[:3],
-            )
-
-        self._usage_stats[best_anchor.mnemonic] += 1
-
-        return QuantizeResult(
-            anchor=best_anchor,
-            confidence=best_score,
-            method="embedding",
-            alternatives=scores[1:4],
-        )
-
-    def get_fallback_buffer(self) -> List[str]:
-        """Get the buffer of low-confidence messages for extension learning."""
-        return self._fallback_buffer.copy()
-
-    def clear_fallback_buffer(self) -> None:
-        """Clear the fallback buffer after extension learning."""
-        self._fallback_buffer.clear()
-
-    def infer_coords_for_text(self, text: str) -> tuple[int, ...]:
-        """
-        Infer semantic coordinates for a text using the enhanced CoordsInferer.
-
-        Returns UCR-compatible coordinate tuple.
-        """
-        self._ensure_model()
-        vec = self._embed_one(text)
-        if self._coords_inferer:
-            sc = self._coords_inferer.infer(text, vec)
-            return semantic_coords_to_tuple(sc)
-        return infer_coords(text, vec)
-
-    def compute_centroid(self, texts: List[str]) -> List[float]:
-        """
-        Compute the centroid embedding for a cluster of texts.
-
-        Useful for creating new extension anchors.
-        """
-        self._ensure_model()
-        embeds = self._embed_batch(texts)
-        centroid = np.mean(embeds, axis=0)
-        # Normalize
-        norm = np.linalg.norm(centroid)
-        if norm > 0:
-            centroid = centroid / norm
-        return centroid.tolist()
-
-    def get_usage_stats(self) -> dict[str, int]:
-        """Get usage statistics for UCR evolution analysis."""
-        return dict(self._usage_stats)
-
-    def get_fallback_rate(self) -> float:
-        """Get the rate of fallback usage."""
-        total = sum(self._usage_stats.values())
-        if total == 0:
-            return 0.0
-        return self._usage_stats["_fallback"] / total
-
-
-# ============ Auto-selecting Quantizer ============
-
-def create_quantizer(
-    ucr: Optional[UCR] = None,
-    prefer_embeddings: bool = True,
-    fallback_threshold: float = 0.3,
-) -> KeywordQuantizer | EmbeddingQuantizer:
-    """
-    Create the best available quantizer.
-
-    Args:
-        ucr: UCR instance to use
-        prefer_embeddings: Try to use embedding quantizer if available
-        fallback_threshold: Confidence threshold for fallback
-
-    Returns:
-        EmbeddingQuantizer if available and preferred, else KeywordQuantizer
-    """
-    if prefer_embeddings:
-        try:
-            return EmbeddingQuantizer(ucr=ucr, fallback_threshold=fallback_threshold)
-        except ImportError:
-            pass
-
-    return KeywordQuantizer(ucr=ucr, fallback_threshold=fallback_threshold)
-
-
-# ============ High-Level API ============
-
-_default_quantizer: Optional[KeywordQuantizer | EmbeddingQuantizer] = None
-
-
-def quantize(thought: str) -> QuantizeResult:
-    """
-    Quantize a thought to a UCR anchor using the default quantizer.
-
-    This is the main entry point for the Think-Quantize-Transmit pattern.
-
-    Example:
-        >>> result = quantize("Please review the authentication code")
-        >>> result.anchor.mnemonic
-        'RequestReview'
-        >>> result.confidence
-        0.67
-    """
-    global _default_quantizer
-    if _default_quantizer is None:
-        _default_quantizer = create_quantizer(prefer_embeddings=False)
-    return _default_quantizer.quantize(thought)
-
-
-def think_quantize_transmit(
-    thought: str,
-    src: str,
-    dst: str,
-    ucr: Optional[UCR] = None,
-) -> str:
-    """
-    The complete Think-Quantize-Transmit flow.
-
-    Takes a natural language thought and produces a wire-ready SLIP message.
-
-    Args:
-        thought: Natural language intent
-        src: Source agent identifier
-        dst: Destination agent identifier
-        ucr: Optional UCR instance
-
-    Returns:
-        Wire-format SLIP message string
-
-    Example:
-        >>> wire = think_quantize_transmit(
-        ...     "I need someone to check this code for security issues",
-        ...     src="developer",
-        ...     dst="reviewer"
-        ... )
-        >>> wire
-        'SLIP v1 developer reviewer RequestReview'
-    """
-    try:
-        from .protocol import slip, fallback as slip_fallback
-    except ImportError:
-        # Inline simple slip format when protocol not available
-        def slip(src, dst, mnemonic, ucr=None):
-            return f"SLIP v1 {src} {dst} {mnemonic}"
-        def slip_fallback(src, dst, payload, ucr=None):
-            return f'SLIP v1 {src} {dst} Fallback "{payload}"'
-
-    result = quantize(thought)
-
-    if result.is_fallback:
-        return slip_fallback(src, dst, thought, ucr)
-    else:
-        return slip(src, dst, result.anchor.mnemonic, ucr=ucr)
-
-
-# ============ Smoke Test ============
-
-if __name__ == "__main__":
-    print("=== Semantic Quantizer Demo ===\n")
-
-    test_thoughts = [
-        "Please review the authentication module for security issues",
-        "I've finished implementing the feature",
-        "What's the current status of the deployment?",
-        "I propose we use Redis for caching instead of Memcached",
-        "Yes, that looks good to me",
-        "There's an error in the payment processing code",
-        "I'm blocked waiting for the API credentials",
-        "Check the auth logs for timing anomalies in the OAuth flow",  # Should fallback
-    ]
-
-    quantizer = KeywordQuantizer()
-
-    for thought in test_thoughts:
-        result = quantizer.quantize(thought)
-        status = "FALLBACK" if result.is_fallback else f"{result.confidence:.2f}"
-        print(f"Thought: {thought[:50]}...")
-        print(f"  → {result.anchor.mnemonic} ({status})")
-        if result.alternatives:
-            alt_str = ", ".join(f"{a.mnemonic}:{s:.2f}" for a, s in result.alternatives[:2])
-            print(f"  Alternatives: {alt_str}")
-        print()
-
-    print(f"Fallback rate: {quantizer.get_fallback_rate():.1%}")
-    print(f"\nUsage stats: {quantizer.get_usage_stats()}")

requirements.txt

@@ -1,5 +1,2 @@
 gradio>=4.0.0
-huggingface_hub
-sentence-transformers>=3.0.0
-numpy
-transformers>=4.40.0
+slipcore>=3.0.0

ucr.py

@@ -1,311 +0,0 @@
-"""
-Universal Concept Reference (UCR) - The Semantic Manifold
-
-The UCR is a quantized semantic coordinate system for agent communication.
-Instead of transmitting embeddings (model-specific, high-dimensional), agents
-communicate via positions in a shared, low-dimensional semantic manifold.
-
-Core concepts:
-- Dimensions: Semantic axes (action, urgency, domain, polarity)
-- Anchors: Named positions in the manifold (common agent intents)
-- Quantization: Map agent thoughts to nearest anchor
-
-Architecture:
-- Core UCR (0x0000-0x7FFF): Standard anchors, immutable per version
-- Extension UCR (0x8000-0xFFFF): Installation-specific, evolvable
-"""
-
-from __future__ import annotations
-from dataclasses import dataclass, field
-from enum import IntEnum
-from typing import Optional
-import json
-from pathlib import Path
-
-
-# ============ Semantic Dimensions ============
-# The axes of our semantic manifold. Kept minimal per design.
-
-class Dimension(IntEnum):
-    """
-    The semantic axes of the UCR manifold.
-    Each dimension represents a fundamental aspect of agent communication.
-    """
-    ACTION = 0      # What type of action: observe, inform, request, propose, evaluate
-    POLARITY = 1    # Direction: initiating vs responding, positive vs negative
-    DOMAIN = 2      # Context: task, plan, observation, evaluation, control
-    URGENCY = 3     # Priority: routine, elevated, critical
-
-
-# Discrete levels per dimension (kept small for token efficiency)
-LEVELS_PER_DIM = 8
-
-
-# ============ UCR Entry (Anchor) ============
-
-@dataclass
-class UCRAnchor:
-    """
-    A named position in the semantic manifold.
-
-    Attributes:
-        index: Unique identifier (0x0000-0xFFFF)
-        mnemonic: Single-token wire representation (e.g., "RequestReview")
-        canonical: Human-readable description
-        coords: Position in the manifold (one value per dimension)
-        is_core: True if part of standard UCR, False if extension
-        centroid: Optional embedding vector for ML-based quantization
-    """
-    index: int
-    mnemonic: str
-    canonical: str
-    coords: tuple[int, ...]  # One int per dimension, each 0 to LEVELS_PER_DIM-1
-    is_core: bool = True
-    centroid: Optional[list[float]] = None  # Embedding vector for ML quantization
-
-    def __post_init__(self):
-        if len(self.coords) != len(Dimension):
-            raise ValueError(f"coords must have {len(Dimension)} values, got {len(self.coords)}")
-        for i, c in enumerate(self.coords):
-            if not (0 <= c < LEVELS_PER_DIM):
-                raise ValueError(f"coord[{i}] must be 0-{LEVELS_PER_DIM-1}, got {c}")
-
-    def to_dict(self) -> dict:
-        result = {
-            "index": self.index,
-            "mnemonic": self.mnemonic,
-            "canonical": self.canonical,
-            "coords": list(self.coords),
-            "is_core": self.is_core,
-        }
-        if self.centroid is not None:
-            result["centroid"] = self.centroid
-        return result
-
-    @classmethod
-    def from_dict(cls, d: dict) -> "UCRAnchor":
-        return cls(
-            index=d["index"],
-            mnemonic=d["mnemonic"],
-            canonical=d["canonical"],
-            coords=tuple(d["coords"]),
-            is_core=d.get("is_core", True),
-            centroid=d.get("centroid"),
-        )
-
-
-# ============ UCR Registry ============
-
-CORE_RANGE_END = 0x8000  # 0x0000-0x7FFF = core, 0x8000-0xFFFF = extensions
-
-
-@dataclass
-class UCR:
-    """
-    The Universal Concept Reference - a semantic manifold for agent communication.
-
-    Contains both core (standard) anchors and extension (local) anchors.
-    Provides lookup by index, mnemonic, and nearest-neighbor by coordinates.
-    """
-    version: str
-    anchors: dict[int, UCRAnchor] = field(default_factory=dict)
-    _mnemonic_index: dict[str, int] = field(default_factory=dict, repr=False)
-
-    def add_anchor(self, anchor: UCRAnchor) -> None:
-        """Add an anchor to the registry."""
-        if anchor.index in self.anchors:
-            raise ValueError(f"Anchor index {anchor.index:#06x} already exists")
-        if anchor.mnemonic in self._mnemonic_index:
-            raise ValueError(f"Anchor mnemonic '{anchor.mnemonic}' already exists")
-
-        # Validate core vs extension range
-        if anchor.is_core and anchor.index >= CORE_RANGE_END:
-            raise ValueError(f"Core anchor index must be < {CORE_RANGE_END:#06x}")
-        if not anchor.is_core and anchor.index < CORE_RANGE_END:
-            raise ValueError(f"Extension anchor index must be >= {CORE_RANGE_END:#06x}")
-
-        self.anchors[anchor.index] = anchor
-        self._mnemonic_index[anchor.mnemonic] = anchor.index
-
-    def get_by_index(self, index: int) -> Optional[UCRAnchor]:
-        """Lookup anchor by numeric index."""
-        return self.anchors.get(index)
-
-    def get_by_mnemonic(self, mnemonic: str) -> Optional[UCRAnchor]:
-        """Lookup anchor by mnemonic string."""
-        idx = self._mnemonic_index.get(mnemonic)
-        return self.anchors.get(idx) if idx is not None else None
-
-    def find_nearest(self, coords: tuple[int, ...]) -> UCRAnchor:
-        """
-        Find the anchor nearest to the given coordinates.
-        Uses Manhattan distance for simplicity and speed.
-        """
-        if not self.anchors:
-            raise ValueError("UCR has no anchors")
-
-        best_anchor = None
-        best_distance = float('inf')
-
-        for anchor in self.anchors.values():
-            distance = sum(abs(a - b) for a, b in zip(anchor.coords, coords))
-            if distance < best_distance:
-                best_distance = distance
-                best_anchor = anchor
-
-        return best_anchor
-
-    def core_anchors(self) -> list[UCRAnchor]:
-        """Return all core (standard) anchors."""
-        return [a for a in self.anchors.values() if a.is_core]
-
-    def extension_anchors(self) -> list[UCRAnchor]:
-        """Return all extension (local) anchors."""
-        return [a for a in self.anchors.values() if not a.is_core]
-
-    def next_extension_index(self) -> int:
-        """Get the next available extension index."""
-        ext_indices = [a.index for a in self.anchors.values() if not a.is_core]
-        if not ext_indices:
-            return CORE_RANGE_END
-        return max(ext_indices) + 1
-
-    def save(self, path: Path) -> None:
-        """Save UCR to JSON file."""
-        data = {
-            "version": self.version,
-            "anchors": [a.to_dict() for a in self.anchors.values()],
-        }
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(data, f, indent=2)
-
-    @classmethod
-    def load(cls, path: Path) -> "UCR":
-        """Load UCR from JSON file."""
-        with open(path, "r", encoding="utf-8") as f:
-            data = json.load(f)
-
-        ucr = cls(version=data["version"])
-        for anchor_data in data["anchors"]:
-            ucr.add_anchor(UCRAnchor.from_dict(anchor_data))
-        return ucr
-
-    def __len__(self) -> int:
-        return len(self.anchors)
-
-
-# ============ Base UCR Factory ============
-
-def create_base_ucr() -> UCR:
-    """
-    Create the base UCR with core anchors for common agent intents.
-
-    Coordinate scheme (4 dimensions, 8 levels each):
-    - ACTION:   0=observe, 1=inform, 2=ask, 3=request, 4=propose, 5=commit, 6=evaluate, 7=meta
-    - POLARITY: 0=negative, 1-3=declining, 4=neutral, 5-6=positive, 7=strong_positive
-    - DOMAIN:   0=task, 1=plan, 2=observation, 3=evaluation, 4=control, 5=resource, 6=error, 7=general
-    - URGENCY:  0=background, 1-2=low, 3-4=normal, 5-6=elevated, 7=critical
-    """
-    ucr = UCR(version="1.0.0")
-
-    # Core coordination anchors
-    # Format: (index, mnemonic, canonical, (action, polarity, domain, urgency))
-
-    core_anchors = [
-        # === Observations (ACTION=0) ===
-        (0x0001, "ObserveState", "Report current system or environment state", (0, 4, 2, 3)),
-        (0x0002, "ObserveChange", "Report a detected change", (0, 4, 2, 4)),
-        (0x0003, "ObserveError", "Report an observed error condition", (0, 2, 6, 6)),
-
-        # === Information (ACTION=1) ===
-        (0x0010, "InformResult", "Share a computed or derived result", (1, 5, 2, 3)),
-        (0x0011, "InformStatus", "Provide status update", (1, 4, 0, 3)),
-        (0x0012, "InformComplete", "Report task completion", (1, 6, 0, 4)),
-        (0x0013, "InformBlocked", "Report being blocked on something", (1, 2, 0, 5)),
-        (0x0014, "InformProgress", "Share progress update", (1, 5, 0, 3)),
-
-        # === Questions (ACTION=2) ===
-        (0x0020, "AskClarify", "Request clarification on requirements", (2, 4, 1, 4)),
-        (0x0021, "AskStatus", "Query current status", (2, 4, 0, 3)),
-        (0x0022, "AskPermission", "Request permission to proceed", (2, 4, 4, 4)),
-        (0x0023, "AskResource", "Query resource availability", (2, 4, 5, 3)),
-
-        # === Requests (ACTION=3) ===
-        (0x0030, "RequestTask", "Request execution of a task", (3, 4, 0, 4)),
-        (0x0031, "RequestPlan", "Request creation of a plan", (3, 4, 1, 4)),
-        (0x0032, "RequestReview", "Request review of work", (3, 4, 3, 3)),
-        (0x0033, "RequestHelp", "Request assistance", (3, 4, 7, 5)),
-        (0x0034, "RequestCancel", "Request cancellation", (3, 1, 4, 5)),
-        (0x0035, "RequestPriority", "Request priority change", (3, 4, 4, 5)),
-        (0x0036, "RequestResource", "Request allocation of resource", (3, 4, 5, 4)),
-
-        # === Proposals (ACTION=4) ===
-        (0x0040, "ProposePlan", "Propose a plan for consideration", (4, 5, 1, 4)),
-        (0x0041, "ProposeChange", "Propose a modification", (4, 5, 0, 4)),
-        (0x0042, "ProposeAlternative", "Propose an alternative approach", (4, 5, 1, 4)),
-        (0x0043, "ProposeRollback", "Propose reverting changes", (4, 3, 4, 5)),
-
-        # === Commitments (ACTION=5) ===
-        (0x0050, "CommitTask", "Commit to performing a task", (5, 6, 0, 4)),
-        (0x0051, "CommitDeadline", "Commit to a deadline", (5, 6, 0, 4)),
-        (0x0052, "CommitResource", "Commit resources", (5, 6, 5, 4)),
-
-        # === Evaluations (ACTION=6) ===
-        (0x0060, "EvalApprove", "Evaluation: approved/positive", (6, 7, 3, 4)),
-        (0x0061, "EvalReject", "Evaluation: rejected/negative", (6, 0, 3, 4)),
-        (0x0062, "EvalNeedsWork", "Evaluation: needs revision", (6, 3, 3, 4)),
-        (0x0063, "EvalComplete", "Evaluation: work is complete", (6, 6, 3, 4)),
-        (0x0064, "EvalBlocked", "Evaluation: blocked by issue", (6, 2, 3, 5)),
-
-        # === Meta/Control (ACTION=7) ===
-        (0x0070, "MetaAck", "Acknowledge receipt", (7, 5, 4, 2)),
-        (0x0071, "MetaSync", "Synchronization ping", (7, 4, 4, 3)),
-        (0x0072, "MetaHandoff", "Hand off responsibility", (7, 4, 4, 4)),
-        (0x0073, "MetaEscalate", "Escalate to higher authority", (7, 3, 4, 6)),
-        (0x0074, "MetaAbort", "Abort current operation", (7, 0, 4, 7)),
-
-        # === Accept/Reject responses ===
-        (0x0080, "Accept", "Accept a proposal or request", (5, 7, 7, 3)),
-        (0x0081, "Reject", "Reject a proposal or request", (5, 0, 7, 3)),
-        (0x0082, "AcceptWithCondition", "Conditional acceptance", (5, 5, 7, 4)),
-        (0x0083, "Defer", "Defer decision", (5, 4, 7, 2)),
-
-        # === Error handling ===
-        (0x0090, "ErrorGeneric", "Generic error occurred", (1, 1, 6, 5)),
-        (0x0091, "ErrorTimeout", "Operation timed out", (1, 1, 6, 5)),
-        (0x0092, "ErrorResource", "Resource unavailable", (1, 1, 6, 5)),
-        (0x0093, "ErrorPermission", "Permission denied", (1, 0, 6, 5)),
-        (0x0094, "ErrorValidation", "Validation failed", (1, 1, 6, 4)),
-
-        # === Fallback ===
-        (0x00FF, "Fallback", "Unquantizable - see payload for natural language", (7, 4, 7, 4)),
-    ]
-
-    for index, mnemonic, canonical, coords in core_anchors:
-        ucr.add_anchor(UCRAnchor(
-            index=index,
-            mnemonic=mnemonic,
-            canonical=canonical,
-            coords=coords,
-            is_core=True,
-        ))
-
-    return ucr
-
-
-# Module-level default UCR instance
-_default_ucr: Optional[UCR] = None
-
-
-def get_default_ucr() -> UCR:
-    """Get or create the default UCR instance."""
-    global _default_ucr
-    if _default_ucr is None:
-        _default_ucr = create_base_ucr()
-    return _default_ucr
-
-
-def set_default_ucr(ucr: UCR) -> None:
-    """Set the default UCR instance."""
-    global _default_ucr
-    _default_ucr = ucr