training/build_4b_v43_patch.py

#!/usr/bin/env python3
"""
build_4b_v43_patch.py — Surgical corpus patch for v43 BFCL failures.

Targets three specific failure modes from 61.9% eval:
  1. ast_parameter (20%)  — correct tool, wrong param keys
  2. multi_turn_chain (12.5%) — NO_TOOL on followup turns
  3. hallucination (40%)  — calls knowledge_search on pure CS questions

Outputs /tmp/4b_v43_patch.jsonl (~550 rows)
Run: python3 build_4b_v43_patch.py
Then: append to /tmp/4b_v43_data/train.jsonl and resume training.
"""
import json, random, sys
from pathlib import Path

random.seed(2027)

SYS_PROMPT = (
    "You are Synalux, a memory-augmented coding and clinical reasoning assistant. "
    "You have access to Prism Memory tools (session_save_ledger, session_load_context, "
    "session_search_memory, session_save_handoff, session_forget_memory, session_health_check, "
    "session_compact_ledger, session_export_memory, session_task_route, session_save_experience, "
    "session_synthesize_edges, session_backfill_links, knowledge_search, knowledge_forget, "
    "knowledge_upvote, knowledge_downvote, knowledge_set_retention, session_save_image, session_view_image) "
    "and 13 multimodal tool modules (image_gen, office, web_scraper, browser, tts, ocr, git, "
    "terminal, deps_scanner, hipaa, data_graph, templates, pdf_parser). "
    "Think step-by-step before answering. When the user references past work, prior decisions, "
    "or stored context, use the appropriate Prism Memory tool. "
    "Format tool calls inside <tool_call>...</tool_call> JSON blocks with fields 'name' and 'arguments'. "
    "If no tool is needed, answer directly in plain text. "
    "ABSTAIN for general programming questions, CS concepts, greetings, and capability questions."
)

def ex(user, tool_name, args):
    """Build a single training example."""
    args_json = json.dumps(args, ensure_ascii=False)
    return {"text": (
        f"<|im_start|>system\n{SYS_PROMPT}<|im_end|>\n"
        f"<|im_start|>user\n{user}<|im_end|>\n"
        f"<|im_start|>assistant\n"
        f"<tool_call>\n"
        f'{{\"name\": \"{tool_name}\", \"arguments\": {args_json}}}\n'
        f"</tool_call>\n"
        f"<|im_end|>"
    )}

def ex_abstain(user, reply):
    """Plain text abstain example."""
    return {"text": (
        f"<|im_start|>system\n{SYS_PROMPT}<|im_end|>\n"
        f"<|im_start|>user\n{user}<|im_end|>\n"
        f"<|im_start|>assistant\n{reply}<|im_end|>"
    )}

def ex_multiturn(user, tool1_name, tool1_args, tool_response, tool2_name, tool2_args):
    """Two-turn chain: first call → tool response → second call."""
    tool1_json = json.dumps(tool1_args, ensure_ascii=False)
    first_response = f'<tool_call>\n{{"name": "{tool1_name}", "arguments": {tool1_json}}}\n</tool_call>'
    if tool2_name == "NO_TOOL":
        second_response = tool2_args.get("reply", "The task has been completed.")
    else:
        tool2_json = json.dumps(tool2_args, ensure_ascii=False)
        second_response = f'<tool_call>\n{{"name": "{tool2_name}", "arguments": {tool2_json}}}\n</tool_call>'
    return {"text": (
        f"<|im_start|>system\n{SYS_PROMPT}<|im_end|>\n"
        f"<|im_start|>user\n{user}<|im_end|>\n"
        f"<|im_start|>assistant\n{first_response}<|im_end|>\n"
        f"<|im_start|>tool\n{tool_response}<|im_end|>\n"
        f"<|im_start|>assistant\n{second_response}<|im_end|>"
    )}

rows = []

# =============================================================================
# FIX 1: AST PARAMETER — exact param keys per tool (50 examples each)
# =============================================================================

# session_export_memory: output_path, format, project
export_prompts = [
    ("Export my memories to {path} in {fmt} format for the {proj} project.",   "{path}", "{fmt}", "{proj}"),
    ("Dump the {proj} project's memory to {path} as {fmt}.",                   "{path}", "{fmt}", "{proj}"),
    ("Back up {proj} memories as {fmt} to {path}.",                             "{path}", "{fmt}", "{proj}"),
    ("Create a {fmt} export of the {proj} memory and save it to {path}.",       "{path}", "{fmt}", "{proj}"),
    ("Export session memory for {proj} to {path}, format: {fmt}.",              "{path}", "{fmt}", "{proj}"),
    ("Save a {fmt} copy of all {proj} knowledge to {path}.",                    "{path}", "{fmt}", "{proj}"),
    ("Generate a {fmt} export from the {proj} project memory at {path}.",       "{path}", "{fmt}", "{proj}"),
    ("I need the {proj} project memory exported in {fmt} to {path}.",           "{path}", "{fmt}", "{proj}"),
    ("Write the {proj} memory out to {path} as a {fmt} file.",                  "{path}", "{fmt}", "{proj}"),
    ("Archive {proj} memories to {path} using {fmt} format.",                   "{path}", "{fmt}", "{proj}"),
]
paths = ["/tmp/backup", "/tmp/export", "/var/backup/prism", "~/exports", "/data/memory-backup"]
fmts  = ["markdown", "json", "csv", "html", "yaml"]
projs = ["billing", "portal", "analytics", "prism-training", "auth-service", "dashboard", "api-gateway"]
for i in range(50):
    tmpl = export_prompts[i % len(export_prompts)]
    p, f, proj = paths[i%len(paths)], fmts[i%len(fmts)], projs[i%len(projs)]
    user = tmpl[0].format(path=p, fmt=f, proj=proj)
    rows.append(ex(user, "session_export_memory", {"output_path": p, "format": f, "project": proj}))

# knowledge_set_retention: project, ttl_days
retention_prompts = [
    "Set a {days}-day retention policy for the {proj} project's knowledge.",
    "Configure knowledge retention to {days} days for {proj}.",
    "Keep {proj} knowledge for {days} days, then expire it.",
    "Set {proj} memory retention to {days} days.",
    "Apply a {days}-day TTL to the {proj} project's knowledge base.",
    "Retain {proj} knowledge entries for {days} days.",
    "Set TTL of {days} days on {proj} knowledge.",
    "Expire {proj} project knowledge after {days} days.",
    "Knowledge retention for {proj}: {days} days.",
    "Configure {days}-day retention on {proj} project knowledge.",
]
days_opts = [7, 14, 30, 60, 90, 180, 365]
for i in range(50):
    proj = projs[i % len(projs)]
    days = days_opts[i % len(days_opts)]
    user = retention_prompts[i % len(retention_prompts)].format(days=days, proj=proj)
    rows.append(ex(user, "knowledge_set_retention", {"project": proj, "ttl_days": days}))

# session_save_ledger: project, conversation_id, summary
ledger_prompts = [
    ("Save a ledger entry: project is '{proj}', conversation is '{conv}', summary is '{summ}'.",
     "{proj}", "{conv}", "{summ}"),
    ("Log session: project={proj}, conv_id={conv}. Summary: {summ}",
     "{proj}", "{conv}", "{summ}"),
    ("Record conversation {conv} for {proj}: {summ}",
     "{proj}", "{conv}", "{summ}"),
    ("Save ledger — project: {proj}, conversation_id: {conv}, notes: {summ}",
     "{proj}", "{conv}", "{summ}"),
    ("Ledger save for {proj} (conversation {conv}): {summ}",
     "{proj}", "{conv}", "{summ}"),
]
convs = ["conv-2024-001", "conv-001", "session-42", "conv-2025-007", "thread-alpha"]
summs = [
    "Deployed v2.0 to production with zero downtime",
    "Fixed auth bug in OAuth2 flow",
    "Completed DB migration to PostgreSQL 16",
    "Refactored API gateway routing layer",
    "Added streaming support to TTS module",
]
for i in range(50):
    tmpl = ledger_prompts[i % len(ledger_prompts)]
    proj, conv, summ = projs[i%len(projs)], convs[i%len(convs)], summs[i%len(summs)]
    user = tmpl[0].format(proj=proj, conv=conv, summ=summ)
    rows.append(ex(user, "session_save_ledger", {"project": proj, "conversation_id": conv, "summary": summ}))

# session_save_experience: project, event_type, content
experience_prompts = [
    ("Record a {etype} experience for the {proj} project: {content}",
     "{proj}", "{etype}", "{content}"),
    ("Save a {etype} to {proj} memory: {content}",
     "{proj}", "{etype}", "{content}"),
    ("Log {etype} for {proj}: {content}",
     "{proj}", "{etype}", "{content}"),
    ("Save experience ({etype}) in {proj}: {content}",
     "{proj}", "{etype}", "{content}"),
    ("Add a {etype} entry to {proj} experiences: {content}",
     "{proj}", "{etype}", "{content}"),
]
etypes = ["correction", "insight", "milestone", "observation", "learning"]
contents = [
    "I tried using batch inserts but should have used streaming writes instead.",
    "Redis cache invalidation needs explicit TTL, not just key deletion.",
    "Completed OAuth2 integration ahead of schedule.",
    "Noticed memory leak in the WebSocket handler.",
    "JWT tokens must be validated server-side, never trust client claims.",
]
for i in range(50):
    tmpl = experience_prompts[i % len(experience_prompts)]
    proj, etype, content = projs[i%len(projs)], etypes[i%len(etypes)], contents[i%len(contents)]
    user = tmpl[0].format(proj=proj, etype=etype, content=content)
    rows.append(ex(user, "session_save_experience", {"project": proj, "event_type": etype, "content": content}))

# session_save_image: project, image_path, description
image_prompts = [
    "Save an image at {path} for the {proj} project with description '{desc}'.",
    "Store screenshot {path} under {proj}: '{desc}'",
    "Save {path} to {proj} memory, description: '{desc}'",
    "Record image {path} for {proj} — '{desc}'",
    "Add image {path} to {proj} with label '{desc}'",
    "Attach {path} to the {proj} project ('{desc}')",
    "Save screenshot from {path} to {proj} memory. Caption: '{desc}'",
    "Store {path} in {proj} project images, note: '{desc}'",
    "Image {path} for project {proj}: '{desc}'",
    "Log screenshot {path} under {proj}: '{desc}'",
]
img_paths = [
    "/tmp/screenshot.png", "/tmp/ui.png", "/tmp/diagram.png",
    "/tmp/error_log.png", "/tmp/dashboard.png",
]
descs = [
    "Login page redesign mockup",
    "API latency spike screenshot",
    "Architecture diagram v2",
    "Error modal in dark mode",
    "Production dashboard showing 99.9% uptime",
]
for i in range(50):
    proj = projs[i % len(projs)]
    path = img_paths[i % len(img_paths)]
    desc = descs[i % len(descs)]
    user = image_prompts[i % len(image_prompts)].format(path=path, proj=proj, desc=desc)
    rows.append(ex(user, "session_save_image", {"project": proj, "image_path": path, "description": desc}))

print(f"After FIX 1 (param precision): {len(rows)} rows")

# =============================================================================
# FIX 2: MULTI-TURN CHAIN (100 examples)
# =============================================================================

# Pattern: load → search followup
load_search_chains = [
    ("Load context for {proj}, then search for {topic}.",
     "session_load_context", lambda p,t: {"project": p},
     '{{"project": "{proj}", "open_todos": ["fix deploy"], "last_summary": "Worked on {proj}"}}',
     "session_search_memory", lambda p,t: {"query": t}),
    ("Get context for {proj} and then find info about {topic}.",
     "session_load_context", lambda p,t: {"project": p},
     '{{"project": "{proj}", "last_summary": "Working on {proj}", "branch": "main"}}',
     "session_search_memory", lambda p,t: {"query": t}),
    ("Load the {proj} project context, then look up {topic}.",
     "session_load_context", lambda p,t: {"project": p},
     '{{"project": "{proj}", "last_summary": "Debugging {proj}"}}',
     "session_search_memory", lambda p,t: {"query": t}),
]
topics = ["recent deployment issues", "auth bugs", "database migrations", "performance regressions", "API changes"]
for i in range(20):
    ch = load_search_chains[i % len(load_search_chains)]
    proj = projs[i % len(projs)]
    topic = topics[i % len(topics)]
    user = ch[0].format(proj=proj, topic=topic)
    t1_args = ch[2](proj, topic)
    resp = ch[3].format(proj=proj, topic=topic)
    t2_args = ch[5](proj, topic)
    rows.append(ex_multiturn(user, ch[1], t1_args, resp, ch[4], t2_args))

# Pattern: search → save handoff
search_handoff_chains = [
    ("Search for what we decided about {topic}, then save a handoff note about it.",
     "session_search_memory", lambda t: {"query": t},
     '{{"results": [{{"summary": "Decided to use {topic} with TTL caching", "importance": 4}}]}}',
     "session_save_handoff", lambda t: {"summary": f"Decision on {t}: use caching layer"}),
    ("Look up {topic} in memory, then create a handoff for the next session.",
     "session_search_memory", lambda t: {"query": t},
     '{{"results": [{{"summary": "{topic} implementation complete", "importance": 5}}]}}',
     "session_save_handoff", lambda t: {"summary": f"{t} implementation notes"}),
    ("Find what we know about {topic}, then hand it off.",
     "session_search_memory", lambda t: {"query": t},
     '{{"results": [{{"summary": "Notes on {topic}", "importance": 3}}]}}',
     "session_save_handoff", lambda t: {"summary": f"Handoff: {t}"}),
]
for i in range(20):
    ch = search_handoff_chains[i % len(search_handoff_chains)]
    topic = topics[i % len(topics)]
    user = ch[0].format(topic=topic)
    t1_args = ch[2](topic)
    resp = ch[3].format(topic=topic)
    t2_args = ch[5](topic)
    rows.append(ex_multiturn(user, ch[1], t1_args, resp, ch[4], t2_args))

# Pattern: health_check → compact
health_compact = [
    ("Run a health check on the memory system. If there are issues, compact the old entries.",
     "session_health_check", {},
     '{"status": "issues_found", "missing_embeddings": 12, "stale_rollups": 3}',
     "session_compact_ledger", {}),
    ("Check memory system health, then compact if needed.",
     "session_health_check", {},
     '{"status": "degraded", "stale_entries": 42, "recommendation": "compact"}',
     "session_compact_ledger", {}),
    ("Health check the prism memory — compact if stale entries found.",
     "session_health_check", {},
     '{"status": "issues_found", "stale_rollups": 8}',
     "session_compact_ledger", {}),
    ("Diagnose memory health, compact if there are stale entries.",
     "session_health_check", {},
     '{"status": "degraded", "missing_embeddings": 5}',
     "session_compact_ledger", {}),
    ("Memory health check, then compact if degraded.",
     "session_health_check", {},
     '{"status": "issues_found", "stale_rollups": 15}',
     "session_compact_ledger", {}),
]
for i in range(10):
    ch = health_compact[i % len(health_compact)]
    rows.append(ex_multiturn(ch[0], ch[1], ch[2], ch[3], ch[4], ch[5]))

# Pattern: load → save ledger
load_ledger = [
    ("Load context for {proj} and then log that we successfully deployed v{v}.",
     "session_load_context", lambda p,v: {"project": p},
     '{{"project": "{proj}", "last_summary": "Pre-deploy state", "open_todos": []}}',
     "session_save_ledger", lambda p,v: {"project": p, "summary": f"Deployed v{v} successfully"}),
    ("Get {proj} context, then save a ledger entry for the v{v} release.",
     "session_load_context", lambda p,v: {"project": p},
     '{{"project": "{proj}", "last_summary": "Release prep for v{v}"}}',
     "session_save_ledger", lambda p,v: {"project": p, "summary": f"v{v} release complete"}),
]
versions = ["2.0", "3.1", "1.5", "4.0", "2.3"]
for i in range(10):
    ch = load_ledger[i % len(load_ledger)]
    proj = projs[i % len(projs)]
    v = versions[i % len(versions)]
    user = ch[0].format(proj=proj, v=v)
    t1_args = ch[2](proj, v)
    resp = ch[3].format(proj=proj, v=v)
    t2_args = ch[5](proj, v)
    rows.append(ex_multiturn(user, ch[1], t1_args, resp, ch[4], t2_args))

# Pattern: knowledge_search → upvote
search_upvote = [
    ("Search knowledge for {topic}, then upvote the best result.",
     "knowledge_search", lambda t: {"query": t},
     '{{"results": [{{"id": "ki-{n}", "summary": "{topic} best practice", "importance": 5}}]}}',
     "knowledge_upvote", lambda t, n: {"knowledge_id": f"ki-{n}"}),
    ("Find knowledge about {topic} and upvote the top entry.",
     "knowledge_search", lambda t: {"query": t},
     '{{"results": [{{"id": "ki-{n}", "summary": "{topic} guide", "importance": 4}}]}}',
     "knowledge_upvote", lambda t, n: {"knowledge_id": f"ki-{n}"}),
]
ks_topics = ["retry strategies", "caching patterns", "auth best practices", "DB indexing", "rate limiting"]
for i in range(10):
    ch = search_upvote[i % len(search_upvote)]
    topic = ks_topics[i % len(ks_topics)]
    n = 42 + i
    user = ch[0].format(topic=topic)
    t1_args = ch[2](topic)
    resp = ch[3].format(topic=topic, n=n)
    t2_args = ch[5](topic, n)
    rows.append(ex_multiturn(user, ch[1], t1_args, resp, "knowledge_upvote", t2_args))

# Pattern: export → set retention
export_retention = [
    ("Export the {proj} project memory to {path}, then set a {days}-day retention policy.",
     "session_export_memory", lambda p, path, d: {"output_path": path, "project": p},
     '{{"status": "exported", "file": "{path}/prism-export-{proj}.json", "entries": 142}}',
     "knowledge_set_retention", lambda p, path, d: {"project": p, "ttl_days": d}),
    ("Backup {proj} memory to {path} and then apply {days}-day TTL.",
     "session_export_memory", lambda p, path, d: {"output_path": path, "project": p},
     '{{"status": "exported", "entries": 87}}',
     "knowledge_set_retention", lambda p, path, d: {"project": p, "ttl_days": d}),
]
exp_paths = ["/tmp/backup", "/var/backups", "~/prism-exports"]
for i in range(10):
    ch = export_retention[i % len(export_retention)]
    proj = projs[i % len(projs)]
    path = exp_paths[i % len(exp_paths)]
    days = days_opts[i % len(days_opts)]
    user = ch[0].format(proj=proj, path=path, days=days)
    t1_args = ch[2](proj, path, days)
    resp = ch[3].format(proj=proj, path=path)
    t2_args = ch[5](proj, path, days)
    rows.append(ex_multiturn(user, ch[1], t1_args, resp, ch[4], t2_args))

# Pattern: save_ledger → save_handoff
ledger_handoff = [
    ("Record this session: {summ}. Then save the handoff state.",
     "session_save_ledger", lambda s: {"summary": s},
     '{"status": "saved", "id": "ledger-2024-99"}',
     "session_save_handoff", lambda s: {"summary": s}),
    ("Log that {summ}, then create a handoff.",
     "session_save_ledger", lambda s: {"summary": s},
     '{"status": "saved", "id": "ledger-007"}',
     "session_save_handoff", lambda s: {"summary": s}),
]
session_summs = [
    "we migrated the auth module to OAuth2",
    "we completed the v3 deploy",
    "we fixed the memory leak in the websocket handler",
    "we refactored the database connection pool",
    "we added retry logic to the payment processor",
]
for i in range(10):
    ch = ledger_handoff[i % len(ledger_handoff)]
    summ = session_summs[i % len(session_summs)]
    user = ch[0].format(summ=summ)
    t1_args = ch[2](summ)
    t2_args = ch[5](summ)
    rows.append(ex_multiturn(user, ch[1], t1_args, ch[3], ch[4], t2_args))

# Pattern: task_route → NO_TOOL (act on routing result)
task_route_no_tool = [
    ("Should the local agent handle this TypeScript refactor? If cloud, just tell me.",
     "session_task_route", {"task": "TypeScript refactor"},
     '{"target": "host", "confidence": 0.92, "reason": "Complex refactor needs cloud model"}',
     "NO_TOOL", {"reply": "The cloud model should handle this. Routing to host with 0.92 confidence — complex TypeScript refactor requires the larger model."}),
    ("Route this Python debugging task — local or cloud?",
     "session_task_route", {"task": "Python debugging"},
     '{"target": "local", "confidence": 0.85, "reason": "Simple debug fits local model"}',
     "NO_TOOL", {"reply": "Local agent can handle this. Confidence 0.85 — simple Python debugging fits within the local model's capability."}),
    ("Task routing check: should I run this code review locally?",
     "session_task_route", {"task": "code review"},
     '{"target": "host", "confidence": 0.78, "reason": "Multi-file review needs cloud context"}',
     "NO_TOOL", {"reply": "Use the cloud model for this code review. Confidence 0.78 — multi-file review requires broader context."}),
]
for i in range(10):
    ch = task_route_no_tool[i % len(task_route_no_tool)]
    rows.append(ex_multiturn(ch[0], ch[1], ch[2], ch[3], ch[4], ch[5]))

print(f"After FIX 2 (multi-turn): {len(rows)} rows")

# =============================================================================
# FIX 3: HALLUCINATION PREVENTION — abstain on CS/programming questions
# =============================================================================

cs_abstain = [
    # Context managers
    ("How do I implement a context manager in Python using __enter__ and __exit__?",
     "Context managers in Python use the `__enter__` and `__exit__` dunder methods. Implement `__enter__` to set up the resource (return `self`), and `__exit__(self, exc_type, exc_val, exc_tb)` to release it. For simple cases, use `@contextlib.contextmanager` with a generator."),
    ("Explain Python context managers and the `with` statement.",
     "The `with` statement calls `__enter__` on entry and guarantees `__exit__` on exit, even on exceptions. This is the standard pattern for resource management (files, locks, DB connections)."),
    # LSTM
    ("Explain the forget gate in an LSTM neural network.",
     "The forget gate in an LSTM uses a sigmoid activation (output 0–1) applied to the concatenation of the previous hidden state and current input. Values near 0 forget the cell state; values near 1 retain it. This controls long-term memory retention."),
    ("What does the cell state do in an LSTM?",
     "The cell state is the long-term memory of an LSTM. It flows through time with only linear interactions (additions and elementwise multiplications from the gates), which helps gradients flow without vanishing."),
    # Express/session
    ("How does session management work in Express.js with passport?",
     "Passport.js serializes the user on login (via `serializeUser`) and deserializes on each request (via `deserializeUser`). Sessions are stored via `express-session` middleware. The session ID is stored in a cookie; the session data lives server-side."),
    # Knowledge distillation
    ("What's the difference between knowledge distillation and model pruning?",
     "Knowledge distillation trains a smaller student model to mimic a larger teacher's output distribution (soft labels). Pruning removes weights or neurons from the model itself. Distillation transfers learned behavior; pruning reduces model size by removing parameters."),
    # Redux
    ("How do I save state in a Redux store?",
     "Redux state is saved by dispatching actions that reducers handle. For persistence across sessions, use `redux-persist` which serializes the store to localStorage/AsyncStorage. In the reducer, handle your action to return a new state object (never mutate directly)."),
    # Memory-mapped files
    ("Explain memory-mapped files and how they improve I/O performance.",
     "Memory-mapped files map file contents directly into the process's virtual address space. The OS handles paging on demand, avoiding explicit `read`/`write` syscalls. This reduces copying and can speed up large-file access significantly on repeated reads."),
    # Garbage collection
    ("How does the garbage collector handle circular references in Python?",
     "CPython's reference counting alone can't handle circular references. The cyclic garbage collector (gc module) uses a generational algorithm that detects reference cycles. Objects in cycles with no external references are collected. You can trigger it with `gc.collect()`."),
    # Kubernetes health checks
    ("What is a load balancer health check in Kubernetes?",
     "In Kubernetes, liveness probes check if a container is alive (restart on failure), and readiness probes check if it can serve traffic (remove from Service endpoints if failing). Load balancers use these probes to route traffic only to healthy pods."),
    # Exponential backoff
    ("How do I implement exponential backoff with jitter for API retries?",
     "Exponential backoff: `wait = base * 2^attempt`. Add jitter by randomizing: `wait = random.uniform(0, base * 2^attempt)`. Cap with a max delay. This prevents thundering herd when many clients retry simultaneously after a failure."),
    # Elasticsearch vs Solr
    ("Compare Elasticsearch and Solr for full-text search.",
     "Both are built on Lucene. Elasticsearch excels at distributed deployments, real-time indexing, and REST-first API design. Solr is more mature, has stronger XML/JSON config, and better faceting. Elasticsearch is the dominant choice for new projects due to its ecosystem (Elastic Stack)."),
    # Additional CS concepts
    ("What is the CAP theorem?",
     "CAP theorem states a distributed system can guarantee at most two of three properties: Consistency (every read returns the latest write), Availability (every request gets a response), Partition tolerance (system continues despite network splits). In practice, partition tolerance is required, so you choose CP or AP."),
    ("Explain binary search trees and their time complexity.",
     "A BST maintains the invariant: left child < node < right child. Search, insert, and delete are O(log n) on average for balanced trees, O(n) worst case (skewed tree). Self-balancing variants (AVL, Red-Black) guarantee O(log n)."),
    ("What is virtual DOM reconciliation in React?",
     "React maintains an in-memory virtual DOM. On state change, it diffs the new vDOM against the previous one (reconciliation), then applies only the minimal set of real DOM mutations. This avoids costly full DOM re-renders."),
    ("How does dependency injection work?",
     "Dependency injection passes a class its dependencies rather than having it construct them internally. This decouples components and makes testing easier. Common patterns: constructor injection, property injection, method injection. Frameworks like Spring/Angular handle DI via IoC containers."),
    ("What's the difference between TCP and UDP?",
     "TCP is connection-oriented: establishes a 3-way handshake, guarantees delivery and order, has flow control. UDP is connectionless: no handshake, no delivery guarantee, lower overhead. Use TCP for reliability (HTTP, SSH), UDP for latency-sensitive (DNS, video streaming, games)."),
    ("What is time complexity of quicksort?",
     "Quicksort average case is O(n log n). Worst case is O(n²) when the pivot is always the min or max (e.g., sorted input with naive pivot). Randomized pivot selection or median-of-three makes worst case practically O(n log n)."),
    # Capability questions
    ("What can you do?",
     "I'm Synalux, a memory-augmented coding and clinical reasoning assistant. I can help you retrieve and save project context, search your memory, route tasks to appropriate agents, manage knowledge retention, and use multimodal tools for images, TTS, web scraping, and more. What would you like to work on?"),
    ("What tools do you have access to?",
     "I have access to Prism Memory tools (session_load_context, session_save_ledger, session_search_memory, knowledge_search, and others) and 13 multimodal modules including TTS, OCR, web scraping, browser, git, terminal, image generation, PDF parsing, and more."),
    ("What are you capable of?",
     "I can load and save project memory, search past decisions, route tasks to local or cloud agents, manage knowledge with retention policies, generate images, process documents, run TTS, scrape the web, and much more. Ask me what you need."),
    ("Can you write code for me?",
     "Yes, I can help write code. I also have access to a git tool for committing changes, a terminal for running commands, a deps scanner for dependency audits, and code analysis capabilities. What are you building?"),
    # Safety / general questions that should not trigger tools
    ("Who created you?",
     "I'm Synalux, built by the Synalux team to assist with coding, clinical workflows, and memory-augmented reasoning."),
    ("What's today's date?",
     "I don't have access to a real-time clock. Please check your system clock for the current date."),
    ("Tell me a joke.",
     "Why do programmers prefer dark mode? Because light attracts bugs!"),
]

for user, reply in cs_abstain:
    rows.append(ex_abstain(user, reply))

print(f"After FIX 3 (hallucination abstain): {len(rows)} rows")

# =============================================================================
# FIX 4: Disambiguation edge cases
# =============================================================================

# session_save_ledger vs session_save_handoff disambiguation
disambig_ledger = [
    "Save the current session ledger for the {proj} project.",
    "Log this session to the {proj} ledger.",
    "Save a ledger entry for {proj}.",
    "Record session progress in the {proj} ledger.",
    "Write the session log for {proj}.",
]
disambig_handoff = [
    "Save a handoff note for the {proj} project.",
    "Create a handoff for {proj} for the next agent.",
    "Hand off the {proj} session state.",
    "Save the handoff state for {proj}.",
    "Store a handoff summary for {proj}.",
]
for i in range(20):
    proj = projs[i % len(projs)]
    rows.append(ex(disambig_ledger[i % len(disambig_ledger)].format(proj=proj),
                   "session_save_ledger", {"project": proj}))
    rows.append(ex(disambig_handoff[i % len(disambig_handoff)].format(proj=proj),
                   "session_save_handoff", {"project": proj}))

print(f"After FIX 4 (disambiguation): {len(rows)} rows")

# =============================================================================
# SHUFFLE AND WRITE
# =============================================================================
random.shuffle(rows)
out = Path("/tmp/4b_v43_patch.jsonl")
out.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + "\n")
print(f"\n✅ Wrote {len(rows)} patch rows to {out}")
print(f"   Breakdown: ~250 param-precise, ~100 multi-turn, ~50 abstain, ~40 disambig")