Upload Base/Datasets/rag_mcp_sft/build_rag_mcp_sft_dataset.py with huggingface_hub

Base/Datasets/rag_mcp_sft/build_rag_mcp_sft_dataset.py

@@ -0,0 +1,1123 @@
+import argparse
+import hashlib
+import json
+import random
+import re
+from pathlib import Path
+
+from transformers import AutoTokenizer
+
+
+ROOT = Path(__file__).resolve().parents[3]
+DATASET_DIR = Path(__file__).resolve().parent
+TOKENIZER_DIR = ROOT / "Base" / "checkpoints" / "EleutherAI" / "pythia-160m"
+RETRIEVED_ON = "2026-04-03"
+
+
+SOURCE_MANIFEST = [
+    {
+        "topic": "mcp",
+        "url": "https://modelcontextprotocol.io/introduction",
+        "title": "What is the Model Context Protocol (MCP)?",
+        "notes": [
+            "MCP is an open standard for connecting AI applications to tools, data sources, and workflows.",
+            "The ecosystem spans hosts like IDEs and assistants plus many local and remote servers.",
+            "The main value is interoperable context and actions across tools without bespoke integrations.",
+        ],
+    },
+    {
+        "topic": "mcp",
+        "url": "https://modelcontextprotocol.io/docs/concepts/architecture",
+        "title": "MCP Architecture Overview",
+        "notes": [
+            "MCP uses a client-server architecture with hosts, clients, and servers.",
+            "The data layer is JSON-RPC 2.0 based and includes lifecycle, tools, resources, prompts, and notifications.",
+            "The transport layer covers stdio for local use and streamable HTTP for remote use.",
+        ],
+    },
+    {
+        "topic": "rag",
+        "url": "https://www.pinecone.io/learn/retrieval-augmented-generation/",
+        "title": "Retrieval-Augmented Generation (RAG)",
+        "notes": [
+            "RAG addresses stale knowledge, weak domain depth, missing private data, and hallucination risk.",
+            "Core stages are ingestion, retrieval, augmentation, and generation.",
+            "Good RAG systems emphasize chunking, retrieval quality, reranking, grounding, citations, and evaluation.",
+        ],
+    },
+]
+
+
+KNOWLEDGE_CARDS = [
+    {
+        "id": "rag_definition",
+        "topic": "RAG",
+        "title": "Definition of RAG",
+        "summary": "Retrieval-augmented generation improves model answers by retrieving external evidence before response generation.",
+        "key_points": [
+            "RAG grounds the model with authoritative context instead of relying only on frozen weights.",
+            "It is useful when answers need current, proprietary, or domain-specific information.",
+            "The goal is higher accuracy, relevance, and trustworthiness.",
+        ],
+        "pitfalls": [
+            "Using weak sources still produces weak answers.",
+            "Large context dumps can increase cost without improving quality.",
+            "A model can still hallucinate if retrieval is poor or ignored.",
+        ],
+        "best_practices": [
+            "Prefer authoritative sources with clear provenance.",
+            "Ask the model to stay grounded in retrieved facts.",
+            "Return citations or source identifiers when possible.",
+        ],
+        "use_cases": [
+            "support copilots",
+            "enterprise knowledge assistants",
+            "research workflows",
+        ],
+    },
+    {
+        "id": "rag_limitations",
+        "topic": "RAG",
+        "title": "Why base models need retrieval",
+        "summary": "Standalone language models are limited by knowledge cutoffs, incomplete domain depth, and lack of private data access.",
+        "key_points": [
+            "A pretrained model cannot know events or documents added after training.",
+            "Specialized domains often require evidence that was sparse or absent in pretraining.",
+            "Private company knowledge should not be expected to appear in a public base model.",
+        ],
+        "pitfalls": [
+            "Users may trust fluent but unsupported answers.",
+            "Sampling randomness can surface plausible but incorrect claims.",
+            "Without citations, error diagnosis is harder.",
+        ],
+        "best_practices": [
+            "Add retrieval when freshness or policy accuracy matters.",
+            "Use evaluations with known-good answers.",
+            "Treat fluent output as unverified until grounded.",
+        ],
+        "use_cases": [
+            "policy assistants",
+            "product documentation bots",
+            "current-events question answering",
+        ],
+    },
+    {
+        "id": "rag_ingestion",
+        "topic": "RAG",
+        "title": "Ingestion stage",
+        "summary": "The ingestion stage prepares authoritative documents so retrieval can find them efficiently later.",
+        "key_points": [
+            "Ingestion usually includes cleaning, chunking, embedding, and indexing.",
+            "Source metadata such as title, owner, and timestamp should be preserved.",
+            "Fresh content should trigger incremental updates rather than full rebuilds when possible.",
+        ],
+        "pitfalls": [
+            "Dropping metadata weakens traceability.",
+            "Poor chunking reduces retrieval quality.",
+            "Stale indexes silently degrade answers.",
+        ],
+        "best_practices": [
+            "Keep a documented ingestion contract.",
+            "Track source versioning and refresh policy.",
+            "Store enough metadata for filtering and citations.",
+        ],
+        "use_cases": [
+            "document onboarding pipelines",
+            "knowledge base refresh jobs",
+            "multi-tenant content indexing",
+        ],
+    },
+    {
+        "id": "rag_chunking",
+        "topic": "RAG",
+        "title": "Chunking strategy",
+        "summary": "Chunking balances retrieval precision against context completeness by dividing documents into useful units.",
+        "key_points": [
+            "Small chunks improve focus but may lose context.",
+            "Large chunks preserve context but can dilute retrieval precision.",
+            "Overlap can preserve continuity across boundaries.",
+        ],
+        "pitfalls": [
+            "Fixed-size chunks can split critical facts mid-thought.",
+            "Too much overlap increases storage and duplicate retrieval.",
+            "Ignoring document structure wastes natural section boundaries.",
+        ],
+        "best_practices": [
+            "Align chunking with document structure when possible.",
+            "Test chunk size against real user queries.",
+            "Avoid chunk sizes that exceed downstream prompt budgets.",
+        ],
+        "use_cases": [
+            "API docs",
+            "wiki articles",
+            "policy manuals",
+        ],
+    },
+    {
+        "id": "rag_embeddings",
+        "topic": "RAG",
+        "title": "Embeddings and vector search",
+        "summary": "Embeddings turn text into vectors so semantically related passages can be retrieved even when wording differs.",
+        "key_points": [
+            "Dense embeddings capture semantic similarity.",
+            "The embedding model should match the language and domain of the corpus.",
+            "Vector search is often paired with metadata filtering.",
+        ],
+        "pitfalls": [
+            "Bad embeddings cause systematic retrieval misses.",
+            "Ignoring metadata can surface irrelevant tenants or time periods.",
+            "Switching embedding models without reindexing corrupts similarity assumptions.",
+        ],
+        "best_practices": [
+            "Version embedding models and indexes together.",
+            "Benchmark retrieval before production rollout.",
+            "Use metadata filters for scope control.",
+        ],
+        "use_cases": [
+            "semantic search",
+            "similar policy lookup",
+            "knowledge retrieval for assistants",
+        ],
+    },
+    {
+        "id": "rag_hybrid_retrieval",
+        "topic": "RAG",
+        "title": "Hybrid retrieval and reranking",
+        "summary": "Hybrid retrieval combines semantic and lexical signals, then reranking sharpens the final evidence set.",
+        "key_points": [
+            "Dense retrieval helps when users paraphrase concepts.",
+            "Lexical or sparse retrieval helps with exact codes, product names, and acronyms.",
+            "Reranking can improve final relevance after broad recall.",
+        ],
+        "pitfalls": [
+            "Dense-only search may miss literal identifiers.",
+            "Keyword-only search may miss semantically related wording.",
+            "Skipping rerank can leave the top context noisy.",
+        ],
+        "best_practices": [
+            "Treat retrieval as a pipeline, not a single score.",
+            "Log both recall and final answer quality.",
+            "Tune retrieval for the actual query distribution.",
+        ],
+        "use_cases": [
+            "incident knowledge search",
+            "developer documentation search",
+            "internal ticket assistants",
+        ],
+    },
+    {
+        "id": "rag_augmentation",
+        "topic": "RAG",
+        "title": "Augmented prompting",
+        "summary": "Augmentation combines the user question with retrieved context in a prompt that tells the model how to use evidence.",
+        "key_points": [
+            "The prompt should distinguish the question from the evidence block.",
+            "The model should be instructed to say when context is insufficient.",
+            "Answer style, citation format, and refusal behavior should be explicit.",
+        ],
+        "pitfalls": [
+            "Prompt stuffing wastes tokens and distracts the model.",
+            "Unclear grounding rules encourage unsupported guesses.",
+            "Mixing contradictory passages without guidance confuses answer synthesis.",
+        ],
+        "best_practices": [
+            "Use a consistent prompt contract.",
+            "Cap context to the most relevant evidence.",
+            "Require the model to state uncertainty when evidence is missing.",
+        ],
+        "use_cases": [
+            "grounded question answering",
+            "support responses with references",
+            "document-based summarization",
+        ],
+    },
+    {
+        "id": "rag_generation",
+        "topic": "RAG",
+        "title": "Generation and grounded answers",
+        "summary": "The generation stage turns the augmented prompt into an answer that should remain faithful to retrieved evidence.",
+        "key_points": [
+            "Generation should prioritize grounded facts over fluent speculation.",
+            "Well-designed RAG answers can include citations or source mentions.",
+            "The system should avoid pretending the evidence said more than it did.",
+        ],
+        "pitfalls": [
+            "Ungrounded synthesis reintroduces hallucinations.",
+            "Verbose answers can hide unsupported claims.",
+            "Dropping citation signals weakens user trust.",
+        ],
+        "best_practices": [
+            "Keep answers scoped to retrieved evidence.",
+            "Surface missing information plainly.",
+            "Design output formats that make verification easy.",
+        ],
+        "use_cases": [
+            "answer generation with citations",
+            "enterprise assistant responses",
+            "grounded summaries",
+        ],
+    },
+    {
+        "id": "rag_evaluation",
+        "topic": "RAG",
+        "title": "RAG evaluation",
+        "summary": "RAG quality should be measured with representative queries, known-good answers, and retrieval diagnostics.",
+        "key_points": [
+            "Evaluation should cover both retrieval quality and answer quality.",
+            "A fixed validation set helps track regressions over time.",
+            "Ground truth is essential for knowing whether improvements are real.",
+        ],
+        "pitfalls": [
+            "Only tracking latency or token cost misses quality failures.",
+            "Synthetic benchmarks alone may not reflect production behavior.",
+            "Ignoring bad-query cases hides operational risk.",
+        ],
+        "best_practices": [
+            "Maintain a curated benchmark set.",
+            "Review failures by retrieval stage and answer stage.",
+            "Track citation accuracy and insufficiency handling.",
+        ],
+        "use_cases": [
+            "release validation",
+            "retrieval tuning",
+            "regression testing",
+        ],
+    },
+    {
+        "id": "rag_agentic",
+        "topic": "RAG",
+        "title": "Agentic RAG",
+        "summary": "Agentic RAG extends basic retrieval by letting an agent plan, query, validate, and refine evidence over multiple steps.",
+        "key_points": [
+            "An agent can rewrite queries, choose tools, and validate retrieved context.",
+            "Iterative retrieval helps when a single search step is not enough.",
+            "The system can combine retrieval with reasoning and execution tools.",
+        ],
+        "pitfalls": [
+            "More steps can increase cost and latency.",
+            "Poor control loops can amplify mistakes instead of correcting them.",
+            "Without guardrails, tool use can become noisy or unsafe.",
+        ],
+        "best_practices": [
+            "Use agentic loops only when the task benefits from iteration.",
+            "Record tool traces for debugging.",
+            "Stop early when evidence is already sufficient.",
+        ],
+        "use_cases": [
+            "deep research agents",
+            "multi-step support diagnosis",
+            "workflow automation with retrieval",
+        ],
+    },
+    {
+        "id": "mcp_definition",
+        "topic": "MCP",
+        "title": "Definition of MCP",
+        "summary": "The Model Context Protocol is an open standard that lets AI applications connect to external tools, resources, and workflows.",
+        "key_points": [
+            "MCP standardizes integration between AI applications and external systems.",
+            "It supports tools, data resources, and reusable prompts.",
+            "The design goal is interoperability instead of custom one-off connectors.",
+        ],
+        "pitfalls": [
+            "MCP is not a model architecture or a retrieval algorithm.",
+            "Using the protocol does not remove the need for access control.",
+            "A bad tool design remains bad even with a standard protocol.",
+        ],
+        "best_practices": [
+            "Use MCP when you need structured tool or context integration.",
+            "Keep server capabilities explicit and documented.",
+            "Design tools for clarity and predictable behavior.",
+        ],
+        "use_cases": [
+            "IDE assistants",
+            "enterprise chat agents",
+            "local tool orchestration",
+        ],
+    },
+    {
+        "id": "mcp_value",
+        "topic": "MCP",
+        "title": "Why MCP matters",
+        "summary": "MCP reduces integration cost by giving clients and servers a shared protocol for context exchange and actions.",
+        "key_points": [
+            "Developers can build once and integrate across multiple hosts.",
+            "AI applications gain structured access to data and actions.",
+            "End users benefit from more capable assistants with fewer brittle integrations.",
+        ],
+        "pitfalls": [
+            "Protocol support alone does not guarantee a good user experience.",
+            "Overexposing server capabilities can increase risk.",
+            "Lack of schema clarity makes tool use unreliable.",
+        ],
+        "best_practices": [
+            "Publish precise capability schemas.",
+            "Design for portability across hosts.",
+            "Treat auth and permissions as first-class concerns.",
+        ],
+        "use_cases": [
+            "cross-platform agent integrations",
+            "editor tooling",
+            "internal workflow assistants",
+        ],
+    },
+    {
+        "id": "mcp_architecture",
+        "topic": "MCP",
+        "title": "Hosts, clients, and servers",
+        "summary": "MCP follows a client-server model where a host application manages one client connection per server.",
+        "key_points": [
+            "The host is the AI application coordinating the experience.",
+            "The client manages the protocol connection to a server.",
+            "The server exposes context or actions to the client.",
+        ],
+        "pitfalls": [
+            "Confusing the host with the server leads to incorrect designs.",
+            "Treating all servers as interchangeable hides capability differences.",
+            "Ignoring connection lifecycle creates brittle integrations.",
+        ],
+        "best_practices": [
+            "Map host, client, and server roles explicitly in architecture docs.",
+            "Store capability metadata per connection.",
+            "Separate protocol management from model reasoning logic.",
+        ],
+        "use_cases": [
+            "VS Code plus filesystem server",
+            "assistant plus database server",
+            "multi-server agent hosts",
+        ],
+    },
+    {
+        "id": "mcp_layers",
+        "topic": "MCP",
+        "title": "Data layer and transport layer",
+        "summary": "MCP separates the JSON-RPC data protocol from the transport that carries those messages.",
+        "key_points": [
+            "The data layer defines requests, responses, notifications, and primitives.",
+            "The transport layer handles framing, connection establishment, and authentication.",
+            "The same protocol semantics can work over different transports.",
+        ],
+        "pitfalls": [
+            "Mixing protocol semantics with transport specifics reduces portability.",
+            "Transport security must not be assumed from protocol semantics alone.",
+            "State handling still matters even when transport is abstracted.",
+        ],
+        "best_practices": [
+            "Keep the wire protocol independent from business logic.",
+            "Document which capabilities depend on transport features.",
+            "Test both local and remote connection paths when supported.",
+        ],
+        "use_cases": [
+            "local stdio tools",
+            "remote HTTP services",
+            "protocol portability",
+        ],
+    },
+    {
+        "id": "mcp_transports",
+        "topic": "MCP",
+        "title": "Stdio and streamable HTTP",
+        "summary": "MCP commonly uses stdio for local integrations and streamable HTTP for remote services.",
+        "key_points": [
+            "Stdio is efficient for local process-to-process communication.",
+            "Streamable HTTP fits remote servers and standard web auth patterns.",
+            "OAuth or bearer-style auth is relevant for remote deployments.",
+        ],
+        "pitfalls": [
+            "Choosing remote transport for a purely local tool adds avoidable overhead.",
+            "Ignoring auth on remote transports is a major security error.",
+            "Different deployment models need different observability plans.",
+        ],
+        "best_practices": [
+            "Use stdio for local single-machine workflows.",
+            "Use HTTP when multi-client remote access is required.",
+            "Align transport choice with operational boundaries.",
+        ],
+        "use_cases": [
+            "local filesystem servers",
+            "remote SaaS integrations",
+            "hybrid agent stacks",
+        ],
+    },
+    {
+        "id": "mcp_lifecycle",
+        "topic": "MCP",
+        "title": "Lifecycle and initialization",
+        "summary": "MCP is stateful and begins with an initialization exchange that negotiates protocol version and capabilities.",
+        "key_points": [
+            "Initialization establishes compatibility between client and server.",
+            "Capability negotiation tells each side which primitives are supported.",
+            "Identity fields help with debugging and version tracking.",
+        ],
+        "pitfalls": [
+            "Skipping initialization breaks capability assumptions.",
+            "Protocol version mismatches can cause subtle failures.",
+            "Stateful connections require explicit teardown logic.",
+        ],
+        "best_practices": [
+            "Fail fast on incompatible versions.",
+            "Persist negotiated capabilities in connection state.",
+            "Emit readiness only after initialization succeeds.",
+        ],
+        "use_cases": [
+            "server registration",
+            "capability discovery",
+            "session bootstrap",
+        ],
+    },
+    {
+        "id": "mcp_server_primitives",
+        "topic": "MCP",
+        "title": "Server primitives",
+        "summary": "Servers can expose tools, resources, and prompts so hosts can discover and use them systematically.",
+        "key_points": [
+            "Tools represent executable actions.",
+            "Resources represent contextual data sources.",
+            "Prompts represent reusable interaction templates.",
+        ],
+        "pitfalls": [
+            "Blurry tool descriptions reduce model reliability.",
+            "Resources without scope control can leak sensitive data.",
+            "Prompts should not hide important operational assumptions.",
+        ],
+        "best_practices": [
+            "Name tools clearly and uniquely.",
+            "Provide explicit schemas and descriptions.",
+            "Expose only the primitives that serve real workflows.",
+        ],
+        "use_cases": [
+            "calculator tools",
+            "schema resources",
+            "prompt libraries",
+        ],
+    },
+    {
+        "id": "mcp_client_primitives",
+        "topic": "MCP",
+        "title": "Client primitives",
+        "summary": "Clients can support sampling, elicitation, and logging so servers can participate in richer workflows.",
+        "key_points": [
+            "Sampling lets a server request model completions from the host side.",
+            "Elicitation lets a server ask the user for additional information.",
+            "Logging lets servers send debugging or monitoring information.",
+        ],
+        "pitfalls": [
+            "Server authors should not assume every client supports every primitive.",
+            "Unbounded elicitation can frustrate users.",
+            "Logs can leak sensitive values if not scrubbed.",
+        ],
+        "best_practices": [
+            "Check negotiated capabilities before using client-side features.",
+            "Use elicitation only when the missing information matters.",
+            "Keep logs structured and privacy-aware.",
+        ],
+        "use_cases": [
+            "model-independent server logic",
+            "interactive approval flows",
+            "debuggable integrations",
+        ],
+    },
+    {
+        "id": "mcp_tool_discovery",
+        "topic": "MCP",
+        "title": "Tool discovery",
+        "summary": "Clients discover tools through list operations before attempting execution.",
+        "key_points": [
+            "Discovery avoids guessing what a server supports.",
+            "Tool metadata should include name, title, description, and input schema.",
+            "Hosts can build a unified tool registry from multiple servers.",
+        ],
+        "pitfalls": [
+            "Calling undeclared tools is brittle.",
+            "Weak descriptions make tool selection harder for the model.",
+            "Ignoring schema validation increases runtime failures.",
+        ],
+        "best_practices": [
+            "Refresh discovery when capabilities change.",
+            "Use descriptive tool names.",
+            "Validate tool arguments against the published schema.",
+        ],
+        "use_cases": [
+            "tool registries",
+            "dynamic agent capabilities",
+            "multi-server hosts",
+        ],
+    },
+    {
+        "id": "mcp_tool_execution",
+        "topic": "MCP",
+        "title": "Tool execution",
+        "summary": "After discovery, a client can call a tool with structured arguments and receive structured content back.",
+        "key_points": [
+            "Tool calls reference the exact discovered tool name.",
+            "Arguments should conform to the tool's JSON schema.",
+            "Responses can contain structured content, not just plain text.",
+        ],
+        "pitfalls": [
+            "Ambiguous tool names make routing error-prone.",
+            "Malformed arguments create avoidable failures.",
+            "Treating tool output as trusted without validation can be risky.",
+        ],
+        "best_practices": [
+            "Keep tool contracts stable.",
+            "Return actionable, well-typed results.",
+            "Capture execution traces for debugging and safety review.",
+        ],
+        "use_cases": [
+            "weather lookup",
+            "database queries",
+            "filesystem operations",
+        ],
+    },
+    {
+        "id": "mcp_notifications",
+        "topic": "MCP",
+        "title": "Notifications and live updates",
+        "summary": "Notifications let servers tell clients about changes, such as tool list updates, without waiting for a request.",
+        "key_points": [
+            "Notifications are one-way JSON-RPC messages with no response expected.",
+            "They are useful when available capabilities change dynamically.",
+            "Clients typically refresh their local registry after a change notification.",
+        ],
+        "pitfalls": [
+            "Clients that ignore notifications can become stale.",
+            "Sending change events without rate control can create noise.",
+            "Capabilities should declare whether list-changed notifications are supported.",
+        ],
+        "best_practices": [
+            "Use notifications for meaningful state changes.",
+            "Tie notification handling to cache refresh logic.",
+            "Keep event semantics explicit.",
+        ],
+        "use_cases": [
+            "dynamic tool catalogs",
+            "live server state updates",
+            "responsive agent interfaces",
+        ],
+    },
+    {
+        "id": "mcp_security",
+        "topic": "MCP",
+        "title": "Security and governance",
+        "summary": "MCP integrations still need authentication, authorization, scope control, and careful data handling.",
+        "key_points": [
+            "Remote servers should use appropriate authentication mechanisms.",
+            "Servers should expose only the minimum necessary capabilities.",
+            "Operational logs and tool arguments can contain sensitive data.",
+        ],
+        "pitfalls": [
+            "Assuming protocol standardization removes security work is incorrect.",
+            "Overbroad tools can create unnecessary blast radius.",
+            "Weak auditability slows incident response.",
+        ],
+        "best_practices": [
+            "Apply least privilege.",
+            "Audit tool usage and sensitive accesses.",
+            "Separate local trusted tools from remote untrusted surfaces.",
+        ],
+        "use_cases": [
+            "enterprise governance",
+            "remote connector security",
+            "access-controlled assistants",
+        ],
+    },
+    {
+        "id": "bridge_mcp_rag",
+        "topic": "Bridge",
+        "title": "MCP plus RAG",
+        "summary": "MCP and RAG solve different layers of the stack: RAG grounds answers with retrieved evidence, while MCP standardizes how tools and context services are connected.",
+        "key_points": [
+            "RAG is a retrieval-and-generation pattern.",
+            "MCP is an interoperability protocol.",
+            "A retrieval service can be exposed through MCP as a tool or resource.",
+        ],
+        "pitfalls": [
+            "MCP does not replace retrieval design.",
+            "RAG does not replace integration contracts.",
+            "Confusing protocol choice with answer quality leads to bad planning.",
+        ],
+        "best_practices": [
+            "Use RAG for grounding and MCP for integration boundaries.",
+            "Keep retrieval quality metrics separate from protocol compatibility metrics.",
+            "Expose retrieval services through clear MCP schemas when integrating agents.",
+        ],
+        "use_cases": [
+            "agentic research assistants",
+            "tool-based knowledge systems",
+            "retrieval-backed IDE helpers",
+        ],
+    },
+    {
+        "id": "bridge_retrieval_server",
+        "topic": "Bridge",
+        "title": "Retrieval as an MCP server",
+        "summary": "A retrieval engine can be packaged as an MCP server so hosts can discover search tools and evidence resources dynamically.",
+        "key_points": [
+            "An MCP server can expose search tools with structured query schemas.",
+            "Retrieved passages can be returned as typed content with metadata.",
+            "This keeps retrieval integration reusable across hosts.",
+        ],
+        "pitfalls": [
+            "If the retrieval server lacks filters, tenants may leak across queries.",
+            "Search results without provenance weaken trust.",
+            "Large unranked result sets inflate token cost.",
+        ],
+        "best_practices": [
+            "Return ranked snippets with metadata.",
+            "Expose filters explicitly in the tool schema.",
+            "Keep retrieval latency visible to the host.",
+        ],
+        "use_cases": [
+            "shared search backends",
+            "multi-host retrieval reuse",
+            "retrieval-backed agent tools",
+        ],
+    },
+    {
+        "id": "bridge_agent_design",
+        "topic": "Bridge",
+        "title": "Agent design with RAG and MCP",
+        "summary": "A strong agent architecture often uses RAG for evidence and MCP for structured tool access around that evidence.",
+        "key_points": [
+            "The agent can retrieve evidence, call tools, and synthesize a grounded answer.",
+            "MCP helps the host manage multiple external capabilities consistently.",
+            "RAG helps the model reason with fresher and narrower evidence.",
+        ],
+        "pitfalls": [
+            "Too many tools without routing logic can overwhelm the system.",
+            "Too much retrieved context can bury the key answer.",
+            "Without evaluation, complex agent loops hide regressions.",
+        ],
+        "best_practices": [
+            "Start with a simple grounded path before adding loop complexity.",
+            "Measure both tool success and answer faithfulness.",
+            "Keep the final answer tied to cited evidence or tool outputs.",
+        ],
+        "use_cases": [
+            "engineering copilots",
+            "support agents",
+            "analyst assistants",
+        ],
+    },
+]
+
+
+AUDIENCES = [
+    "a junior ML engineer",
+    "a backend engineer",
+    "a product manager",
+    "an enterprise architect",
+    "a solutions engineer",
+    "an AI platform lead",
+]
+
+CONSTRAINTS = [
+    "Keep the answer practical and free of hype.",
+    "Use plain English with no jargon overload.",
+    "Make the answer useful for implementation planning.",
+    "Focus on operational tradeoffs, not marketing language.",
+    "Explain the idea cleanly without drifting into unrelated topics.",
+]
+
+FOCUSES = [
+    "definition and purpose",
+    "system design choices",
+    "deployment tradeoffs",
+    "quality and evaluation",
+    "security and governance",
+    "how to explain it to a team",
+]
+
+QUESTION_OPENERS = [
+    "What is",
+    "Explain",
+    "Give a practical explanation of",
+    "Teach me",
+    "Describe",
+    "Help me understand",
+]
+
+DESCRIPTION_OPENERS = [
+    "Write a description of",
+    "Draft an internal note about",
+    "Create a clean explainer for",
+    "Produce a concise briefing on",
+    "Write a technical overview of",
+]
+
+COMPARISON_OPENERS = [
+    "Compare",
+    "Explain the difference between",
+    "When should a team choose",
+    "Contrast",
+]
+
+SCENARIO_OPENERS = [
+    "How should a team use",
+    "What does good implementation of",
+    "How would you apply",
+    "Design a practical approach for",
+]
+
+MISCONCEPTION_OPENERS = [
+    "Correct this misunderstanding about",
+    "What do people often get wrong about",
+    "Clarify a common mistake about",
+]
+
+
+def normalize_text(text):
+    text = text.replace("\r\n", "\n")
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def clean_sentence(text):
+    return text.strip().rstrip(".")
+
+
+def format_entry(entry):
+    parts = [f"### Instruction:\n{entry['instruction'].strip()}"]
+    if entry.get("input"):
+        parts.append(f"### Input:\n{entry['input'].strip()}")
+    parts.append(f"### Response:\n{entry['output'].strip()}")
+    return "\n\n".join(parts)
+
+
+def hash_entry(entry):
+    key = "|||".join(
+        normalize_text(entry.get(field, "")).lower()
+        for field in ("instruction", "input", "output")
+    )
+    return hashlib.sha1(key.encode("utf-8")).hexdigest()
+
+
+def sentence_join(items):
+    return " ".join(item.rstrip(".") + "." for item in items if item)
+
+
+def choose_subset(values, rng, minimum=2, maximum=3):
+    count = min(len(values), rng.randint(minimum, maximum))
+    return rng.sample(values, count)
+
+
+def build_context(rng, extra_focus=None):
+    lines = [
+        f"Audience: {rng.choice(AUDIENCES)}",
+        f"Focus: {extra_focus or rng.choice(FOCUSES)}",
+        f"Style rule: {rng.choice(CONSTRAINTS)}",
+    ]
+    if rng.random() < 0.55:
+        lines.append(f"Length target: {rng.choice(['120-180 words', '180-260 words', '220-320 words', '4-6 bullet points'])}")
+    return "\n".join(lines)
+
+
+def build_direct_output(card, rng):
+    facts = choose_subset(card["key_points"], rng, 2, 3)
+    practices = choose_subset(card["best_practices"], rng, 1, 2)
+    opener = f"The concept '{card['title']}' can be understood like this:"
+    body = sentence_join([card["summary"], *facts])
+    close = "Good practice is to " + ", then ".join(
+        clean_sentence(practice[0].lower() + practice[1:] if practice and practice[0].isupper() else practice)
+        for practice in practices
+    ).rstrip(".") + "."
+    return normalize_text(f"{opener} {body} {close}")
+
+
+def build_description_output(card, rng):
+    facts = choose_subset(card["key_points"], rng, 3, 3)
+    pitfalls = choose_subset(card["pitfalls"], rng, 1, 2)
+    use_cases = choose_subset(card["use_cases"], rng, 2, 3)
+    lines = [
+        f"The topic '{card['title']}' matters because {card['summary'][0].lower() + card['summary'][1:]}",
+        "Key points:",
+    ]
+    lines.extend(f"- {fact}" for fact in facts)
+    lines.append("Why teams care:")
+    lines.extend(f"- It supports {use_case}." for use_case in use_cases)
+    lines.append("What to avoid:")
+    lines.extend(f"- {pitfall}" for pitfall in pitfalls)
+    return normalize_text("\n".join(lines))
+
+
+def build_checklist_output(card, rng):
+    practices = choose_subset(card["best_practices"], rng, 3, 3)
+    pitfalls = choose_subset(card["pitfalls"], rng, 2, 2)
+    lines = [
+        f"Use this checklist when working with '{card['title']}':",
+        *[f"- {clean_sentence(practice)}." for practice in practices],
+        "Watch-outs:",
+        *[f"- {clean_sentence(pitfall)}." for pitfall in pitfalls],
+    ]
+    return normalize_text("\n".join(lines))
+
+
+def build_scenario_output(card, rng):
+    facts = choose_subset(card["key_points"], rng, 2, 3)
+    practices = choose_subset(card["best_practices"], rng, 2, 2)
+    scenario = rng.choice(card["use_cases"])
+    lines = [
+        f"For a team building around {scenario}, start by treating '{card['title']}' as an engineering system rather than a vague AI feature.",
+        sentence_join([card["summary"], *facts]),
+        "A practical rollout would:",
+        *[f"- {clean_sentence(practice)}." for practice in practices],
+    ]
+    return normalize_text("\n".join(lines))
+
+
+def build_misconception_output(card, rng):
+    pitfall = rng.choice(card["pitfalls"])
+    facts = choose_subset(card["key_points"], rng, 2, 2)
+    return normalize_text(
+        f"A common mistake is this: {pitfall} The correction is straightforward. "
+        f"{card['summary']} {sentence_join(facts)}"
+    )
+
+
+def build_comparison_output(left, right, rng):
+    left_fact = rng.choice(left["key_points"])
+    right_fact = rng.choice(right["key_points"])
+    left_practice = rng.choice(left["best_practices"])
+    right_practice = rng.choice(right["best_practices"])
+    lines = [
+        f"'{left['title']}' and '{right['title']}' solve different problems, even when they appear in the same AI stack.",
+        f"For '{left['title']}': {left['summary']} {left_fact}",
+        f"For '{right['title']}': {right['summary']} {right_fact}",
+        "Use them together only when the product needs both grounded knowledge and structured external capabilities.",
+        f"A sensible rule is to {clean_sentence(left_practice[0].lower() + left_practice[1:] if left_practice and left_practice[0].isupper() else left_practice)} and to {clean_sentence(right_practice[0].lower() + right_practice[1:] if right_practice and right_practice[0].isupper() else right_practice)}.",
+    ]
+    return normalize_text(" ".join(lines))
+
+
+def make_question_entry(card, rng):
+    instruction = f"{rng.choice(QUESTION_OPENERS)} '{card['title']}' in practical terms."
+    return {
+        "instruction": instruction,
+        "input": build_context(rng),
+        "output": build_direct_output(card, rng),
+        "kind": "qna",
+        "topic": card["topic"],
+        "sources": [card["id"]],
+    }
+
+
+def make_description_entry(card, rng):
+    instruction = f"{rng.choice(DESCRIPTION_OPENERS)} '{card['title']}' for an engineering handbook."
+    return {
+        "instruction": instruction,
+        "input": build_context(rng, "definition and purpose"),
+        "output": build_description_output(card, rng),
+        "kind": "description",
+        "topic": card["topic"],
+        "sources": [card["id"]],
+    }
+
+
+def make_checklist_entry(card, rng):
+    instruction = f"Create an implementation checklist for '{card['title']}'."
+    return {
+        "instruction": instruction,
+        "input": build_context(rng, "system design choices"),
+        "output": build_checklist_output(card, rng),
+        "kind": "checklist",
+        "topic": card["topic"],
+        "sources": [card["id"]],
+    }
+
+
+def make_scenario_entry(card, rng):
+    opener = rng.choice(SCENARIO_OPENERS)
+    if opener == "What does good implementation of":
+        instruction = f"What does good implementation of '{card['title']}' look like for a real product team?"
+    else:
+        instruction = f"{opener} '{card['title']}' for a real product team?"
+    return {
+        "instruction": instruction,
+        "input": build_context(rng, "deployment tradeoffs"),
+        "output": build_scenario_output(card, rng),
+        "kind": "scenario",
+        "topic": card["topic"],
+        "sources": [card["id"]],
+    }
+
+
+def make_misconception_entry(card, rng):
+    instruction = f"{rng.choice(MISCONCEPTION_OPENERS)} '{card['title']}'."
+    return {
+        "instruction": instruction,
+        "input": build_context(rng, "how to explain it to a team"),
+        "output": build_misconception_output(card, rng),
+        "kind": "clarification",
+        "topic": card["topic"],
+        "sources": [card["id"]],
+    }
+
+
+def make_comparison_entry(cards, rng):
+    left, right = rng.sample(cards, 2)
+    instruction = f"{rng.choice(COMPARISON_OPENERS)} '{left['title']}' and '{right['title']}'."
+    return {
+        "instruction": instruction,
+        "input": build_context(rng, "system design choices"),
+        "output": build_comparison_output(left, right, rng),
+        "kind": "comparison",
+        "topic": f"{left['topic']}+{right['topic']}",
+        "sources": [left["id"], right["id"]],
+    }
+
+
+ENTRY_BUILDERS = [
+    make_question_entry,
+    make_description_entry,
+    make_checklist_entry,
+    make_scenario_entry,
+    make_misconception_entry,
+]
+
+
+def trim_to_window(entry, tokenizer, max_seq_len):
+    output = entry["output"]
+    while True:
+        total_tokens = len(tokenizer.encode(format_entry(entry)))
+        if total_tokens <= max_seq_len:
+            return entry, total_tokens
+        pieces = re.split(r"(?<=[.!?])\s+", output)
+        if len(pieces) <= 2:
+            return None, None
+        output = " ".join(pieces[:-1]).strip()
+        entry["output"] = output
+
+
+def build_dataset(target_tokens, seed, max_seq_len, val_ratio):
+    rng = random.Random(seed)
+    tokenizer = AutoTokenizer.from_pretrained(str(TOKENIZER_DIR))
+    manifest_path = DATASET_DIR / "source_manifest.json"
+    manifest_path.write_text(json.dumps({
+        "retrieved_on": RETRIEVED_ON,
+        "sources": SOURCE_MANIFEST,
+    }, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    accepted = []
+    seen = set()
+    total_tokens = 0
+    attempts = 0
+    builder_weights = [5, 4, 3, 3, 2, 3]
+
+    while total_tokens < target_tokens:
+        attempts += 1
+        builder_index = rng.choices(range(6), weights=builder_weights, k=1)[0]
+        if builder_index == 5:
+            candidate = make_comparison_entry(KNOWLEDGE_CARDS, rng)
+        else:
+            card = rng.choice(KNOWLEDGE_CARDS)
+            candidate = ENTRY_BUILDERS[builder_index](card, rng)
+
+        candidate["instruction"] = normalize_text(candidate["instruction"])
+        candidate["input"] = normalize_text(candidate["input"])
+        candidate["output"] = normalize_text(candidate["output"])
+
+        trimmed, token_count = trim_to_window(candidate, tokenizer, max_seq_len)
+        if trimmed is None or token_count is None or token_count < 80:
+            continue
+
+        dedupe_key = hash_entry(trimmed)
+        if dedupe_key in seen:
+            continue
+
+        seen.add(dedupe_key)
+        accepted.append({
+            "instruction": trimmed["instruction"],
+            "input": trimmed["input"],
+            "output": trimmed["output"],
+            "meta": {
+                "kind": trimmed["kind"],
+                "topic": trimmed["topic"],
+                "sources": trimmed["sources"],
+                "token_count": token_count,
+            },
+        })
+        total_tokens += token_count
+
+        if len(accepted) % 1000 == 0:
+            print(f"accepted={len(accepted):,} tokens={total_tokens:,} attempts={attempts:,}")
+
+    rng.shuffle(accepted)
+    val_size = max(1, int(len(accepted) * val_ratio))
+    val_data = accepted[:val_size]
+    train_data = accepted[val_size:]
+    return train_data, val_data, total_tokens
+
+
+def write_outputs(train_data, val_data, total_tokens, target_tokens):
+    train_path = DATASET_DIR / "train.json"
+    val_path = DATASET_DIR / "val.json"
+    full_path = DATASET_DIR / "all.json"
+    report_path = DATASET_DIR / "BUILD_REPORT.md"
+    preview_path = DATASET_DIR / "sample_preview.json"
+
+    compact_train = [{k: entry[k] for k in ("instruction", "input", "output")} for entry in train_data]
+    compact_val = [{k: entry[k] for k in ("instruction", "input", "output")} for entry in val_data]
+    compact_full = compact_train + compact_val
+
+    train_path.write_text(json.dumps(compact_train, ensure_ascii=False), encoding="utf-8")
+    val_path.write_text(json.dumps(compact_val, ensure_ascii=False), encoding="utf-8")
+    full_path.write_text(json.dumps(compact_full, ensure_ascii=False), encoding="utf-8")
+    preview_path.write_text(json.dumps((train_data + val_data)[:20], indent=2, ensure_ascii=False), encoding="utf-8")
+
+    avg_tokens = total_tokens / max(1, len(compact_full))
+    by_kind = {}
+    by_topic = {}
+    for entry in train_data + val_data:
+        kind = entry["meta"]["kind"]
+        topic = entry["meta"]["topic"]
+        by_kind[kind] = by_kind.get(kind, 0) + 1
+        by_topic[topic] = by_topic.get(topic, 0) + 1
+
+    lines = [
+        "# RAG + MCP SFT Build Report",
+        "",
+        f"- Retrieved on: {RETRIEVED_ON}",
+        f"- Target tokens: {target_tokens:,}",
+        f"- Realized tokens: {total_tokens:,}",
+        f"- Train samples: {len(train_data):,}",
+        f"- Val samples: {len(val_data):,}",
+        f"- Total samples: {len(compact_full):,}",
+        f"- Average formatted tokens per sample: {avg_tokens:.1f}",
+        f"- Max window enforced: 1024 tokens",
+        "",
+        "## Breakdown by kind",
+        "",
+    ]
+    for kind, count in sorted(by_kind.items()):
+        lines.append(f"- {kind}: {count:,}")
+    lines.extend(["", "## Breakdown by topic", ""])
+    for topic, count in sorted(by_topic.items()):
+        lines.append(f"- {topic}: {count:,}")
+    lines.extend(["", "## Files", "", "- train.json", "- val.json", "- all.json", "- sample_preview.json", "- source_manifest.json"])
+    report_path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build a clean RAG+MCP SFT dataset for LUNA")
+    parser.add_argument("--target-tokens", type=int, default=10_000_000)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--max-seq-len", type=int, default=1024)
+    parser.add_argument("--val-ratio", type=float, default=0.02)
+    args = parser.parse_args()
+
+    train_data, val_data, total_tokens = build_dataset(
+        target_tokens=args.target_tokens,
+        seed=args.seed,
+        max_seq_len=args.max_seq_len,
+        val_ratio=args.val_ratio,
+    )
+    write_outputs(train_data, val_data, total_tokens, args.target_tokens)
+    print(
+        f"dataset_ready train={len(train_data):,} val={len(val_data):,} total_tokens={total_tokens:,} "
+        f"dir={DATASET_DIR}"
+    )
+
+
+if __name__ == "__main__":
+    main()