Day 1 Complete: Tokenizer setup — Qwen2.5-Coder-7B base + 22 MINDI special tokens (vocab 151,685), wrapper class, full format test

.env.example

@@ -28,7 +28,7 @@ E2B_API_KEY=e2b_your_key_here
 SANDBOX_TYPE=e2b
 
 # ── Model Settings ──
-MODEL_NAME=deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct
+MODEL_NAME=Qwen/Qwen2.5-Coder-7B-Instruct
 BASE_MODEL_PATH=./checkpoints/base
 FINETUNED_MODEL_PATH=./checkpoints/finetuned
 

configs/data_config.yaml

@@ -36,7 +36,7 @@ dataset:
 
   # Processing
   processing:
-    tokenizer: "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
+    tokenizer: "Qwen/Qwen2.5-Coder-7B-Instruct"
     max_length: 8192
     min_length: 64
     dedup_strategy: "minhash"

configs/model_config.yaml

@@ -8,10 +8,10 @@ model:
   
   # Base coding model (Apache 2.0 licensed)
   base:
-    name: "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
-    parameters: "16B"
+    name: "Qwen/Qwen2.5-Coder-7B-Instruct"
+    parameters: "7.61B"
     license: "Apache-2.0"
-    context_length: 8192
+    context_length: 32768
     dtype: "bfloat16"
 
   # Vision encoder for UI screenshot understanding

data/tokenizer/base_tokenizer/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

data/tokenizer/base_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

data/tokenizer/base_tokenizer/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

data/tokenizer/mindi_tokenizer/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

data/tokenizer/mindi_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d9f5b46199913fa238437fd9bbee25cef9eb1fb59bd860a347af628f161062
+size 11425720

data/tokenizer/mindi_tokenizer/tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|mindi_start|>",
+    "<|mindi_end|>",
+    "<|code_start|>",
+    "<|code_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|critique_start|>",
+    "<|critique_end|>",
+    "<|suggest_start|>",
+    "<|suggest_end|>",
+    "<|think_start|>",
+    "<|think_end|>",
+    "<|file_start|>",
+    "<|file_end|>",
+    "<|search_start|>",
+    "<|search_end|>",
+    "<|sandbox_start|>",
+    "<|sandbox_end|>",
+    "<|error_start|>",
+    "<|error_end|>",
+    "<|fix_start|>",
+    "<|fix_end|>"
+  ],
+  "is_local": true,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

scripts/add_special_tokens.py

@@ -0,0 +1,109 @@
+"""
+MINDI 1.5 Vision-Coder — Step 4: Add MINDI Special Tokens
+
+Loads the base Qwen2.5-Coder tokenizer, adds 22 MINDI-specific
+special tokens, saves the updated tokenizer, and reports vocab changes.
+"""
+
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+MINDI_SPECIAL_TOKENS = [
+    "<|mindi_start|>",
+    "<|mindi_end|>",
+    "<|code_start|>",
+    "<|code_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|critique_start|>",
+    "<|critique_end|>",
+    "<|suggest_start|>",
+    "<|suggest_end|>",
+    "<|think_start|>",
+    "<|think_end|>",
+    "<|file_start|>",
+    "<|file_end|>",
+    "<|search_start|>",
+    "<|search_end|>",
+    "<|sandbox_start|>",
+    "<|sandbox_end|>",
+    "<|error_start|>",
+    "<|error_end|>",
+    "<|fix_start|>",
+    "<|fix_end|>",
+]
+
+
+def main():
+    from transformers import AutoTokenizer
+
+    base_dir = PROJECT_ROOT / "data" / "tokenizer" / "base_tokenizer"
+    save_dir = PROJECT_ROOT / "data" / "tokenizer" / "mindi_tokenizer"
+
+    print(f"\n{'='*60}")
+    print(f"  Step 4: Adding MINDI Special Tokens")
+    print(f"{'='*60}")
+
+    # Load base tokenizer
+    print(f"\n  Loading base tokenizer from: {base_dir}")
+    tokenizer = AutoTokenizer.from_pretrained(str(base_dir), trust_remote_code=True)
+    original_vocab_size = len(tokenizer)
+    print(f"  ✅ Base vocab size: {original_vocab_size:,}")
+
+    # Add special tokens
+    print(f"\n  Adding {len(MINDI_SPECIAL_TOKENS)} MINDI special tokens...")
+    num_added = tokenizer.add_special_tokens({
+        "additional_special_tokens": MINDI_SPECIAL_TOKENS
+    })
+    new_vocab_size = len(tokenizer)
+    print(f"  ✅ Tokens added: {num_added}")
+    print(f"  ✅ New vocab size: {new_vocab_size:,}")
+    print(f"  ✅ Delta: +{new_vocab_size - original_vocab_size}")
+
+    # Save updated tokenizer
+    save_dir.mkdir(parents=True, exist_ok=True)
+    tokenizer.save_pretrained(str(save_dir))
+    print(f"\n  ✅ Saved MINDI tokenizer to: {save_dir}")
+
+    # Show token ID mapping
+    print(f"\n{'='*60}")
+    print(f"  Special Token ID Mapping")
+    print(f"{'='*60}")
+    for token in MINDI_SPECIAL_TOKENS:
+        token_id = tokenizer.convert_tokens_to_ids(token)
+        print(f"    {token:<25} → ID {token_id}")
+
+    # Verify round-trip for each special token
+    print(f"\n{'='*60}")
+    print(f"  Round-trip Verification")
+    print(f"{'='*60}")
+    all_pass = True
+    for token in MINDI_SPECIAL_TOKENS:
+        token_id = tokenizer.convert_tokens_to_ids(token)
+        decoded = tokenizer.decode([token_id])
+        match = decoded == token
+        if not match:
+            all_pass = False
+        status = "✅" if match else "❌"
+        print(f"    {status} {token} → {token_id} → \"{decoded}\"")
+
+    # Summary
+    print(f"\n{'='*60}")
+    print(f"  SUMMARY")
+    print(f"{'='*60}")
+    print(f"  Original vocab size:  {original_vocab_size:,}")
+    print(f"  New vocab size:       {new_vocab_size:,}")
+    print(f"  Special tokens added: {num_added}")
+    if all_pass:
+        print(f"  Round-trip test:      ✅ ALL {len(MINDI_SPECIAL_TOKENS)} PASSED")
+    else:
+        print(f"  Round-trip test:      ❌ SOME FAILED")
+    print(f"{'='*60}\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/download_tokenizer.py

@@ -0,0 +1,91 @@
+"""
+MINDI 1.5 Vision-Coder — Step 3: Download Tokenizer & Test
+
+Downloads ONLY the tokenizer (not model weights) from Qwen/Qwen2.5-Coder-7B-Instruct,
+saves it locally, and runs encoding/decoding tests on 8 code strings.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# Ensure project root
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PROJECT_ROOT))
+
+from dotenv import load_dotenv
+load_dotenv(PROJECT_ROOT / ".env")
+
+
+def main():
+    from transformers import AutoTokenizer
+
+    model_name = "Qwen/Qwen2.5-Coder-7B-Instruct"
+    save_dir = PROJECT_ROOT / "data" / "tokenizer" / "base_tokenizer"
+    hf_token = os.environ.get("HUGGINGFACE_TOKEN", "")
+
+    # ── Download tokenizer ──
+    print(f"\n{'='*60}")
+    print(f"  Downloading tokenizer: {model_name}")
+    print(f"  Save to: {save_dir}")
+    print(f"{'='*60}\n")
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_name,
+        token=hf_token if hf_token else None,
+        trust_remote_code=True,
+    )
+
+    # Save locally
+    save_dir.mkdir(parents=True, exist_ok=True)
+    tokenizer.save_pretrained(str(save_dir))
+    print(f"  ✅ Tokenizer saved to {save_dir}")
+    print(f"  ✅ Vocab size: {tokenizer.vocab_size:,}")
+    print(f"  ✅ Model max length: {tokenizer.model_max_length:,}")
+
+    # ── List saved files ──
+    print(f"\n  Saved files:")
+    for f in sorted(save_dir.iterdir()):
+        size_kb = f.stat().st_size / 1024
+        print(f"    {f.name} ({size_kb:.1f} KB)")
+
+    # ── Run tokenizer tests ──
+    test_strings = [
+        "Build me a Next.js dashboard",
+        "import React from 'react'",
+        "className='flex items-center gap-4'",
+        "'use client'",
+        "const [state, setState] = useState(null)",
+        "export default function Page() {",
+        "npm install framer-motion",
+        "async function getData() {",
+    ]
+
+    print(f"\n{'='*60}")
+    print(f"  Tokenizer Tests — 8 Code Strings")
+    print(f"{'='*60}")
+
+    all_pass = True
+    for i, text in enumerate(test_strings, 1):
+        ids = tokenizer.encode(text, add_special_tokens=False)
+        decoded = tokenizer.decode(ids)
+        match = decoded == text
+        if not match:
+            all_pass = False
+
+        print(f"\n  Test {i}: \"{text}\"")
+        print(f"    Token count: {len(ids)}")
+        print(f"    Token IDs:   {ids}")
+        print(f"    Decoded:     \"{decoded}\"")
+        print(f"    Match:       {'✅ PERFECT' if match else '❌ MISMATCH'}")
+
+    print(f"\n{'='*60}")
+    if all_pass:
+        print(f"  ✅ ALL 8 TESTS PASSED — Perfect reconstruction!")
+    else:
+        print(f"  ⚠️  Some tests had reconstruction differences (whitespace normalization is normal)")
+    print(f"{'='*60}\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/save_everything.py

@@ -0,0 +1,150 @@
+"""
+MINDI 1.5 Vision-Coder — Step 6: Smoke-test MindiTokenizer wrapper & generate test report.
+"""
+
+import sys
+import datetime
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT / "src"))
+
+from tokenizer.tokenizer import MindiTokenizer, MINDI_SPECIAL_TOKENS
+
+print("=" * 70)
+print("STEP 6: SAVE EVERYTHING — WRAPPER SMOKE TEST + REPORT")
+print("=" * 70)
+
+# ── 1. Load via wrapper class ────────────────────────────────────────
+print("\n1️⃣  Loading MindiTokenizer wrapper...")
+tok = MindiTokenizer()
+print(f"   ✅ Loaded from: {tok.tokenizer_path}")
+print(f"   Vocab size: {tok.get_vocab_size():,}")
+
+# ── 2. Test encode / decode ──────────────────────────────────────────
+print("\n2️⃣  encode() / decode()...")
+text = "export default function Hero() { return <h1>Hello</h1>; }"
+ids = tok.encode(text)
+decoded = tok.decode(ids)
+assert decoded.strip() == text.strip(), f"Round-trip failed: {decoded!r}"
+print(f"   ✅ Round-trip OK — {len(ids)} tokens")
+
+# ── 3. Test encode_with_special_tokens ───────────────────────────────
+print("\n3️⃣  encode_with_special_tokens()...")
+special_text = "<|code_start|>\nconsole.log('hi');\n<|code_end|>"
+ids2 = tok.encode_with_special_tokens(special_text)
+decoded2 = tok.decode(ids2)
+assert decoded2.strip() == special_text.strip(), f"Special round-trip failed"
+code_start_id = tok.get_special_token_id("code_start")
+code_end_id = tok.get_special_token_id("code_end")
+assert code_start_id in ids2, "code_start token not found"
+assert code_end_id in ids2, "code_end token not found"
+print(f"   ✅ Special tokens preserved — {len(ids2)} tokens")
+
+# ── 4. Test encode_conversation ──────────────────────────────────────
+print("\n4️⃣  encode_conversation()...")
+messages = [
+    {"role": "system", "content": "You are MINDI 1.5 Vision-Coder."},
+    {"role": "user", "content": "Build a navbar."},
+    {"role": "assistant", "content": "<|think_start|>\nPlanning navbar...\n<|think_end|>\n\n<|code_start|>\nexport default function Navbar() { return <nav>Nav</nav>; }\n<|code_end|>"},
+]
+conv_ids = tok.encode_conversation(messages, wrap_mindi=True)
+conv_decoded = tok.decode(conv_ids)
+assert "<|mindi_start|>" in conv_decoded, "mindi_start missing"
+assert "<|mindi_end|>" in conv_decoded, "mindi_end missing"
+assert "<|im_start|>" in conv_decoded, "im_start missing"
+assert "<|think_start|>" in conv_decoded, "think_start missing"
+assert "<|code_start|>" in conv_decoded, "code_start missing"
+print(f"   ✅ Conversation encoded — {len(conv_ids)} tokens, mindi/im/think/code all present")
+
+# ── 5. Test get_special_token_ids ────────────────────────────────────
+print("\n5️⃣  get_special_token_ids()...")
+all_ids = tok.get_special_token_ids()
+assert len(all_ids) == 22, f"Expected 22, got {len(all_ids)}"
+for name, tid in all_ids.items():
+    assert isinstance(tid, int) and tid > 0, f"Bad ID for {name}: {tid}"
+print(f"   ✅ 22 special token IDs returned, all valid integers")
+
+# ── 6. Test get_vocab_size ───────────────────────────────────────────
+print("\n6️⃣  get_vocab_size()...")
+vs = tok.get_vocab_size()
+assert vs == 151685, f"Expected 151685, got {vs}"
+print(f"   ✅ Vocab size: {vs:,}")
+
+# ── Generate test report ─────────────────────────────────────────────
+print("\n" + "─" * 70)
+print("📄 Generating test report...")
+
+report_lines = [
+    "=" * 70,
+    "MINDI 1.5 VISION-CODER — TOKENIZER TEST REPORT",
+    f"Generated: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+    "=" * 70,
+    "",
+    "BASE MODEL: Qwen/Qwen2.5-Coder-7B-Instruct",
+    f"VOCAB SIZE: {vs:,}",
+    f"SPECIAL TOKENS: {len(all_ids)} (22 MINDI tokens)",
+    f"TOKENIZER PATH: data/tokenizer/mindi_tokenizer/",
+    "",
+    "─" * 70,
+    "SPECIAL TOKEN REGISTRY",
+    "─" * 70,
+]
+
+for name, tid in sorted(all_ids.items(), key=lambda x: x[1]):
+    token_str = MINDI_SPECIAL_TOKENS[name]
+    report_lines.append(f"  {token_str:<25} → ID {tid}")
+
+report_lines += [
+    "",
+    "─" * 70,
+    "WRAPPER CLASS API TESTS",
+    "─" * 70,
+    "  ✅ encode()                    — round-trip plain text",
+    "  ✅ decode()                    — reconstructs original text",
+    "  ✅ encode_with_special_tokens() — preserves special tokens as single IDs",
+    "  ✅ encode_conversation()        — formats system/user/assistant with im_start/end + mindi wrapper",
+    "  ✅ get_vocab_size()            — returns 151,685",
+    "  ✅ get_special_token_ids()     — returns all 22 MINDI token IDs",
+    "  ✅ get_special_token_id(name)  — individual token lookup",
+    "",
+    "─" * 70,
+    "CONVERSATION FORMAT TEST (from Step 5)",
+    "─" * 70,
+    "  Total tokens:       971",
+    "  Round-trip:         PERFECT MATCH",
+    "  Special tokens:     22/22 preserved as single tokens",
+    "  Qwen chat tokens:   im_start ×3, im_end ×3",
+    "  Context usage:      971 / 32,768 = 3.0%",
+    "",
+    "─" * 70,
+    "FILES SAVED",
+    "─" * 70,
+    "  data/tokenizer/base_tokenizer/     — Original Qwen tokenizer (3 files)",
+    "  data/tokenizer/mindi_tokenizer/    — MINDI tokenizer with 22 special tokens",
+    "  src/tokenizer/tokenizer.py         — MindiTokenizer wrapper class",
+    "  logs/tokenizer_test.txt            — This report",
+    "  scripts/download_tokenizer.py      — Tokenizer download script",
+    "  scripts/add_special_tokens.py      — Special token addition script",
+    "  scripts/test_mindi_format.py       — Conversation format test script",
+    "",
+    "=" * 70,
+    "STATUS: ALL TESTS PASSED ✅",
+    "=" * 70,
+]
+
+report_text = "\n".join(report_lines)
+
+logs_dir = PROJECT_ROOT / "logs"
+logs_dir.mkdir(parents=True, exist_ok=True)
+report_path = logs_dir / "tokenizer_test.txt"
+report_path.write_text(report_text, encoding="utf-8")
+print(f"   ✅ Saved to: {report_path}")
+
+# ── Final summary ────────────────────────────────────────────────────
+print("\n" + "=" * 70)
+print("✅ STEP 6 COMPLETE: Everything saved!")
+print("   • MindiTokenizer wrapper class — 6/6 API methods tested")
+print("   • Test report — logs/tokenizer_test.txt")
+print(f"   • Tokenizer files — data/tokenizer/mindi_tokenizer/")
+print("=" * 70)

scripts/test_mindi_format.py

@@ -0,0 +1,262 @@
+"""
+MINDI 1.5 Vision-Coder — Step 5: Test MINDI Conversation Format
+Tests full conversation tokenization with all special tokens.
+"""
+
+from pathlib import Path
+from transformers import AutoTokenizer
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+TOKENIZER_PATH = PROJECT_ROOT / "data" / "tokenizer" / "mindi_tokenizer"
+
+# ── Load MINDI tokenizer ──────────────────────────────────────────────
+print("=" * 70)
+print("STEP 5: TEST MINDI CONVERSATION FORMAT")
+print("=" * 70)
+
+print(f"\n📂 Loading MINDI tokenizer from: {TOKENIZER_PATH}")
+tokenizer = AutoTokenizer.from_pretrained(str(TOKENIZER_PATH), trust_remote_code=True)
+print(f"   Vocab size: {len(tokenizer):,}")
+
+# ── Define MINDI special tokens ──────────────────────────────────────
+MINDI_SPECIAL_TOKENS = [
+    "<|mindi_start|>", "<|mindi_end|>",
+    "<|code_start|>", "<|code_end|>",
+    "<|vision_start|>", "<|vision_end|>",
+    "<|critique_start|>", "<|critique_end|>",
+    "<|suggest_start|>", "<|suggest_end|>",
+    "<|think_start|>", "<|think_end|>",
+    "<|file_start|>", "<|file_end|>",
+    "<|search_start|>", "<|search_end|>",
+    "<|sandbox_start|>", "<|sandbox_end|>",
+    "<|error_start|>", "<|error_end|>",
+    "<|fix_start|>", "<|fix_end|>",
+]
+
+# ── Build test conversation ──────────────────────────────────────────
+system_message = (
+    "You are MINDI 1.5 Vision-Coder, a multimodal agentic AI coding assistant "
+    "created by MINDIGENOUS.AI. You generate production-ready Next.js 14 + "
+    "Tailwind CSS + TypeScript code, see your own output via vision, critique "
+    "UI/UX, search the web for best practices, test in a sandbox, and self-fix errors."
+)
+
+user_message = "Build me a hero section for a SaaS landing page with a gradient background, animated headline, and a CTA button."
+
+assistant_message = """<|think_start|>
+The user wants a hero section for a SaaS landing page. I need to:
+1. Search for current hero section best practices
+2. Create a Next.js 14 component with Tailwind CSS
+3. Include gradient background, animated headline, and CTA button
+4. Critique the result for UI/UX quality
+<|think_end|>
+
+<|search_start|>
+query: "best SaaS hero section design patterns 2025 Next.js Tailwind"
+results: [
+  {"title": "Modern Hero Sections", "url": "https://example.com/hero-patterns"},
+  {"title": "SaaS Landing Best Practices", "url": "https://example.com/saas-landing"}
+]
+<|search_end|>
+
+<|file_start|>
+path: src/components/HeroSection.tsx
+language: typescript
+framework: next.js 14
+<|file_end|>
+
+<|code_start|>
+'use client';
+
+import { motion } from 'framer-motion';
+
+export default function HeroSection() {
+  return (
+    <section className="relative min-h-screen flex items-center justify-center bg-gradient-to-br from-indigo-600 via-purple-600 to-pink-500 overflow-hidden">
+      <div className="absolute inset-0 bg-[url('/grid.svg')] opacity-10" />
+      <div className="relative z-10 text-center px-4 max-w-4xl mx-auto">
+        <motion.h1
+          initial={{ opacity: 0, y: 20 }}
+          animate={{ opacity: 1, y: 0 }}
+          transition={{ duration: 0.8, ease: 'easeOut' }}
+          className="text-5xl md:text-7xl font-bold text-white mb-6 leading-tight"
+        >
+          Ship faster with{' '}
+          <span className="bg-clip-text text-transparent bg-gradient-to-r from-yellow-200 to-yellow-400">
+            AI-powered code
+          </span>
+        </motion.h1>
+        <motion.p
+          initial={{ opacity: 0, y: 20 }}
+          animate={{ opacity: 1, y: 0 }}
+          transition={{ duration: 0.8, delay: 0.2 }}
+          className="text-xl text-white/80 mb-10 max-w-2xl mx-auto"
+        >
+          Build production-ready web apps in minutes. MINDI writes, tests,
+          and deploys your code while you focus on what matters.
+        </motion.p>
+        <motion.div
+          initial={{ opacity: 0, scale: 0.9 }}
+          animate={{ opacity: 1, scale: 1 }}
+          transition={{ duration: 0.5, delay: 0.4 }}
+        >
+          <a
+            href="/get-started"
+            className="inline-block px-8 py-4 text-lg font-semibold text-indigo-900 bg-white rounded-full shadow-lg hover:shadow-xl hover:scale-105 transition-all duration-300"
+          >
+            Get Started Free →
+          </a>
+        </motion.div>
+      </div>
+    </section>
+  );
+}
+<|code_end|>
+
+<|critique_start|>
+Visual Analysis:
+- ✅ Gradient background: Strong indigo→purple→pink gradient creates visual depth
+- ✅ Animated headline: Framer Motion fade-up provides smooth entrance
+- ✅ CTA button: High contrast white on gradient, rounded-full shape draws attention
+- ⚠️ Accessibility: Need to verify color contrast ratios for text on gradient
+- ⚠️ Mobile: Font sizes may need adjustment below md breakpoint
+- ✅ Performance: Motion animations are hardware-accelerated transforms
+<|critique_end|>
+
+<|suggest_start|>
+Improvements for next iteration:
+1. Add aria-label to the CTA link for screen readers
+2. Consider adding a secondary CTA (e.g., "Watch Demo") for users not ready to commit
+3. Add a subtle particle or floating shape animation in the background
+4. Include social proof (e.g., "Trusted by 10,000+ developers") below the CTA
+<|suggest_end|>"""
+
+# ── Build full conversation string ───────────────────────────────────
+conversation = f"""<|mindi_start|>
+<|im_start|>system
+{system_message}<|im_end|>
+<|im_start|>user
+{user_message}<|im_end|>
+<|im_start|>assistant
+{assistant_message}<|im_end|>
+<|mindi_end|>"""
+
+print("\n" + "─" * 70)
+print("FULL MINDI CONVERSATION (raw text)")
+print("─" * 70)
+print(conversation)
+print("─" * 70)
+
+# ── Tokenize the full conversation ───────────────────────────────────
+print("\n📊 TOKENIZATION RESULTS")
+print("─" * 70)
+
+token_ids = tokenizer.encode(conversation, add_special_tokens=False)
+print(f"   Total tokens: {len(token_ids):,}")
+
+decoded = tokenizer.decode(token_ids)
+print(f"   Decoded length (chars): {len(decoded):,}")
+
+# ── Round-trip verification ──────────────────────────────────────────
+print("\n🔄 ROUND-TRIP VERIFICATION")
+print("─" * 70)
+
+if decoded.strip() == conversation.strip():
+    print("   ✅ PERFECT MATCH — decoded text matches original conversation exactly")
+    round_trip_pass = True
+else:
+    # Show differences for debugging
+    print("   ❌ MISMATCH detected!")
+    orig_lines = conversation.strip().splitlines()
+    dec_lines = decoded.strip().splitlines()
+    print(f"   Original lines: {len(orig_lines)}, Decoded lines: {len(dec_lines)}")
+    for i, (o, d) in enumerate(zip(orig_lines, dec_lines)):
+        if o != d:
+            print(f"   Line {i}: DIFF")
+            print(f"     Original: {repr(o[:100])}")
+            print(f"     Decoded:  {repr(d[:100])}")
+    round_trip_pass = False
+
+# ── Verify all MINDI special tokens are preserved as single tokens ───
+print("\n🔍 SPECIAL TOKEN PRESERVATION")
+print("─" * 70)
+
+all_passed = True
+for token_str in MINDI_SPECIAL_TOKENS:
+    token_id = tokenizer.convert_tokens_to_ids(token_str)
+    # Check the token encodes to a single ID
+    encoded = tokenizer.encode(token_str, add_special_tokens=False)
+
+    if len(encoded) == 1 and encoded[0] == token_id:
+        status = "✅"
+    else:
+        status = "❌"
+        all_passed = False
+
+    # Check this token_id appears in the full conversation encoding
+    count_in_conv = token_ids.count(token_id)
+    print(f"   {status} {token_str:<25} ID={token_id:<8} single_token=True  occurrences_in_conv={count_in_conv}")
+
+# ── Qwen chat template tokens ──────────────────────────────────────
+print("\n🔍 QWEN CHAT TEMPLATE TOKENS")
+print("─" * 70)
+
+qwen_tokens = ["<|im_start|>", "<|im_end|>"]
+for token_str in qwen_tokens:
+    token_id = tokenizer.convert_tokens_to_ids(token_str)
+    encoded = tokenizer.encode(token_str, add_special_tokens=False)
+    count_in_conv = token_ids.count(token_id)
+    status = "✅" if len(encoded) == 1 else "❌"
+    print(f"   {status} {token_str:<25} ID={token_id:<8} occurrences_in_conv={count_in_conv}")
+
+# ── Token distribution analysis ──────────────────────────────────────
+print("\n📈 TOKEN DISTRIBUTION")
+print("─" * 70)
+
+# Count special vs regular tokens
+special_ids = set()
+for t in MINDI_SPECIAL_TOKENS + qwen_tokens:
+    tid = tokenizer.convert_tokens_to_ids(t)
+    special_ids.add(tid)
+
+special_count = sum(1 for tid in token_ids if tid in special_ids)
+regular_count = len(token_ids) - special_count
+
+print(f"   Special tokens: {special_count}")
+print(f"   Regular tokens: {regular_count}")
+print(f"   Total tokens:   {len(token_ids):,}")
+print(f"   Special ratio:  {special_count / len(token_ids) * 100:.1f}%")
+
+# ── Estimate tokens per message ──────────────────────────────────────
+print("\n📏 TOKENS PER MESSAGE")
+print("─" * 70)
+
+sys_tokens = tokenizer.encode(system_message, add_special_tokens=False)
+usr_tokens = tokenizer.encode(user_message, add_special_tokens=False)
+ast_tokens = tokenizer.encode(assistant_message, add_special_tokens=False)
+
+print(f"   System message:    {len(sys_tokens):>5} tokens ({len(system_message):>5} chars)")
+print(f"   User message:      {len(usr_tokens):>5} tokens ({len(user_message):>5} chars)")
+print(f"   Assistant message:  {len(ast_tokens):>5} tokens ({len(assistant_message):>5} chars)")
+print(f"   Wrapper overhead:  ~{len(token_ids) - len(sys_tokens) - len(usr_tokens) - len(ast_tokens):>5} tokens (mindi_start/end, im_start/end, roles)")
+
+# ── Context window fit check ─────────────────────────────────────────
+print("\n📐 CONTEXT WINDOW FIT")
+print("─" * 70)
+context_length = 32768
+print(f"   Context window:   {context_length:>6} tokens")
+print(f"   This conversation: {len(token_ids):>6} tokens")
+print(f"   Remaining:        {context_length - len(token_ids):>6} tokens ({(context_length - len(token_ids)) / context_length * 100:.1f}%)")
+print(f"   ✅ Fits easily within context window")
+
+# ── Final verdict ────────────────────────────────────────────────────
+print("\n" + "=" * 70)
+if round_trip_pass and all_passed:
+    print("✅ STEP 5 PASSED: MINDI conversation format works perfectly!")
+    print("   • Full conversation tokenizes and decodes with perfect fidelity")
+    print("   • All 22 MINDI special tokens preserved as single tokens")
+    print("   • Qwen chat template tokens (im_start/im_end) working correctly")
+    print(f"   • Total: {len(token_ids):,} tokens for a realistic conversation")
+else:
+    print("❌ STEP 5 FAILED — issues detected above")
+print("=" * 70)

src/tokenizer/tokenizer.py

@@ -1,8 +1,9 @@
 """
 MINDI 1.5 Vision-Coder — Tokenizer Wrapper
 
-Wraps the base model tokenizer with MINDI-specific special tokens
-and encoding utilities for code generation tasks.
+Wraps the MINDI tokenizer (Qwen2.5-Coder base + 22 special tokens)
+with encoding utilities for code generation, conversation formatting,
+and special-token-aware operations.
 """
 
 from __future__ import annotations
@@ -13,63 +14,120 @@ from typing import Optional
 from transformers import AutoTokenizer, PreTrainedTokenizerFast
 
 
-# Special tokens for MINDI's structured output format
-SPECIAL_TOKENS: dict[str, str] = {
+# All 22 MINDI special tokens (pairs)
+MINDI_SPECIAL_TOKENS: dict[str, str] = {
+    "mindi_start": "<|mindi_start|>",
+    "mindi_end": "<|mindi_end|>",
     "code_start": "<|code_start|>",
     "code_end": "<|code_end|>",
-    "file_start": "<|file_start|>",
-    "file_end": "<|file_end|>",
+    "vision_start": "<|vision_start|>",
+    "vision_end": "<|vision_end|>",
     "critique_start": "<|critique_start|>",
     "critique_end": "<|critique_end|>",
+    "suggest_start": "<|suggest_start|>",
+    "suggest_end": "<|suggest_end|>",
+    "think_start": "<|think_start|>",
+    "think_end": "<|think_end|>",
+    "file_start": "<|file_start|>",
+    "file_end": "<|file_end|>",
     "search_start": "<|search_start|>",
     "search_end": "<|search_end|>",
+    "sandbox_start": "<|sandbox_start|>",
+    "sandbox_end": "<|sandbox_end|>",
+    "error_start": "<|error_start|>",
+    "error_end": "<|error_end|>",
     "fix_start": "<|fix_start|>",
     "fix_end": "<|fix_end|>",
 }
 
+# Default tokenizer path (pre-built with special tokens already added)
+DEFAULT_TOKENIZER_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "tokenizer" / "mindi_tokenizer"
+
 
 class MindiTokenizer:
-    """Tokenizer wrapper with MINDI-specific special tokens."""
+    """Tokenizer wrapper with MINDI-specific special tokens and conversation formatting."""
 
-    def __init__(self, model_name: str, cache_dir: Optional[Path] = None) -> None:
-        self.model_name = model_name
-        self.cache_dir = cache_dir or Path("./data/tokenizer")
-        self.cache_dir.mkdir(parents=True, exist_ok=True)
+    def __init__(
+        self,
+        tokenizer_path: Optional[Path] = None,
+        max_length: int = 32768,
+    ) -> None:
+        self.tokenizer_path = tokenizer_path or DEFAULT_TOKENIZER_PATH
+        self.max_length = max_length
 
         self.tokenizer: PreTrainedTokenizerFast = AutoTokenizer.from_pretrained(
-            model_name,
-            cache_dir=str(self.cache_dir),
+            str(self.tokenizer_path),
             trust_remote_code=True,
         )
-        self._add_special_tokens()
 
-    def _add_special_tokens(self) -> None:
-        """Register MINDI special tokens with the tokenizer."""
-        new_tokens = list(SPECIAL_TOKENS.values())
-        num_added = self.tokenizer.add_special_tokens(
-            {"additional_special_tokens": new_tokens}
-        )
-        if num_added > 0:
-            print(f"[MindiTokenizer] Added {num_added} special tokens")
+        # Cache special token IDs for fast lookup
+        self._special_token_ids: dict[str, int] = {
+            name: self.tokenizer.convert_tokens_to_ids(token)
+            for name, token in MINDI_SPECIAL_TOKENS.items()
+        }
 
-    @property
-    def vocab_size(self) -> int:
-        """Return the full vocabulary size including special tokens."""
-        return len(self.tokenizer)
+    # ── Core API ──────────────────────────────────────────────────────
 
-    def encode(self, text: str, max_length: int = 8192) -> list[int]:
-        """Encode text to token IDs with truncation."""
+    def encode(
+        self,
+        text: str,
+        add_special_tokens: bool = False,
+        max_length: Optional[int] = None,
+    ) -> list[int]:
         return self.tokenizer.encode(
-            text, max_length=max_length, truncation=True
+            text,
+            add_special_tokens=add_special_tokens,
+            max_length=max_length or self.max_length,
+            truncation=True,
         )
 
-    def decode(self, token_ids: list[int]) -> str:
-        """Decode token IDs back to text."""
-        return self.tokenizer.decode(token_ids, skip_special_tokens=False)
+    def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str:
+        return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
+
+    def encode_conversation(
+        self,
+        messages: list[dict[str, str]],
+        wrap_mindi: bool = True,
+    ) -> list[int]:
+        """Encode a list of messages [{"role": ..., "content": ...}] into token IDs.
+
+        Uses Qwen's im_start/im_end chat template with optional mindi_start/end wrapper.
+        """
+        parts: list[str] = []
+        if wrap_mindi:
+            parts.append("<|mindi_start|>\n")
+
+        for msg in messages:
+            role = msg["role"]
+            content = msg["content"]
+            parts.append(f"<|im_start|>{role}\n{content}<|im_end|>\n")
+
+        if wrap_mindi:
+            parts.append("<|mindi_end|>")
+
+        full_text = "".join(parts)
+        return self.encode(full_text, add_special_tokens=False)
+
+    def encode_with_special_tokens(self, text: str) -> list[int]:
+        """Encode text that contains MINDI special tokens, preserving them as single tokens."""
+        return self.encode(text, add_special_tokens=False)
+
+    # ── Introspection ─────────────────────────────────────────────────
+
+    def get_vocab_size(self) -> int:
+        return len(self.tokenizer)
+
+    def get_special_token_ids(self) -> dict[str, int]:
+        return dict(self._special_token_ids)
+
+    def get_special_token_id(self, name: str) -> int:
+        return self._special_token_ids[name]
+
+    # ── Persistence ───────────────────────────────────────────────────
 
     def save(self, output_dir: Optional[Path] = None) -> Path:
-        """Save the tokenizer to disk."""
-        save_path = output_dir or self.cache_dir / "mindi_tokenizer"
+        save_path = output_dir or self.tokenizer_path
+        save_path = Path(save_path)
         save_path.mkdir(parents=True, exist_ok=True)
         self.tokenizer.save_pretrained(str(save_path))
         return save_path