ufo_cuneiform_vocab_mapper.py

import os
import struct
import numpy as np
from transformers import AutoTokenizer

TOKENIZER_DIR = "j:/Language-U/Language-U-V2/qwen-3.5-0.8b-local"
OUTPUT_BIN = "j:/Language-U/qwen_vocab_cuneiform.bin"

def classify_token(token_str):
    # Normalize token string (replace special character G/Ġ representing space)
    s = token_str.replace("Ġ", " ").replace(" ", "").lower()
    
    # 1. DOMAIN & SUBDOMAIN
    domain = 0
    subdomain = 0
    
    # Domain 1: Hardware & LoRA Networks
    hw_net_kw = ['gpio', 'lora', 'chirp', 'reset', 'pin', 'spi', 'sx1302', 'pi4', 'pi5', 
                 'concentrator', 'antenna', 'frequency', 'mhz', 'dbm', 'packet', 'transmit', 
                 'tx', 'rx', 'sf7', 'fec', 'parity', 'duty', 'dwell', 'mac', 'gateway', 'beacon']
    # Domain 2: Mathematics, Logic & Information Theory
    math_kw = ['svd', 'dct', 'quant', 'math', 'entropy', 'shannon', 'gradient', 'atom', 
               'ortho', 'eigen', 'projection', 'reconstruct', 'rank', 'matrix', 'vector', 
               'basis', 'decompo', 'smooth', 'laplace', 'probability', 'logits', 'prior', 
               'bypass', 'q_proj', 'v_proj', 'k_proj', 'o_proj']
    # Domain 3: Dialogue, Persona & Authors
    dialogue_kw = ['zymatica', 'collective', 'dialogue', 'persona', 'chat', 'assistant', 
                   'speak', 'talk', 'bot', 'agent', 'she', 'astronaut', 'devsone', 'bouldiez', 
                   'partner', 'art']
    # Domain 4: Software, Systems & Runtimes
    sw_kw = ['rust', 'cpp', 'go', 'python', 'swift', 'java', 'typescript', 'compile', 
             'code', 'exec', 'run', 'lib', 'class', 'struct', 'header', 'import', 'from', 
             'package', 'build', 'cmake', 'cargo']

    if any(k in s for k in hw_net_kw):
        domain = 1
        if 'lora' in s or 'chirp' in s or 'fec' in s:
            subdomain = 1
        elif 'gpio' in s or 'pin' in s or 'reset' in s:
            subdomain = 2
        elif 'packet' in s or 'beacon' in s:
            subdomain = 3
    elif any(k in s for k in math_kw):
        domain = 2
        if 'svd' in s or 'matrix' in s or 'projection' in s:
            subdomain = 1
        elif 'entropy' in s or 'shannon' in s or 'bypass' in s:
            subdomain = 2
        elif 'logits' in s or 'prior' in s or 'smooth' in s:
            subdomain = 3
    elif any(k in s for k in dialogue_kw):
        domain = 3
        if 'zymatica' in s or 'collective' in s:
            subdomain = 1
        elif 'persona' in s or 'dialogue' in s or 'speak' in s:
            subdomain = 2
    elif any(k in s for k in sw_kw):
        domain = 4
        if 'rust' in s or 'go' in s or 'cpp' in s:
            subdomain = 1
        elif 'python' in s or 'typescript' in s or 'java' in s:
            subdomain = 2

    # 2. OPERATION (Actions)
    # Map key verbs to operation IDs (1 to 15)
    operations = [
        'reset', 'clear', 'toggle', 'write', 'read', 'set', 'get', 
        'encode', 'decode', 'compress', 'decompress', 'train', 'heal', 
        'eval', 'test', 'load', 'save', 'grow', 'shrink', 'bypass'
    ]
    operation = 0
    for i, op in enumerate(operations, 1):
        if op in s:
            operation = i % 16
            break

    # 3. MODALITY (Data layouts / formats)
    modalities = [
        'bin', 'zlib', 'json', 'capsule', 'genesis', 'llm', 'file', 
        'packet', 'byte', 'bit', 'char', 'string', 'token', 'wave', 'hal'
    ]
    modality = 0
    for i, mod in enumerate(modalities, 1):
        if mod in s:
            modality = i % 16
            break

    # 4. DEPTH (Complexity scale)
    depth = 0
    if domain == 1:
        if 'seed' in s or 'genesis' in s:
            depth = 8
        elif 'dct' in s:
            depth = 4
        elif 'atom' in s:
            depth = 6
        elif 'qa' in s or 'facts' in s:
            depth = 1
    else:
        depth = len(s) % 16

    # 5. POLARITY (Outcome states)
    polarity = 0
    pos_states = ['ack', 'success', 'ok', 'pass', 'valid', 'correct', 'true']
    neg_states = ['nack', 'fail', 'error', 'wrong', 'miss', 'warn', 'oom', 'crash', 'abort', 'false']
    if any(k in s for k in pos_states):
        polarity = 1
    elif any(k in s for k in neg_states):
        polarity = 2

    return domain, subdomain, operation, modality, depth, polarity

def main():
    print(f"Loading tokenizer from: {TOKENIZER_DIR}")
    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
    vocab = tokenizer.get_vocab()
    vocab_size = len(vocab)
    print(f"Total vocabulary size: {vocab_size}")

    # Build the binary map in token ID order
    # Each record is 3 bytes: R_C, R_F, R_A
    packed_data = bytearray(vocab_size * 3)

    domain_counts = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}
    op_counts = 0
    polarity_counts = {0: 0, 1: 0, 2: 0}

    for token_str, token_id in vocab.items():
        if token_id >= vocab_size:
            continue
        
        domain, subdomain, operation, modality, depth, polarity = classify_token(token_str)

        # Pack into radicals
        rc = (domain << 4) | (subdomain & 0xF)
        rf = (operation << 4) | (modality & 0xF)
        ra = (depth << 4) | (polarity & 0xF)

        packed_data[token_id * 3]     = rc
        packed_data[token_id * 3 + 1] = rf
        packed_data[token_id * 3 + 2] = ra

        # Metrics collection
        domain_counts[domain] = domain_counts.get(domain, 0) + 1
        if operation > 0:
            op_counts += 1
        polarity_counts[polarity] = polarity_counts.get(polarity, 0) + 1

    # Write to output file
    with open(OUTPUT_BIN, "wb") as f:
        f.write(packed_data)

    print(f"\n[+] Successfully generated binary map at: {OUTPUT_BIN}")
    print(f"    Total tokens written: {vocab_size}")
    print("\n--- Mapping Classification Metrics ---")
    print(f"    Domain 0 (General English): {domain_counts[0]:,} tokens ({domain_counts[0]/vocab_size*100:.2f}%)")
    print(f"    Domain 1 (Hardware & LoRA): {domain_counts[1]:,} tokens ({domain_counts[1]/vocab_size*100:.2f}%)")
    print(f"    Domain 2 (Math & Info Theory): {domain_counts[2]:,} tokens ({domain_counts[2]/vocab_size*100:.2f}%)")
    print(f"    Domain 3 (Dialogue & Persona): {domain_counts[3]:,} tokens ({domain_counts[3]/vocab_size*100:.2f}%)")
    print(f"    Domain 4 (Software & Systems): {domain_counts[4]:,} tokens ({domain_counts[4]/vocab_size*100:.2f}%)")
    print(f"    Active Operations Detected: {op_counts:,} tokens")
    print(f"    Neutral Polarity (0):       {polarity_counts[0]:,} tokens")
    print(f"    Positive Polarity (1 - ACK): {polarity_counts[1]:,} tokens")
    print(f"    Negative Polarity (2 - ERR): {polarity_counts[2]:,} tokens")

if __name__ == "__main__":
    main()