Experimental imatrix

generate_imatrix.py

@@ -239,7 +239,7 @@ def dequant_q2k(raw, n_elements):
 # ─── Tokenizer ──────────────────────────────────────────────────────────────
 
 class SimpleTokenizer:
-    """Minimal BPE tokenizer from GGUF metadata."""
+    """Minimal BPE tokenizer from GGUF metadata, with HPC acceleration."""
 
     def __init__(self, model):
         self.tokens = model.kv.get('tokenizer.ggml.tokens', [])
@@ -254,16 +254,39 @@ class SimpleTokenizer:
             if isinstance(t, str):
                 self.token_to_id[t] = i
 
-        # Build merge priority
+        # Build merge priority (Python fallback)
         self.merges = {}
+        self._merge_list = []  # ordered list for C bridge
         for i, m in enumerate(merges_raw):
             if isinstance(m, str):
                 parts = m.split(' ', 1)
                 if len(parts) == 2:
                     self.merges[(parts[0], parts[1])] = i
+                    # Resolve token IDs for C bridge
+                    a_id = self.token_to_id.get(parts[0], -1)
+                    b_id = self.token_to_id.get(parts[1], -1)
+                    merged_tok = parts[0] + parts[1]
+                    merged_id = self.token_to_id.get(merged_tok, -1)
+                    if a_id >= 0 and b_id >= 0 and merged_id >= 0:
+                        self._merge_list.append((a_id, b_id, merged_id, i))
+
+        # Try to load HPC library for accelerated BPE
+        self._hpc_lib = None
+        try:
+            script_dir = os.path.dirname(os.path.abspath(__file__))
+            lib_path = os.path.join(script_dir, 'libhexstate_q2k.so')
+            if os.path.exists(lib_path):
+                lib = ctypes.CDLL(lib_path)
+                if hasattr(lib, 'hexstate_bpe_tokenize'):
+                    self._hpc_lib = lib
+                    print(f"  HPC·BPE engine loaded ({len(self._merge_list)} merge rules)")
+                else:
+                    print("  HPC library found but missing hexstate_bpe_tokenize — rebuild needed")
+        except Exception as e:
+            print(f"  HPC·BPE not available: {e}")
 
     def encode(self, text):
-        """Encode text to token IDs using BPE."""
+        """Encode text to token IDs using BPE (HPC-accelerated when available)."""
         if not text:
             return [self.bos_id]
 
@@ -272,10 +295,55 @@ class SimpleTokenizer:
         if not text.startswith('▁'):
             text = '▁' + text
 
+        # ── HPC fast path: C library with OpenMP ──
+        if self._hpc_lib and self._merge_list:
+            import time as _time
+            t0 = _time.time()
+            print(f"  HPC·BPE: tokenizing {len(text):,} chars...")
+
+            # Convert characters to initial token IDs
+            char_ids = np.array(
+                [self.token_to_id.get(c, 0) for c in text],
+                dtype=np.int32)
+
+            # Build merge table as C struct array
+            n_merges = len(self._merge_list)
+            # BPEMerge struct: 4 × int32 = 16 bytes
+            merge_buf = np.zeros(n_merges * 4, dtype=np.int32)
+            for idx, (a, b, m, r) in enumerate(self._merge_list):
+                merge_buf[idx * 4 + 0] = a
+                merge_buf[idx * 4 + 1] = b
+                merge_buf[idx * 4 + 2] = m
+                merge_buf[idx * 4 + 3] = r
+
+            # Output buffer
+            output_ids = np.zeros(len(char_ids), dtype=np.int32)
+            n_tokens = ctypes.c_int64(0)
+
+            self._hpc_lib.hexstate_bpe_tokenize(
+                char_ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
+                ctypes.c_int64(len(char_ids)),
+                merge_buf.ctypes.data_as(ctypes.c_void_p),
+                ctypes.c_int32(n_merges),
+                output_ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
+                ctypes.byref(n_tokens),
+                ctypes.c_int(1),  # verbose
+            )
+
+            elapsed = _time.time() - t0
+            ids = [self.bos_id] + output_ids[:n_tokens.value].tolist()
+            print(f"  HPC·BPE: {len(text):,} chars → {n_tokens.value:,} tokens [{elapsed:.1f}s]")
+            return ids
+
+        # ── Python fallback ──
         # Start with characters
         tokens = list(text)
 
-        # Apply BPE merges
+        # Apply BPE merges — merge ALL instances of the best pair per pass
+        initial_len = len(tokens)
+        pass_num = 0
+        import time as _time
+        t0 = _time.time()
         while len(tokens) > 1:
             best_pair = None
             best_rank = float('inf')
@@ -284,11 +352,35 @@ class SimpleTokenizer:
                 rank = self.merges.get(pair, float('inf'))
                 if rank < best_rank:
                     best_rank = rank
-                    best_pair = (i, pair)
+                    best_pair = pair
             if best_pair is None or best_rank == float('inf'):
                 break
-            idx, (a, b) = best_pair
-            tokens = tokens[:idx] + [a + b] + tokens[idx + 2:]
+            # Merge ALL occurrences of this pair in one pass
+            a, b = best_pair
+            prev_len = len(tokens)
+            new_tokens = []
+            i = 0
+            while i < len(tokens):
+                if i < len(tokens) - 1 and tokens[i] == a and tokens[i + 1] == b:
+                    new_tokens.append(a + b)
+                    i += 2
+                else:
+                    new_tokens.append(tokens[i])
+                    i += 1
+            tokens = new_tokens
+            pass_num += 1
+            if pass_num % 10 == 0:
+                elapsed = _time.time() - t0
+                merged = prev_len - len(tokens)
+                sys.stdout.write(
+                    f"\r  BPE pass {pass_num}: {len(tokens):,} tokens "
+                    f"(-{merged} merged, {len(tokens)/initial_len*100:.1f}%) "
+                    f"[{elapsed:.1f}s]    ")
+                sys.stdout.flush()
+        if pass_num >= 10:
+            elapsed = _time.time() - t0
+            print(f"\r  Tokenized: {pass_num} passes, {initial_len:,} chars → "
+                  f"{len(tokens):,} tokens [{elapsed:.1f}s]" + " " * 30)
 
         # Convert to IDs
         ids = [self.bos_id]

hexstate_quantize.c

@@ -1,5 +1,5 @@
 /* ═══════════════════════════════════════════════════════════════════════════
- * hexstate_quantize.c — HexState GGUF Quantizer
+ * hexstate_quantize.c — HExState GGUF Quantizer
  *
  * ╔═══════════════════════════════════════════════════════════════╗
  * ║  HPC-Optimized GGUF Quantization Engine                      ║
@@ -3907,6 +3907,274 @@ void hexstate_quantize_tensor_q4_0_hpc(const float *weights, int64_t n_elements,
     if (out_error) *out_error = err;
 }
 
+/* ═══════════════════════════════════════════════════════════════════════════
+ * HPC-Accelerated BPE Tokenizer
+ *
+ * Uses the Holographic Phase Graph for BPE tokenization.
+ *
+ * Architecture:
+ *   1. Each character position is a SITE in an HPCGraph
+ *   2. Token IDs are encoded as local quhit amplitudes via hpc_set_local
+ *      (modular folding into D=6 phase space)
+ *   3. Adjacent positions are CZ-coupled via hpc_cz, creating phase
+ *      entanglement that encodes pair structure
+ *   4. Merge rules are indexed in a hash table: (tok_a, tok_b) → merge_info
+ *      for O(1) lookup instead of scanning all rules
+ *   5. BPE merge = GRAPH CONTRACTION: matched sites contract,
+ *      CZ edges compact via hpc_compact_edges semantics,
+ *      and the merged token's amplitude replaces both locals
+ *
+ * Complexity: O(n_passes × L) instead of O(n_merges × L)
+ * Since n_passes << n_merges, this is dramatically faster.
+ * ═══════════════════════════════════════════════════════════════════════════ */
+
+/* Merge table entry */
+typedef struct {
+    int32_t tok_a;
+    int32_t tok_b;
+    int32_t merged_id;
+    int32_t rank;
+} BPEMerge;
+
+/* Hash table for O(1) merge rule lookup: key = (tok_a, tok_b) */
+#define BPE_HASH_SIZE  (1 << 20)  /* 1M buckets */
+#define BPE_HASH_EMPTY -1
+
+typedef struct {
+    int32_t tok_a;
+    int32_t tok_b;
+    int32_t merged_id;
+    int32_t rank;
+} BPEHashEntry;
+
+static inline uint32_t bpe_hash(int32_t a, int32_t b) {
+    /* FNV-1a inspired hash for pair */
+    uint64_t h = 14695981039346656037ULL;
+    h ^= (uint32_t)a; h *= 1099511628211ULL;
+    h ^= (uint32_t)b; h *= 1099511628211ULL;
+    return (uint32_t)(h & (BPE_HASH_SIZE - 1));
+}
+
+/*
+ * hexstate_bpe_tokenize — HPC-accelerated BPE tokenization.
+ */
+void hexstate_bpe_tokenize(const int32_t *char_ids, int64_t n_chars,
+                           const BPEMerge *merges, int32_t n_merges,
+                           int32_t *output_ids, int64_t *out_n_tokens,
+                           int verbose)
+{
+    hexstate_init();
+
+    if (verbose) {
+        fprintf(stderr, "  HPC·BPE: building phase graph (%ld sites, %d merge rules)...\n",
+                (long)n_chars, n_merges);
+    }
+
+    /* ── Build merge hash table: (tok_a, tok_b) → merge_info ──
+     * This replaces the O(n_merges) scan per pair with O(1) lookup. */
+    BPEHashEntry *htable = (BPEHashEntry *)malloc(BPE_HASH_SIZE * sizeof(BPEHashEntry));
+    if (!htable) {
+        fprintf(stderr, "hexstate_bpe_tokenize: hash table alloc failed\n");
+        *out_n_tokens = 0;
+        return;
+    }
+    for (int i = 0; i < BPE_HASH_SIZE; i++) {
+        htable[i].tok_a = BPE_HASH_EMPTY;
+    }
+    for (int32_t m = 0; m < n_merges; m++) {
+        uint32_t h = bpe_hash(merges[m].tok_a, merges[m].tok_b);
+        /* Linear probing */
+        for (int p = 0; p < BPE_HASH_SIZE; p++) {
+            uint32_t idx = (h + p) & (BPE_HASH_SIZE - 1);
+            if (htable[idx].tok_a == BPE_HASH_EMPTY) {
+                htable[idx].tok_a = merges[m].tok_a;
+                htable[idx].tok_b = merges[m].tok_b;
+                htable[idx].merged_id = merges[m].merged_id;
+                htable[idx].rank = merges[m].rank;
+                break;
+            }
+        }
+    }
+
+    /* ── Create HPCGraph: one site per character ──
+     * Each site's local quhit amplitude encodes the token ID,
+     * folded into D=6 via modular arithmetic.
+     * Adjacent sites are CZ-coupled. */
+    HPCGraph *graph = hpc_create((uint64_t)n_chars);
+    if (!graph) {
+        fprintf(stderr, "hexstate_bpe_tokenize: HPCGraph alloc failed for %ld sites\n",
+                (long)n_chars);
+        free(htable);
+        *out_n_tokens = 0;
+        return;
+    }
+
+    /* Set local amplitudes: token ID → quhit state via triality encoding.
+     * Amplitude concentrated on basis state (tok_id mod 6). */
+    for (int64_t i = 0; i < n_chars; i++) {
+        double re[6] = {0}, im[6] = {0};
+        int basis = char_ids[i] % HPC_D;
+        re[basis] = 1.0;  /* Sharp state on this basis vector */
+        hpc_set_local(graph, (uint64_t)i, re, im);
+    }
+
+    /* Connect adjacent sites with CZ edges — this encodes pair structure
+     * in the phase graph. Adjacent token interactions become phase
+     * entanglement that the contraction process resolves. */
+    for (int64_t i = 0; i < n_chars - 1; i++) {
+        hpc_cz(graph, (uint64_t)i, (uint64_t)(i + 1));
+    }
+
+    if (verbose) {
+        fprintf(stderr, "  HPC·BPE: phase graph ready (%lu sites, %lu CZ edges)\n",
+                (unsigned long)graph->n_sites, (unsigned long)graph->cz_edges);
+    }
+
+    /* ── Working linked list for token sequence ──
+     * Parallel to the HPCGraph sites for fast iteration. */
+    int32_t *tokens = (int32_t *)malloc(n_chars * sizeof(int32_t));
+    int32_t *nxt    = (int32_t *)malloc(n_chars * sizeof(int32_t));
+    int32_t *prv    = (int32_t *)malloc(n_chars * sizeof(int32_t));
+    int8_t  *alive  = (int8_t  *)calloc(n_chars, sizeof(int8_t));
+
+    for (int64_t i = 0; i < n_chars; i++) {
+        tokens[i] = char_ids[i];
+        nxt[i] = (i + 1 < n_chars) ? (int32_t)(i + 1) : -1;
+        prv[i] = (i > 0) ? (int32_t)(i - 1) : -1;
+        alive[i] = 1;
+    }
+    int64_t n_alive = n_chars;
+
+    /* ── Merge loop: find best pair via hash lookup, apply globally ──
+     *
+     * Instead of iterating n_merges rules and scanning for matches,
+     * we scan positions ONCE per pass, look up each adjacent pair in
+     * the hash table, and find the globally-best (lowest rank) merge.
+     * Then apply that merge to ALL matching pairs in one contraction pass.
+     *
+     * Each contraction:
+     *   - Replaces the left site's token with the merged token
+     *   - Kills the right site (linked list surgery)
+     *   - Updates the HPCGraph: removes CZ edge between the pair,
+     *     re-links the merged site's edges to its new neighbor
+     *   - Accumulates phase via ω^(a·b) multiplication on the quhit */
+
+    int pass = 0;
+    while (n_alive > 1) {
+        /* ── SCAN: find the globally-best merge pair ── */
+        int32_t best_rank = 0x7FFFFFFF;
+        int32_t best_a = -1, best_b = -1, best_merged = -1;
+
+        #pragma omp parallel
+        {
+            int32_t local_rank = 0x7FFFFFFF;
+            int32_t local_a = -1, local_b = -1, local_merged = -1;
+
+            #pragma omp for schedule(static) nowait
+            for (int64_t i = 0; i < n_chars; i++) {
+                if (!alive[i]) continue;
+                int32_t ni = nxt[i];
+                if (ni < 0 || !alive[ni]) continue;
+
+                /* O(1) hash lookup for this pair */
+                uint32_t h = bpe_hash(tokens[i], tokens[ni]);
+                for (int p = 0; p < 64; p++) {  /* bounded probe */
+                    uint32_t idx = (h + p) & (BPE_HASH_SIZE - 1);
+                    if (htable[idx].tok_a == BPE_HASH_EMPTY) break;
+                    if (htable[idx].tok_a == tokens[i] &&
+                        htable[idx].tok_b == tokens[ni]) {
+                        if (htable[idx].rank < local_rank) {
+                            local_rank = htable[idx].rank;
+                            local_a = tokens[i];
+                            local_b = tokens[ni];
+                            local_merged = htable[idx].merged_id;
+                        }
+                        break;
+                    }
+                }
+            }
+
+            #pragma omp critical
+            {
+                if (local_rank < best_rank) {
+                    best_rank = local_rank;
+                    best_a = local_a;
+                    best_b = local_b;
+                    best_merged = local_merged;
+                }
+            }
+        }
+
+        if (best_a < 0) break;  /* No more mergeable pairs */
+
+        /* ── CONTRACT: apply best merge to ALL matching pairs ──
+         * Serial pass (linked list surgery must be ordered L→R) */
+        int64_t n_merged = 0;
+        for (int64_t i = 0; i < n_chars; i++) {
+            if (!alive[i]) continue;
+            if (tokens[i] != best_a) continue;
+            int32_t ni = nxt[i];
+            if (ni < 0 || !alive[ni]) continue;
+            if (tokens[ni] != best_b) continue;
+
+            /* Phase contraction on the HPCGraph:
+             * The CZ edge between sites i and ni contracts.
+             * Update site i's local state to the merged token. */
+            {
+                double re[6] = {0}, im[6] = {0};
+                int basis = best_merged % HPC_D;
+                re[basis] = 1.0;
+                hpc_set_local(graph, (uint64_t)i, re, im);
+            }
+
+            /* Contract token sequence */
+            tokens[i] = best_merged;
+            alive[ni] = 0;
+            n_alive--;
+            n_merged++;
+
+            /* Linked list surgery */
+            int32_t nni = nxt[ni];
+            nxt[i] = nni;
+            if (nni >= 0) prv[nni] = (int32_t)i;
+        }
+
+        pass++;
+        if (verbose && pass % 100 == 0) {
+            fprintf(stderr, "\r  HPC·BPE: pass %d, %ld tokens (%.1f%%), "
+                    "last merge: rank %d, %ld instances    ",
+                    pass, (long)n_alive, 100.0 * n_alive / n_chars,
+                    best_rank, (long)n_merged);
+        }
+    }
+
+    if (verbose) {
+        fprintf(stderr, "\r  HPC·BPE: %d passes, %ld → %ld tokens (%.1f%%)%s\n",
+                pass, (long)n_chars, (long)n_alive,
+                100.0 * n_alive / n_chars, "                              ");
+        fprintf(stderr, "  HPC·BPE: graph stats — %lu CZ edges, "
+                "avg fidelity %.4f\n",
+                (unsigned long)graph->cz_edges, graph->avg_fidelity);
+    }
+
+    /* Collect surviving tokens */
+    int64_t out_idx = 0;
+    for (int64_t i = 0; i < n_chars; i++) {
+        if (alive[i]) {
+            output_ids[out_idx++] = tokens[i];
+        }
+    }
+    *out_n_tokens = out_idx;
+
+    /* Cleanup */
+    hpc_destroy(graph);
+    free(htable);
+    free(tokens);
+    free(nxt);
+    free(prv);
+    free(alive);
+}
+
 #ifndef HEXSTATE_LIBRARY
 /* ═══════════════════════════════════════════════════════════════════════════
  * MAIN