Delete pattern_constructor_h.cpp

pattern_constructor_h.cpp

@@ -1,1362 +0,0 @@
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <cstdint>
-#include <cmath>
-#include <deque>
-#include <cstdio>
-#include <fstream>
-#include <iostream>
-#include <limits>
-#include <memory>
-#include <numeric>
-#include <stdexcept>
-#include <string>
-#include <string_view>
-#include <tuple>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-#ifdef _OPENMP
-#include <omp.h>
-#else
-static int omp_get_max_threads(){ return 1; }
-static int omp_get_thread_num(){ return 0; }
-#endif
-
-// ============================================================================
-// Configuration
-// ============================================================================
-struct Config {
-    std::size_t max_response_tokens = 32;
-    std::size_t top_k = 3;
-    std::size_t chunk_size = 8192;
-    bool show_retrieval = false;
-    bool chat_mode = false;
-};
-static Config config;
-
-// ============================================================================
-// Character helpers
-// ============================================================================
-static inline unsigned char uchar(char c){ return static_cast<unsigned char>(c); }
-static inline bool is_alpha(char c){ return std::isalpha(uchar(c)) != 0; }
-static inline bool is_digit(char c){ return std::isdigit(uchar(c)) != 0; }
-static inline bool is_alnum(char c){ return std::isalnum(uchar(c)) != 0; }
-static inline bool is_space(char c){ return std::isspace(uchar(c)) != 0; }
-static inline bool is_lower(char c){ return std::islower(uchar(c)) != 0; }
-static inline bool is_upper(char c){ return std::isupper(uchar(c)) != 0; }
-
-// ============================================================================
-// Deterministic hashes
-// ============================================================================
-static constexpr std::uint64_t FNV_OFFSET = 14695981039346656037ULL;
-static constexpr std::uint64_t FNV_PRIME  = 1099511628211ULL;
-
-static std::uint64_t fnv1a64(std::string_view s){
-    std::uint64_t h = FNV_OFFSET;
-    for(unsigned char c : s){ h ^= c; h *= FNV_PRIME; }
-    return h;
-}
-
-static std::uint64_t mix64(std::uint64_t x){
-    x += 0x9E3779B97F4A7C15ULL;
-    x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL;
-    x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL;
-    return x ^ (x >> 31);
-}
-
-static std::uint64_t combine_hashes(std::uint64_t a,std::uint64_t b){
-    return mix64(a ^ (mix64(b)+0x9E3779B97F4A7C15ULL+(a<<6)+(a>>2)));
-}
-
-// ============================================================================
-// Compact string pool (memory optimization)
-// ============================================================================
-struct StringRef { std::uint32_t offset=0, length=0; };
-
-struct StringPool {
-    std::string bytes;
-    void clear(){ bytes.clear(); }
-    std::uint32_t append(std::string_view s){
-        const std::uint32_t off=static_cast<std::uint32_t>(bytes.size());
-        bytes.append(s);
-        return off;
-    }
-    std::string_view view(std::uint32_t offset,std::uint32_t length) const {
-        if(static_cast<std::size_t>(offset)+length>bytes.size()) return {};
-        return std::string_view(bytes.data()+offset,length);
-    }
-};
-static StringPool string_pool;
-
-// ============================================================================
-// Integer token dictionary
-// ============================================================================
-struct TokenHash {
-    std::size_t operator()(std::string_view s) const noexcept {
-        return static_cast<std::size_t>(fnv1a64(s));
-    }
-};
-
-struct TokenDictionary {
-    std::deque<std::string> text;
-    std::unordered_map<std::string_view,std::uint32_t,TokenHash> to_id;
-
-    void clear(){ text.clear(); to_id.clear(); }
-
-    std::uint32_t intern(std::string_view s){
-        auto it=to_id.find(s);
-        if(it!=to_id.end()) return it->second;
-        const std::uint32_t id=static_cast<std::uint32_t>(text.size());
-        text.emplace_back(s);
-        to_id.emplace(std::string_view(text.back()),id);
-        return id;
-    }
-
-    std::string_view view(std::uint32_t id) const {
-        return id<text.size()?std::string_view(text[id]):std::string_view{};
-    }
-};
-static TokenDictionary token_dict;
-
-// ============================================================================
-// Tokenization
-// ============================================================================
-struct RawToken {
-    std::string_view text;
-    std::string_view space_before;
-    std::size_t start=0,end=0;
-};
-
-static bool is_identifier_char(char c){ return is_alnum(c) || c=='_'; }
-
-static std::vector<RawToken> tokenize_views(std::string_view text){
-    std::vector<RawToken> out;
-    out.reserve(text.size()/4+1);
-    std::size_t i=0,space_begin=0;
-    while(i<text.size()){
-        if(is_space(text[i])){ ++i; continue; }
-        const std::size_t start=i;
-        if(is_identifier_char(text[i])){
-            while(i<text.size() && is_identifier_char(text[i])) ++i;
-        } else {
-            ++i;
-        }
-        out.push_back({text.substr(start,i-start),text.substr(space_begin,start-space_begin),start,i});
-        space_begin=i;
-    }
-    return out;
-}
-
-// ============================================================================
-// Structural classes
-// ============================================================================
-enum class TKind : std::uint8_t { EMPTY, PUNCT, MIXNUM, NUM, MIX, UPP, LOW, OTHER };
-
-enum class ShapeKind : std::uint8_t {
-    EMPTY=0, ALPHA=1, DIGIT=2, ALNUM=3, SPACE=4, PUNCT=5,
-    ALPHA_DIGIT=6, DIGIT_ALPHA=7, MIXED_ALPHA=8, OTHER=9
-};
-
-static TKind token_kind(std::string_view token){
-    if(token.empty()) return TKind::EMPTY;
-    bool alpha=false,digit=false,lower=false,upper=false;
-    for(char c:token){
-        if(is_alpha(c)){ alpha=true; lower|=is_lower(c); upper|=is_upper(c); }
-        else if(is_digit(c)) digit=true;
-    }
-    if(!is_alnum(token.front()) && token.front()!='_') return TKind::PUNCT;
-    if(alpha&&digit) return TKind::MIXNUM;
-    if(digit) return TKind::NUM;
-    if(upper&&lower) return TKind::MIX;
-    if(upper) return TKind::UPP;
-    if(lower) return TKind::LOW;
-    return TKind::OTHER;
-}
-
-static bool variable_candidate(TKind k){
-    return k==TKind::LOW || k==TKind::UPP || k==TKind::MIX || k==TKind::MIXNUM || k==TKind::NUM;
-}
-
-static ShapeKind shape_kind(std::string_view token){
-    if(token.empty()) return ShapeKind::EMPTY;
-    bool alpha=false,digit=false,space=false,punct=false,lower=false,upper=false;
-    for(char c:token){
-        alpha|=is_alpha(c); digit|=is_digit(c); space|=is_space(c); punct|=!is_alnum(c)&&!is_space(c);
-        lower|=is_lower(c); upper|=is_upper(c);
-    }
-    if(punct && !alpha && !digit) return ShapeKind::PUNCT;
-    if(space && !alpha && !digit && !punct) return ShapeKind::SPACE;
-    if(alpha&&digit){
-        bool first_alpha=is_alpha(token.front()), first_digit=is_digit(token.front());
-        return first_alpha ? ShapeKind::ALPHA_DIGIT : (first_digit ? ShapeKind::DIGIT_ALPHA : ShapeKind::ALNUM);
-    }
-    if(alpha){
-        if(lower&&upper) return ShapeKind::MIXED_ALPHA;
-        return ShapeKind::ALPHA;
-    }
-    if(digit) return ShapeKind::DIGIT;
-    return ShapeKind::OTHER;
-}
-
-static std::uint64_t morphology_hash(std::string_view token){
-    std::uint64_t h=FNV_OFFSET;
-    for(char c:token){
-        std::uint8_t k=0;
-        if(is_alpha(c)) k=is_lower(c)?1:2;
-        else if(is_digit(c)) k=3;
-        else if(is_space(c)) k=4;
-        else k=5;
-        h ^= k; h *= FNV_PRIME;
-    }
-    h=combine_hashes(h,static_cast<std::uint64_t>(token.size()));
-    return h;
-}
-
-static std::uint64_t transition_hash(std::string_view token){
-    std::uint64_t h=FNV_OFFSET;
-    std::uint8_t prev=0;
-    bool first=true;
-    for(char c:token){
-        std::uint8_t k=0;
-        if(is_alpha(c)) k=is_lower(c)?1:2;
-        else if(is_digit(c)) k=3;
-        else if(is_space(c)) k=4;
-        else k=5;
-        if(first){ h=combine_hashes(h,k); first=false; }
-        else { h=combine_hashes(h,(static_cast<std::uint64_t>(prev)<<8)|k); }
-        prev=k;
-    }
-    return combine_hashes(h,token.size());
-}
-
-// ============================================================================
-// Profiles
-// ============================================================================
-struct Profile {
-    float distinct_ratio=0.0f;
-    float alpha_ratio=0.0f;
-    float num_ratio=0.0f;
-    float punct_ratio=0.0f;
-    float space_ratio=0.0f;
-};
-
-static Profile extract_profile(std::string_view text){
-    std::array<std::uint8_t,256> seen{};
-    std::size_t distinct=0,alpha=0,num=0,punct=0,space=0;
-    for(char c:text){
-        const auto u=uchar(c);
-        if(!seen[u]){seen[u]=1;++distinct;}
-        if(is_alpha(c)) ++alpha;
-        else if(is_digit(c)) ++num;
-        else if(is_space(c)) ++space;
-        else ++punct;
-    }
-    if(text.empty()) return {};
-    const float total=static_cast<float>(text.size());
-    return {static_cast<float>(distinct)/total,
-            static_cast<float>(alpha)/total,
-            static_cast<float>(num)/total,
-            static_cast<float>(punct)/total,
-            static_cast<float>(space)/total};
-}
-
-static float profile_distance(const Profile&a,const Profile&b){
-    return std::fabs(a.distinct_ratio-b.distinct_ratio)+
-           std::fabs(a.alpha_ratio-b.alpha_ratio)+
-           std::fabs(a.num_ratio-b.num_ratio)+
-           std::fabs(a.punct_ratio-b.punct_ratio)+
-           std::fabs(a.space_ratio-b.space_ratio);
-}
-
-// ============================================================================
-// Generic CSR index
-// ============================================================================
-struct CSRIndex {
-    std::vector<std::uint64_t> keys;
-    std::vector<std::uint32_t> offsets;
-    std::vector<std::uint32_t> postings;
-    void clear(){keys.clear();offsets.clear();postings.clear();}
-    std::pair<const std::uint32_t*,std::size_t> find(std::uint64_t key) const {
-        auto it=std::lower_bound(keys.begin(),keys.end(),key);
-        if(it==keys.end()||*it!=key) return {nullptr,0};
-        const std::size_t idx=static_cast<std::size_t>(it-keys.begin());
-        const std::size_t a=offsets[idx],b=offsets[idx+1];
-        return {postings.data()+a,b-a};
-    }
-};
-
-struct BuildIndex {
-    std::unordered_map<std::uint64_t,std::vector<std::uint32_t>> postings;
-    void clear(){postings.clear();}
-};
-
-static void merge_index(BuildIndex& dst,BuildIndex& src){
-    for(auto& kv:src.postings){
-        auto& v=dst.postings[kv.first];
-        v.insert(v.end(),kv.second.begin(),kv.second.end());
-    }
-}
-
-static void compile_csr(const BuildIndex& src,CSRIndex& dst){
-    std::vector<std::uint64_t> keys;
-    keys.reserve(src.postings.size());
-    for(const auto& kv:src.postings) keys.push_back(kv.first);
-    std::sort(keys.begin(),keys.end());
-    dst.clear();
-    dst.keys=std::move(keys);
-    dst.offsets.reserve(dst.keys.size()+1);
-    std::size_t total=0;
-    for(std::uint64_t k:dst.keys) total+=src.postings.at(k).size();
-    dst.postings.reserve(total);
-    dst.offsets.push_back(0);
-    for(std::uint64_t k:dst.keys){
-        const auto& v=src.postings.at(k);
-        dst.postings.insert(dst.postings.end(),v.begin(),v.end());
-        dst.offsets.push_back(static_cast<std::uint32_t>(dst.postings.size()));
-    }
-}
-
-// ============================================================================
-// Pattern graph / relations / constraints
-// ============================================================================
-struct Edge {
-    std::uint32_t from=0,to=0;
-    std::uint8_t type=0;
-};
-
-struct Relation {
-    std::uint32_t a=0,b=0;
-    std::int32_t distance=0;
-    std::uint8_t type=0;
-};
-
-struct Constraint {
-    std::uint32_t a=0,b=0;
-    std::uint8_t type=0;
-};
-
-// ============================================================================
-// Canonical and structural pattern representation
-// ============================================================================
-struct Pattern {
-    StringRef text;
-    Profile profile;
-    std::vector<std::uint32_t> token_ids;
-    std::vector<std::uint32_t> canonical_ids;
-    std::vector<std::uint8_t> kinds;
-    std::vector<std::uint8_t> shapes;
-    std::vector<std::uint64_t> token_morph_hashes;
-    std::vector<std::uint64_t> token_transition_hashes;
-    std::vector<Relation> relations;
-    std::vector<Constraint> constraints;
-    std::vector<Edge> graph;
-    std::uint64_t canonical_hash=0;
-    std::uint64_t graph_hash=0;
-    std::uint64_t relation_hash=0;
-    std::uint64_t transition_hash=0;
-    std::uint32_t observations=1;
-    std::uint32_t structural_variants=1;
-    std::uint32_t exact_matches=0;
-    std::uint32_t parent=std::numeric_limits<std::uint32_t>::max();
-    std::vector<std::uint32_t> children;
-};
-
-static std::vector<std::uint32_t> canonicalize(const std::vector<std::uint32_t>& ids){
-    std::unordered_map<std::uint32_t,std::uint32_t> renumber;
-    renumber.reserve(ids.size()*2+1);
-    std::vector<std::uint32_t> out(ids.size());
-    std::uint32_t next=0;
-    for(std::size_t i=0;i<ids.size();++i){
-        auto it=renumber.find(ids[i]);
-        if(it==renumber.end()) it=renumber.emplace(ids[i],next++).first;
-        out[i]=it->second;
-    }
-    return out;
-}
-
-static std::uint64_t canonical_hash(const std::vector<std::uint32_t>& c,
-                                    const std::vector<std::uint8_t>& k){
-    std::uint64_t h=FNV_OFFSET;
-    for(std::size_t i=0;i<c.size();++i){
-        h=combine_hashes(h,c[i]);
-        h=combine_hashes(h,k[i]);
-    }
-    return h;
-}
-
-static std::vector<Relation> build_relations(const std::vector<std::uint32_t>& c){
-    std::vector<Relation> r;
-    const std::size_t n=c.size();
-    if(n<2) return r;
-    r.reserve(n*2);
-    for(std::size_t i=0;i<n;++i){
-        for(std::size_t j=i+1;j<n;++j){
-            if(c[i]==c[j]) r.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),static_cast<std::int32_t>(j-i),0});
-        }
-    }
-    for(std::size_t i=0;i+1<n;++i)
-        r.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(i+1),1,1});
-    std::sort(r.begin(),r.end(),[](const Relation&a,const Relation&b){
-        return std::tie(a.type,a.distance,a.a,a.b)<std::tie(b.type,b.distance,b.a,b.b);
-    });
-    return r;
-}
-
-static std::uint64_t relation_hash(const std::vector<Relation>& r){
-    std::uint64_t h=FNV_OFFSET;
-    for(const auto& x:r){
-        h=combine_hashes(h,x.type);
-        h=combine_hashes(h,x.a);
-        h=combine_hashes(h,x.b);
-        h=combine_hashes(h,static_cast<std::uint32_t>(x.distance));
-    }
-    return h;
-}
-
-static std::vector<Constraint> build_constraints(const std::vector<std::uint32_t>& c){
-    std::vector<Constraint> out;
-    for(std::size_t i=0;i<c.size();++i){
-        for(std::size_t j=i+1;j<c.size();++j){
-            if(c[i]==c[j]) out.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),0});
-            else if(i+1==j) out.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),1});
-        }
-    }
-    return out;
-}
-
-static std::vector<Edge> build_graph(const std::vector<std::uint32_t>& c){
-    std::vector<Edge> g;
-    if(c.size()<2) return g;
-    g.reserve(c.size()*3);
-    for(std::size_t i=0;i+1<c.size();++i) g.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(i+1),0});
-    for(std::size_t i=0;i<c.size();++i){
-        for(std::size_t j=i+1;j<c.size();++j){
-            if(c[i]==c[j]) g.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),1});
-        }
-    }
-    return g;
-}
-
-static std::uint64_t graph_hash(const std::vector<Edge>& g){
-    std::uint64_t h=FNV_OFFSET;
-    for(const auto&e:g){h=combine_hashes(h,e.from);h=combine_hashes(h,e.to);h=combine_hashes(h,e.type);}
-    return h;
-}
-
-static std::uint64_t sequence_transition_hash(const std::vector<std::uint64_t>& h){
-    std::uint64_t x=FNV_OFFSET;
-    for(std::uint64_t v:h) x=combine_hashes(x,v);
-    return x;
-}
-
-// ============================================================================
-// Generic local token analysis
-// ============================================================================
-static void analyze_token_sequence(std::string_view text,
-                                   std::vector<std::uint32_t>& ids,
-                                   std::vector<std::uint8_t>& kinds,
-                                   std::vector<std::uint8_t>& shapes,
-                                   std::vector<std::uint64_t>& morph,
-                                   std::vector<std::uint64_t>& transitions){
-    auto raw=tokenize_views(text);
-    ids.resize(raw.size()); kinds.resize(raw.size()); shapes.resize(raw.size());
-    morph.resize(raw.size()); transitions.resize(raw.size());
-    for(std::size_t i=0;i<raw.size();++i){
-        ids[i]=token_dict.intern(raw[i].text);
-        kinds[i]=static_cast<std::uint8_t>(token_kind(raw[i].text));
-        shapes[i]=static_cast<std::uint8_t>(shape_kind(raw[i].text));
-        morph[i]=morphology_hash(raw[i].text);
-        transitions[i]=transition_hash(raw[i].text);
-    }
-}
-
-static std::vector<std::uint32_t> longest_common_structure(const std::vector<std::uint32_t>& a,
-                                                            const std::vector<std::uint32_t>& b){
-    if(a.empty()||b.empty()) return {};
-    if(a.size()*b.size()>2000000ULL) return {};
-    std::vector<std::uint32_t> prev(b.size()+1),cur(b.size()+1);
-    for(std::size_t i=1;i<=a.size();++i){
-        for(std::size_t j=1;j<=b.size();++j)
-            cur[j]=(a[i-1]==b[j-1])?prev[j-1]+1:std::max(prev[j],cur[j-1]);
-        prev.swap(cur); std::fill(cur.begin(),cur.end(),0);
-    }
-    std::vector<std::uint32_t> out(prev.back());
-    std::iota(out.begin(),out.end(),0u);
-    return out;
-}
-
-// ============================================================================
-// Pattern construction
-// ============================================================================
-static Pattern construct_pattern(const std::string& text,StringRef ref){
-    Pattern p;
-    p.text=ref;
-    p.profile=extract_profile(text);
-    analyze_token_sequence(text,p.token_ids,p.kinds,p.shapes,p.token_morph_hashes,p.token_transition_hashes);
-    p.canonical_ids=canonicalize(p.token_ids);
-    p.canonical_hash=canonical_hash(p.canonical_ids,p.kinds);
-    p.relations=build_relations(p.canonical_ids);
-    p.relation_hash=relation_hash(p.relations);
-    p.constraints=build_constraints(p.canonical_ids);
-    p.graph=build_graph(p.canonical_ids);
-    p.graph_hash=graph_hash(p.graph);
-    p.transition_hash=sequence_transition_hash(p.token_transition_hashes);
-    return p;
-}
-
-// ============================================================================
-// Efficient Suffix Arrays for variable-length prefix lookups
-// ============================================================================
-struct SAElem {
-    std::uint32_t pat_id;
-    std::uint32_t offset;
-};
-
-static std::vector<SAElem> text_sa;
-static std::vector<SAElem> shape_sa;
-
-static void build_suffix_array(const std::vector<Pattern>& pats, std::vector<SAElem>& sa, bool use_shapes) {
-    std::size_t total_size = 0;
-    for (const auto& p : pats) {
-        total_size += use_shapes ? p.canonical_ids.size() : p.token_ids.size();
-    }
-    sa.clear();
-    sa.resize(total_size);
-
-    std::unordered_map<std::uint32_t, std::vector<SAElem>> buckets;
-    for (std::uint32_t i = 0; i < pats.size(); ++i) {
-        const auto& vec = use_shapes ? pats[i].canonical_ids : pats[i].token_ids;
-        for (std::uint32_t j = 0; j < vec.size(); ++j) {
-            buckets[vec[j]].push_back({i, j});
-        }
-    }
-
-    std::vector<std::vector<SAElem>*> bucket_ptrs;
-    bucket_ptrs.reserve(buckets.size());
-    for (auto& kv : buckets) {
-        bucket_ptrs.push_back(&kv.second);
-    }
-
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic)
-#endif
-    for (int i = 0; i < static_cast<int>(bucket_ptrs.size()); ++i) {
-        auto* b = bucket_ptrs[i];
-        std::sort(b->begin(), b->end(), [&](const SAElem& a, const SAElem& b_elem) {
-            const auto& va = use_shapes ? pats[a.pat_id].canonical_ids : pats[a.pat_id].token_ids;
-            const auto& vb = use_shapes ? pats[b_elem.pat_id].canonical_ids : pats[b_elem.pat_id].token_ids;
-            const std::size_t len_a = va.size() - a.offset;
-            const std::size_t len_b = vb.size() - b_elem.offset;
-            const std::size_t m = std::min(len_a, len_b);
-            const std::uint32_t* pa = va.data() + a.offset;
-            const std::uint32_t* pb = vb.data() + b_elem.offset;
-
-            for (std::size_t k = 1; k < m; ++k) {
-                if (pa[k] != pb[k]) return pa[k] < pb[k];
-            }
-            return len_a < len_b;
-        });
-    }
-
-    std::vector<std::uint32_t> keys;
-    keys.reserve(buckets.size());
-    for (const auto& kv : buckets) {
-        keys.push_back(kv.first);
-    }
-    std::sort(keys.begin(), keys.end());
-
-    std::size_t offset = 0;
-    for (std::uint32_t k : keys) {
-        const auto& b = buckets[k];
-        std::copy(b.begin(), b.end(), sa.begin() + offset);
-        offset += b.size();
-    }
-}
-
-// Longest Common Prefix (LCP) Suffix Array search for variable-length prefixes
-static std::tuple<std::size_t, std::size_t, std::size_t> search_sa_longest(
-    const std::vector<SAElem>& sa, const std::vector<Pattern>& pats,
-    const std::vector<std::uint32_t>& query, bool use_shapes) {
-
-    if (query.empty() || sa.empty()) return {0, 0, 0};
-
-    auto cmp_lower = [&](const SAElem& a, const std::vector<std::uint32_t>& q) {
-        const auto& va = use_shapes ? pats[a.pat_id].canonical_ids : pats[a.pat_id].token_ids;
-        const std::size_t m = std::min(va.size() - a.offset, q.size());
-        const std::uint32_t* pa = va.data() + a.offset;
-        for (std::size_t k = 0; k < m; ++k) {
-            if (pa[k] != q[k]) return pa[k] < q[k];
-        }
-        return (va.size() - a.offset) < q.size();
-    };
-
-    auto it_low = std::lower_bound(sa.begin(), sa.end(), query, cmp_lower);
-
-    auto get_lcp = [&](auto it) {
-        if (it == sa.end()) return std::size_t(0);
-        const auto& v = use_shapes ? pats[it->pat_id].canonical_ids : pats[it->pat_id].token_ids;
-        std::size_t m = std::min(v.size() - it->offset, query.size());
-        std::size_t lcp = 0;
-        const std::uint32_t* p = v.data() + it->offset;
-        while (lcp < m && p[lcp] == query[lcp]) ++lcp;
-        return lcp;
-    };
-
-    std::size_t lcp_low = get_lcp(it_low);
-    std::size_t lcp_prev = (it_low != sa.begin()) ? get_lcp(it_low - 1) : 0;
-    std::size_t max_lcp = std::max(lcp_low, lcp_prev);
-
-    if (max_lcp == 0) return {0, 0, 0};
-
-    std::vector<std::uint32_t> sub_query(query.begin(), query.begin() + max_lcp);
-
-    auto cmp_upper = [&](const std::vector<std::uint32_t>& q, const SAElem& a) {
-        const auto& va = use_shapes ? pats[a.pat_id].canonical_ids : pats[a.pat_id].token_ids;
-        const std::size_t m = std::min(va.size() - a.offset, q.size());
-        const std::uint32_t* pa = va.data() + a.offset;
-        for (std::size_t k = 0; k < m; ++k) {
-            if (q[k] != pa[k]) return q[k] < pa[k];
-        }
-        return false;
-    };
-
-    auto final_start = std::lower_bound(sa.begin(), sa.end(), sub_query, cmp_lower);
-    auto final_end = std::upper_bound(final_start, sa.end(), sub_query, cmp_upper);
-
-    return {static_cast<std::size_t>(std::distance(sa.begin(), final_start)),
-            static_cast<std::size_t>(std::distance(sa.begin(), final_end)),
-            max_lcp};
-}
-
-// ============================================================================
-// Global indexes
-// ============================================================================
-static std::vector<Pattern> patterns;
-static CSRIndex morph_csr,relation_csr,graph_csr;
-static std::unordered_map<std::uint64_t,std::uint32_t> canonical_pattern_map;
-
-static void build_pattern_indexes(){
-    BuildIndex morph,relation,graph;
-    canonical_pattern_map.clear();
-    const std::size_t n=patterns.size();
-    const int threads=std::max(1,omp_get_max_threads());
-    std::vector<BuildIndex> locals_morph(static_cast<std::size_t>(threads));
-    std::vector<BuildIndex> locals_rel(static_cast<std::size_t>(threads));
-    std::vector<BuildIndex> locals_graph(static_cast<std::size_t>(threads));
-
-#ifdef _OPENMP
-#pragma omp parallel
-#endif
-    {
-        const int tid=omp_get_thread_num();
-        BuildIndex &lm=locals_morph[static_cast<std::size_t>(tid)],
-                   &lr=locals_rel[static_cast<std::size_t>(tid)],
-                   &lg=locals_graph[static_cast<std::size_t>(tid)];
-#ifdef _OPENMP
-#pragma omp for schedule(dynamic,16)
-#endif
-        for(int ii=0;ii<static_cast<int>(n);++ii){
-            const std::uint32_t id=static_cast<std::uint32_t>(ii);
-            const Pattern&p=patterns[id];
-            for(auto h:p.token_morph_hashes) lm.postings[h].push_back(id);
-            if(p.relation_hash) lr.postings[p.relation_hash].push_back(id);
-            if(p.graph_hash) lg.postings[p.graph_hash].push_back(id);
-        }
-    }
-
-    for(auto&x:locals_morph) merge_index(morph,x);
-    for(auto&x:locals_rel) merge_index(relation,x);
-    for(auto&x:locals_graph) merge_index(graph,x);
-    compile_csr(morph,morph_csr); compile_csr(relation,relation_csr); compile_csr(graph,graph_csr);
-
-#ifdef _OPENMP
-#pragma omp parallel sections
-#endif
-    {
-#ifdef _OPENMP
-#pragma omp section
-#endif
-        build_suffix_array(patterns, text_sa, false);
-#ifdef _OPENMP
-#pragma omp section
-#endif
-        build_suffix_array(patterns, shape_sa, true);
-    }
-
-    canonical_pattern_map.reserve(n*2+1);
-    for(std::uint32_t i=0;i<n;++i){
-        auto it=canonical_pattern_map.find(patterns[i].canonical_hash);
-        if(it==canonical_pattern_map.end()) canonical_pattern_map.emplace(patterns[i].canonical_hash,i);
-        else patterns[it->second].observations+=patterns[i].observations;
-    }
-}
-
-// ============================================================================
-// Input cleaning and file loading
-// ============================================================================
-static std::string strip_unneeded_input(std::string_view text){
-    std::string result;
-    result.reserve(text.size());
-    bool line=false,block=false;
-    for(std::size_t i=0;i<text.size();++i){
-        if(line){ if(text[i]=='\n'){line=false;result.push_back('\n');} continue; }
-        if(block){
-            if(i+1<text.size()&&text[i]=='*'&&text[i+1]=='/'){block=false;++i;}
-            continue;
-        }
-        if(i+1<text.size()&&text[i]=='/'&&text[i+1]=='/'){line=true;++i;continue;}
-        if(i+1<text.size()&&text[i]=='/'&&text[i+1]=='*'){block=true;++i;continue;}
-        if(text[i]!='\r') result.push_back(text[i]);
-    }
-    return result;
-}
-
-static bool read_file(const std::string& filename,std::string& output){
-    std::ifstream is(filename.c_str(),std::ios::binary|std::ios::ate);
-    if(!is) return false;
-    const std::streampos end=is.tellg();
-    if(end<0) return false;
-    output.clear(); output.resize(static_cast<std::size_t>(end));
-    is.seekg(0,std::ios::beg);
-    return output.empty() || static_cast<bool>(is.read(output.data(),static_cast<std::streamsize>(output.size())));
-}
-
-static void split_text(const std::string& text,std::size_t chunk_size,std::vector<std::string>& out){
-    if(text.empty()) return;
-    if(!chunk_size) chunk_size=8192;
-    std::string current; current.reserve(chunk_size);
-    std::size_t line_start=0;
-    while(line_start<text.size()){
-        std::size_t line_end=text.find('\n',line_start);
-        if(line_end==std::string::npos) line_end=text.size();
-        std::string line=text.substr(line_start,line_end-line_start);
-        if(line_end<text.size()) line.push_back('\n');
-        if(line.size()>chunk_size){
-            if(!current.empty()){out.push_back(std::move(current));current.clear();current.reserve(chunk_size);}
-            for(std::size_t pos=0;pos<line.size();){
-                const std::size_t take=std::min(chunk_size,line.size()-pos);
-                out.push_back(line.substr(pos,take)); pos+=take;
-            }
-        } else {
-            if(!current.empty()&&current.size()+line.size()>chunk_size){out.push_back(std::move(current));current.clear();current.reserve(chunk_size);}
-            current+=line;
-        }
-        if(line_end==text.size()) break;
-        line_start=line_end+1;
-    }
-    if(!current.empty()) out.push_back(std::move(current));
-}
-
-static bool load_learning_file(const std::string& filename,std::vector<std::string>& dataset){
-    std::string text;
-    if(!read_file(filename,text)){std::cerr<<"Could not read file: "<<filename<<'\n';return false;}
-    if(text.empty()) return false;
-    text=strip_unneeded_input(text);
-    const std::size_t before=dataset.size();
-    split_text(text,config.chunk_size,dataset);
-    std::cerr<<"Learn source: "<<filename<<" -> "<<dataset.size()-before<<" pattern(s)\n";
-    return dataset.size()>before;
-}
-
-static bool load_learning_files(const std::vector<std::string>& files,std::vector<std::string>& dataset){
-    if(files.empty()) return false;
-    std::vector<std::vector<std::string>> local(files.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic)
-#endif
-    for(int i=0;i<static_cast<int>(files.size());++i)
-        load_learning_file(files[static_cast<std::size_t>(i)],local[static_cast<std::size_t>(i)]);
-    std::size_t total=dataset.size();
-    for(const auto&v:local) total+=v.size();
-    dataset.reserve(total);
-    for(auto&v:local) dataset.insert(dataset.end(),std::make_move_iterator(v.begin()),std::make_move_iterator(v.end()));
-    return !dataset.empty();
-}
-
-static void deduplicate(std::vector<std::string>& data){
-    if(data.size()<2) return;
-    std::unordered_set<std::string> seen; seen.reserve(data.size()*2);
-    std::vector<std::string> unique; unique.reserve(data.size());
-    for(std::string&s:data) if(seen.insert(s).second) unique.push_back(std::move(s));
-    data.swap(unique);
-}
-
-// ============================================================================
-// Pattern merging / specialization tree
-// ============================================================================
-static void merge_equivalent_patterns(){
-    if(patterns.empty()) return;
-    std::vector<Pattern> merged;
-    merged.reserve(patterns.size());
-    std::unordered_map<std::uint64_t,std::uint32_t> seen;
-    seen.reserve(patterns.size()*2+1);
-    for(Pattern&p:patterns){
-        auto it=seen.find(p.canonical_hash);
-        if(it==seen.end()){
-            const std::uint32_t id=static_cast<std::uint32_t>(merged.size());
-            seen.emplace(p.canonical_hash,id);
-            merged.push_back(std::move(p));
-        } else {
-            Pattern&q=merged[it->second];
-            const auto common=longest_common_structure(q.canonical_ids,p.canonical_ids);
-            q.observations += p.observations;
-            q.structural_variants += static_cast<std::uint32_t>(std::max<std::size_t>(1,common.size()));
-            q.exact_matches += p.exact_matches;
-            if(profile_distance(p.profile,{})<profile_distance(q.profile,{})) q.profile=p.profile;
-        }
-    }
-    patterns.swap(merged);
-}
-
-static void build_specialization_tree(){
-    if(patterns.empty()) return;
-    std::unordered_map<std::uint64_t,std::uint32_t> parent_map;
-    parent_map.reserve(patterns.size()*2+1);
-    for(std::uint32_t i=0;i<patterns.size();++i){
-        Pattern&p=patterns[i]; p.parent=std::numeric_limits<std::uint32_t>::max(); p.children.clear();
-        if(p.canonical_ids.empty()) continue;
-        std::vector<std::uint32_t> generalized=p.canonical_ids;
-        if(!generalized.empty()) generalized.back()=0;
-        const std::uint64_t h=canonical_hash(generalized,p.kinds);
-        auto it=parent_map.find(h);
-        if(it!=parent_map.end()&&it->second!=i) p.parent=it->second;
-        parent_map.emplace(p.canonical_hash,i);
-    }
-    for(std::uint32_t i=0;i<patterns.size();++i)
-        if(patterns[i].parent<patterns.size()) patterns[patterns[i].parent].children.push_back(i);
-}
-
-static void build_knowledge_base(const std::vector<std::string>& dataset){
-    patterns.clear(); string_pool.clear(); token_dict.clear();
-    morph_csr.clear(); relation_csr.clear(); graph_csr.clear();
-    text_sa.clear(); shape_sa.clear();
-
-    if(dataset.empty()) return;
-
-    std::vector<StringRef> refs(dataset.size());
-    string_pool.bytes.reserve(std::accumulate(dataset.begin(),dataset.end(),std::size_t(0),[](std::size_t n,const std::string&s){return n+s.size();}));
-    std::unordered_set<std::string_view,TokenHash> unique_tokens;
-    unique_tokens.reserve(dataset.size()*8+1);
-    for(std::size_t i=0;i<dataset.size();++i){
-        refs[i]={string_pool.append(dataset[i]),static_cast<std::uint32_t>(dataset[i].size())};
-        const auto raw=tokenize_views(dataset[i]);
-        for(const auto&t:raw) unique_tokens.insert(t.text);
-    }
-    token_dict.to_id.reserve(unique_tokens.size()*2+1);
-    for(const auto&t:unique_tokens) token_dict.intern(t);
-
-    patterns.resize(dataset.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic,16)
-#endif
-    for(int i=0;i<static_cast<int>(dataset.size());++i)
-        patterns[static_cast<std::size_t>(i)]=construct_pattern(dataset[static_cast<std::size_t>(i)],refs[static_cast<std::size_t>(i)]);
-    merge_equivalent_patterns();
-    build_specialization_tree();
-    build_pattern_indexes();
-}
-
-// ============================================================================
-// Query representation
-// ============================================================================
-struct Query {
-    std::string text;
-    std::vector<std::uint32_t> token_ids;
-    std::vector<std::uint8_t> kinds;
-    std::vector<std::uint8_t> shapes;
-    std::vector<std::uint64_t> morph;
-    std::vector<std::uint64_t> transitions;
-    std::vector<std::uint32_t> canonical_ids;
-    Profile profile;
-    std::uint64_t relation_hash=0;
-    std::uint64_t graph_hash=0;
-    std::uint64_t transition_hash_value=0;
-};
-
-static Query make_query(const std::string& text){
-    Query q; q.text=text;
-    analyze_token_sequence(text,q.token_ids,q.kinds,q.shapes,q.morph,q.transitions);
-    q.canonical_ids=canonicalize(q.token_ids);
-    q.profile=extract_profile(text);
-    q.relation_hash=relation_hash(build_relations(q.canonical_ids));
-    q.graph_hash=graph_hash(build_graph(q.canonical_ids));
-    q.transition_hash_value=sequence_transition_hash(q.transitions);
-    return q;
-}
-
-// ============================================================================
-// Deterministic scoring
-// ============================================================================
-struct MatchScore {
-    std::uint32_t exact=0;
-    std::uint32_t structural=0;
-    std::uint32_t relational=0;
-    std::uint32_t contextual=0;
-    std::uint32_t profile=0;
-    std::uint32_t length=0;
-    std::uint32_t observations=0;
-    std::uint32_t id=0;
-};
-
-static bool better_score(const MatchScore&a,const MatchScore&b){
-    return std::tie(a.exact,a.structural,a.relational,a.contextual,a.profile,a.length,a.observations)
-         >std::tie(b.exact,b.structural,b.relational,b.contextual,b.profile,b.length,b.observations)
-        || (a.exact==b.exact&&a.structural==b.structural&&a.relational==b.relational&&a.contextual==b.contextual&&a.profile==b.profile&&a.length==b.length&&a.observations==b.observations&&a.id<b.id);
-}
-
-static std::uint32_t profile_score(const Profile& a,const Profile&b){
-    const double d=profile_distance(a,b);
-    const double x=std::max(0.0,1000000.0*(1.0-d));
-    return static_cast<std::uint32_t>(x);
-}
-
-static std::uint32_t sequence_exact_prefix(const std::vector<std::uint32_t>& a,const std::vector<std::uint32_t>& b){
-    const std::size_t n=std::min(a.size(),b.size()); std::uint32_t x=0;
-    for(std::size_t i=0;i<n;++i){if(a[i]!=b[i]) break;++x;} return x;
-}
-
-static std::uint32_t common_structure_count(const std::vector<std::uint32_t>& a,const std::vector<std::uint32_t>& b){
-    const std::size_t n=std::min(a.size(),b.size()); std::uint32_t x=0;
-    for(std::size_t i=0;i<n;++i) if(a[i]==b[i]) ++x;
-    return x;
-}
-
-// ============================================================================
-// Candidate retrieval with sparse epochs
-// ============================================================================
-struct CandidateAccumulator {
-    std::vector<std::uint32_t> stamp;
-    std::vector<MatchScore> score;
-    std::vector<std::uint32_t> touched;
-    std::uint32_t epoch=0;
-
-    void prepare(std::size_t n){
-        if(stamp.size()<n){stamp.resize(n,0);score.resize(n);}
-        if(++epoch==0){std::fill(stamp.begin(),stamp.end(),0);epoch=1;}
-        touched.clear();
-    }
-
-    MatchScore& touch(std::uint32_t id){
-        if(stamp[id]!=epoch){
-            stamp[id]=epoch; score[id]={0,0,0,0,0,0,0,id}; touched.push_back(id);
-        }
-        return score[id];
-    }
-};
-
-static void add_csr_hits(const CSRIndex& idx,const std::vector<std::uint64_t>& keys, CandidateAccumulator& c, std::uint32_t weight_type){
-    for(auto key:keys){
-        auto [ptr,count]=idx.find(key);
-        for(std::size_t i=0;i<count;++i){
-            MatchScore&s=c.touch(ptr[i]);
-            if(weight_type==0) ++s.structural;
-            else if(weight_type==1) ++s.relational;
-            else ++s.exact;
-        }
-    }
-}
-
-static std::vector<MatchScore> search_matches(const Query&q,std::size_t k){
-    std::vector<MatchScore> out;
-    if(patterns.empty()||q.token_ids.empty()||!k) return out;
-    CandidateAccumulator acc;
-    acc.prepare(patterns.size());
-
-    // Variable-length exact text matching using Suffix Array LCP
-    for (std::size_t i = 0; i < q.token_ids.size(); ++i) {
-        std::vector<std::uint32_t> q_suffix(q.token_ids.begin() + i, q.token_ids.end());
-        auto [start, end, lcp] = search_sa_longest(text_sa, patterns, q_suffix, false);
-        for (std::size_t j = start; j < end; ++j) {
-            acc.touch(text_sa[j].pat_id).exact += static_cast<std::uint32_t>(lcp * lcp);
-        }
-    }
-
-    // Variable-length structural shape matching using Suffix Array LCP
-    for (std::size_t i = 0; i < q.canonical_ids.size(); ++i) {
-        std::vector<std::uint32_t> q_suffix(q.canonical_ids.begin() + i, q.canonical_ids.end());
-        auto [start, end, lcp] = search_sa_longest(shape_sa, patterns, q_suffix, true);
-        for (std::size_t j = start; j < end; ++j) {
-            acc.touch(shape_sa[j].pat_id).structural += static_cast<std::uint32_t>(lcp * lcp);
-        }
-    }
-
-    add_csr_hits(morph_csr,q.morph,acc,0);
-    if(q.relation_hash){ auto [p,n]=relation_csr.find(q.relation_hash); for(std::size_t i=0;i<n;++i) ++acc.touch(p[i]).relational; }
-    if(q.graph_hash){ auto [p,n]=graph_csr.find(q.graph_hash); for(std::size_t i=0;i<n;++i) ++acc.touch(p[i]).structural; }
-
-    std::vector<MatchScore> tmp(acc.touched.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(static)
-#endif
-    for(int i=0;i<static_cast<int>(acc.touched.size());++i){
-        const std::uint32_t id=acc.touched[static_cast<std::size_t>(i)];
-        MatchScore&s=acc.score[id]; const Pattern&p=patterns[id];
-        s.exact += sequence_exact_prefix(q.token_ids,p.token_ids);
-        s.structural += common_structure_count(q.canonical_ids,p.canonical_ids);
-        s.relational += static_cast<std::uint32_t>(std::min<std::size_t>(p.relations.size(),q.canonical_ids.size()));
-        s.profile=profile_score(q.profile,p.profile);
-        const std::size_t m=std::min(q.token_ids.size(),p.token_ids.size());
-        s.length=static_cast<std::uint32_t>(m);
-        s.observations=p.observations;
-        tmp[static_cast<std::size_t>(i)]=s;
-    }
-    if(tmp.size()>k){
-        std::nth_element(tmp.begin(),tmp.begin()+static_cast<std::ptrdiff_t>(k),tmp.end(),better_score);
-        tmp.resize(k);
-    }
-    std::sort(tmp.begin(),tmp.end(),better_score);
-    return tmp;
-}
-
-// ============================================================================
-// Structural alignment and bindings
-// ============================================================================
-struct BindingState {
-    std::unordered_map<std::uint32_t,std::uint32_t> forward;
-    std::unordered_set<std::uint32_t> used_targets;
-    void reserve(std::size_t n){forward.reserve(n*2+1);used_targets.reserve(n*2+1);}
-};
-
-static bool compatible_ids(std::uint32_t source,std::uint32_t target){
-    if(source==target) return true;
-    const std::string_view a=token_dict.view(source),b=token_dict.view(target);
-    const TKind ka=token_kind(a),kb=token_kind(b);
-    return variable_candidate(ka)&&variable_candidate(kb)&&ka==kb;
-}
-
-static bool add_binding(BindingState&b,std::uint32_t s,std::uint32_t t){
-    if(s==t) return true;
-    if(!compatible_ids(s,t)) return false;
-    auto it=b.forward.find(s);
-    if(it!=b.forward.end()) return it->second==t;
-    if(!b.used_targets.insert(t).second) return false;
-    b.forward.emplace(s,t); return true;
-}
-
-struct Alignment {
-    bool valid=false;
-    std::size_t position=0;
-    BindingState bindings;
-    std::uint32_t exact=0,structural=0,context=0;
-};
-
-static bool align_at(const Pattern&p,const Query&q,std::size_t pos,Alignment&out){
-    if(pos+q.token_ids.size()>p.token_ids.size()) return false;
-    Alignment a; a.valid=true;a.position=pos;a.bindings.reserve(q.token_ids.size());
-    for(std::size_t i=0;i<q.token_ids.size();++i){
-        const auto s=p.token_ids[pos+i],t=q.token_ids[i];
-        if(s==t) ++a.exact;
-        else { if(!add_binding(a.bindings,s,t)) return false; ++a.structural; }
-    }
-    a.context=static_cast<std::uint32_t>(std::min(pos,p.token_ids.size()-pos-q.token_ids.size()));
-    out=std::move(a);return true;
-}
-
-static bool better_alignment(const Alignment&a,const Alignment&b){
-    if(a.exact!=b.exact) return a.exact>b.exact;
-    if(a.structural!=b.structural) return a.structural>b.structural;
-    if(a.context!=b.context) return a.context>b.context;
-    return a.position<b.position;
-}
-
-static bool infer_best_alignment(const Pattern&p,const Query&q,Alignment&best){
-    if(q.token_ids.empty()||p.token_ids.size()<q.token_ids.size()) return false;
-    std::size_t pivot=0; int selectivity=-1;
-    for(std::size_t i=0;i<q.token_ids.size();++i){
-        const auto k=token_kind(token_dict.view(q.token_ids[i]));
-        const int s=(k==TKind::PUNCT?4:(variable_candidate(k)?1:3));
-        if(s>selectivity){selectivity=s;pivot=i;}
-    }
-    bool found=false;
-    for(std::size_t s=pivot;s<p.token_ids.size();++s){
-        if(s<pivot) continue;
-        if(!compatible_ids(p.token_ids[s],q.token_ids[pivot])) continue;
-        const std::size_t pos=s-pivot;
-        if(pos+q.token_ids.size()>p.token_ids.size()) continue;
-        Alignment a;
-        if(align_at(p,q,pos,a)&&(!found||better_alignment(a,best))){best=std::move(a);found=true;}
-    }
-    return found;
-}
-
-// ============================================================================
-// Rendering without repeated tokenization
-// ============================================================================
-static std::string render_suffix(const Pattern&p,std::size_t begin,const BindingState&bindings,std::size_t max_tokens){
-    if(begin>=p.token_ids.size()) return {};
-    const auto raw=tokenize_views(string_pool.view(p.text.offset,p.text.length));
-    const std::size_t end=std::min(p.token_ids.size(),begin+max_tokens);
-    std::size_t reserve=0;
-    for(std::size_t i=begin;i<end;++i) reserve+=raw[i].space_before.size()+raw[i].text.size();
-    std::string result; result.reserve(reserve);
-    for(std::size_t i=begin;i<end;++i){
-        result.append(raw[i].space_before);
-        auto it=bindings.forward.find(p.token_ids[i]);
-        if(it!=bindings.forward.end()){
-            std::string src(token_dict.view(p.token_ids[i]));
-            std::string tgt(token_dict.view(it->second));
-
-            std::cout << "\n[Approval Required] Apply variable binding '" << src << "' -> '" << tgt << "'? (y/n): ";
-            std::string ans;
-            if (std::getline(std::cin, ans) && (ans == "y" || ans == "Y")) {
-                result.append(tgt);
-            } else {
-                result.append(raw[i].text);
-            }
-        }
-        else {
-            result.append(raw[i].text);
-        }
-    }
-    return result;
-}
-
-static std::string produce_completion(const Query&q,const std::vector<MatchScore>&matches){
-    for(const auto&m:matches){
-        const Pattern&p=patterns[m.id];
-        Alignment a;
-        if(!infer_best_alignment(p,q,a)) continue;
-        const std::size_t begin=a.position+q.token_ids.size();
-        if(begin>=p.token_ids.size()) continue;
-        std::string suffix=render_suffix(p,begin,a.bindings,config.max_response_tokens);
-        std::string output=q.text;
-        output.reserve(q.text.size()+suffix.size());
-        output+=suffix;
-        return output;
-    }
-    // Dynamic variable-length suffix fallback utilizing Suffix Array LCP
-    if(!q.token_ids.empty()){
-        for (std::size_t i = 0; i < q.token_ids.size(); ++i) {
-            std::vector<std::uint32_t> q_suffix(q.token_ids.begin() + i, q.token_ids.end());
-            auto [start, end, lcp] = search_sa_longest(text_sa, patterns, q_suffix, false);
-
-            if (lcp == q_suffix.size()) {
-                for(std::size_t j = start; j < end; ++j){
-                    const Pattern& p = patterns[text_sa[j].pat_id];
-                    Alignment a;
-                    if(!infer_best_alignment(p, q, a)) continue;
-                    const std::size_t begin = a.position + q.token_ids.size();
-                    if(begin >= p.token_ids.size()) continue;
-                    std::string suffix = render_suffix(p, begin, a.bindings, config.max_response_tokens);
-                    if(!suffix.empty()){
-                        std::string output = q.text;
-                        output += suffix;
-                        return output;
-                    }
-                }
-            }
-        }
-    }
-    return {};
-}
-
-// ============================================================================
-// Binary persistence
-// ============================================================================
-static constexpr std::uint64_t KB_MAGIC=0x5041545445524E55ULL;
-static constexpr std::uint64_t KB_VERSION=12ULL;
-
-static void write_varuint(std::ostream&os,std::uint64_t v){while(v>=0x80){os.put(static_cast<char>((v&0x7F)|0x80));v>>=7;}os.put(static_cast<char>(v));if(!os)throw std::runtime_error("KB write failed");}
-static std::uint64_t read_varuint(std::istream&is){std::uint64_t v=0;unsigned shift=0;for(;;){unsigned char c=0;if(!is.read(reinterpret_cast<char*>(&c),1))throw std::runtime_error("truncated KB");v|=static_cast<std::uint64_t>(c&0x7F)<<shift;if(!(c&0x80))return v;shift+=7;if(shift>=64)throw std::runtime_error("invalid KB varint");}}
-static void write_float(std::ostream&os,float x){os.write(reinterpret_cast<const char*>(&x),sizeof(x));if(!os)throw std::runtime_error("KB write failed");}
-static float read_float(std::istream&is){float x=0;if(!is.read(reinterpret_cast<char*>(&x),sizeof(x)))throw std::runtime_error("truncated KB");return x;}
-static void write_string(std::ostream&os,std::string_view s){write_varuint(os,s.size());if(!s.empty())os.write(s.data(),static_cast<std::streamsize>(s.size()));}
-static std::string read_string(std::istream&is){const auto n=read_varuint(is);if(n>1024ULL*1024ULL*1024ULL)throw std::runtime_error("KB string too large");std::string s(static_cast<std::size_t>(n),'\0');if(n&&!is.read(s.data(),static_cast<std::streamsize>(n)))throw std::runtime_error("truncated KB");return s;}
-
-static void save_u32_vector(std::ostream&os,const std::vector<std::uint32_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u32_vector(std::istream&is,std::vector<std::uint32_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=static_cast<std::uint32_t>(read_varuint(is));}
-static void save_u8_vector(std::ostream&os,const std::vector<std::uint8_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u8_vector(std::istream&is,std::vector<std::uint8_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=static_cast<std::uint8_t>(read_varuint(is));}
-static void save_u64_vector(std::ostream&os,const std::vector<std::uint64_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u64_vector(std::istream&is,std::vector<std::uint64_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=read_varuint(is);}
-static void save_relations(std::ostream&os,const std::vector<Relation>&v){write_varuint(os,v.size());for(auto x:v){write_varuint(os,x.a);write_varuint(os,x.b);write_varuint(os,static_cast<std::uint32_t>(x.distance));write_varuint(os,x.type);}}
-static void load_relations(std::istream&is,std::vector<Relation>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v){x.a=static_cast<std::uint32_t>(read_varuint(is));x.b=static_cast<std::uint32_t>(read_varuint(is));x.distance=static_cast<std::int32_t>(read_varuint(is));x.type=static_cast<std::uint8_t>(read_varuint(is));}}
-static void save_edges(std::ostream&os,const std::vector<Edge>&v){write_varuint(os,v.size());for(auto x:v){write_varuint(os,x.from);write_varuint(os,x.to);write_varuint(os,x.type);}}
-static void load_edges(std::istream&is,std::vector<Edge>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v){x.from=static_cast<std::uint32_t>(read_varuint(is));x.to=static_cast<std::uint32_t>(read_varuint(is));x.type=static_cast<std::uint8_t>(read_varuint(is));}}
-
-static bool save_knowledge_base(const std::string&filename){
-    const std::string tmp=filename+".tmp";
-    try{
-        std::ofstream os(tmp.c_str(),std::ios::binary|std::ios::trunc);
-        if(!os)throw std::runtime_error("cannot open temporary KB");
-        write_varuint(os,KB_MAGIC);write_varuint(os,KB_VERSION);write_varuint(os,config.chunk_size);
-        write_varuint(os,token_dict.text.size());
-        for(const auto&s:token_dict.text) write_string(os,s);
-        write_varuint(os,patterns.size());
-        for(const auto&p:patterns){
-            write_string(os,string_pool.view(p.text.offset,p.text.length));
-            write_float(os,p.profile.distinct_ratio);write_float(os,p.profile.alpha_ratio);write_float(os,p.profile.num_ratio);write_float(os,p.profile.punct_ratio);write_float(os,p.profile.space_ratio);
-            save_u32_vector(os,p.token_ids);save_u32_vector(os,p.canonical_ids);save_u8_vector(os,p.kinds);save_u8_vector(os,p.shapes);save_u64_vector(os,p.token_morph_hashes);save_u64_vector(os,p.token_transition_hashes);
-            save_relations(os,p.relations);save_edges(os,p.graph);
-            write_varuint(os,p.canonical_hash);write_varuint(os,p.graph_hash);write_varuint(os,p.relation_hash);write_varuint(os,p.transition_hash);write_varuint(os,p.observations);write_varuint(os,p.structural_variants);write_varuint(os,p.exact_matches);
-        }
-        os.flush();if(!os)throw std::runtime_error("KB write failed");os.close();std::remove(filename.c_str());
-        if(std::rename(tmp.c_str(),filename.c_str())!=0){std::remove(tmp.c_str());throw std::runtime_error("KB rename failed");}
-        return true;
-    }catch(const std::exception&e){std::cerr<<"Save error: "<<e.what()<<'\n';std::remove(tmp.c_str());return false;}
-}
-
-static bool load_knowledge_base(const std::string&filename){
-    try{
-        std::ifstream is(filename.c_str(),std::ios::binary);if(!is)throw std::runtime_error("cannot open KB");
-        if(read_varuint(is)!=KB_MAGIC)throw std::runtime_error("invalid KB magic");
-        if(read_varuint(is)!=KB_VERSION)throw std::runtime_error("unsupported KB version");
-        config.chunk_size=static_cast<std::size_t>(read_varuint(is));
-        token_dict.clear();string_pool.clear();patterns.clear();
-        const std::size_t token_count=static_cast<std::size_t>(read_varuint(is));token_dict.to_id.reserve(token_count*2+1);
-        for(std::size_t i=0;i<token_count;++i) token_dict.intern(read_string(is));
-        const std::size_t count=static_cast<std::size_t>(read_varuint(is));if(count>100000000ULL)throw std::runtime_error("unreasonable KB pattern count");
-        patterns.resize(count);
-        for(Pattern&p:patterns){
-            const std::string s=read_string(is);const std::uint32_t off=string_pool.append(s);p.text={off,static_cast<std::uint32_t>(s.size())};
-            p.profile.distinct_ratio=read_float(is);p.profile.alpha_ratio=read_float(is);p.profile.num_ratio=read_float(is);p.profile.punct_ratio=read_float(is);p.profile.space_ratio=read_float(is);
-            load_u32_vector(is,p.token_ids);load_u32_vector(is,p.canonical_ids);load_u8_vector(is,p.kinds);load_u8_vector(is,p.shapes);load_u64_vector(is,p.token_morph_hashes);load_u64_vector(is,p.token_transition_hashes);
-            load_relations(is,p.relations);load_edges(is,p.graph);
-            p.canonical_hash=read_varuint(is);p.graph_hash=read_varuint(is);p.relation_hash=read_varuint(is);p.transition_hash=read_varuint(is);p.observations=static_cast<std::uint32_t>(read_varuint(is));p.structural_variants=static_cast<std::uint32_t>(read_varuint(is));p.exact_matches=static_cast<std::uint32_t>(read_varuint(is));
-        }
-        build_specialization_tree();build_pattern_indexes();
-        std::cerr<<"Loaded "<<patterns.size()<<" pattern(s), "<<token_dict.text.size()<<" token type(s)\n";
-        return true;
-    }catch(const std::exception&e){std::cerr<<"Load error: "<<e.what()<<'\n';return false;}
-}
-
-static bool print_kb_info(const std::string&filename){
-    try{std::ifstream is(filename.c_str(),std::ios::binary);if(!is)throw std::runtime_error("cannot open KB");if(read_varuint(is)!=KB_MAGIC)throw std::runtime_error("invalid KB magic");
-        const auto version=read_varuint(is),chunk=read_varuint(is);const auto tokens=read_varuint(is);std::cout<<"KB: "<<filename<<"\nVersion: "<<version<<"\nChunk size: "<<chunk<<"\nToken types: "<<tokens<<'\n';return true;
-    }catch(const std::exception&e){std::cerr<<"KB info error: "<<e.what()<<'\n';return false;}
-}
-
-// ============================================================================
-// CLI
-// ============================================================================
-struct CommandLine{std::vector<std::string>learn_files;std::string load_kb,save_kb,info_kb,prompt,retrieve;bool help=false,chat=false,show_retrieval=false;};
-
-static bool parse_size(const std::string&s,std::size_t&out){try{std::size_t pos=0;const auto n=std::stoull(s,&pos,10);if(pos!=s.size()||!n)return false;out=static_cast<std::size_t>(n);return true;}catch(...){return false;}}
-
-static void print_help(const char* program) {
-    std::cout << "\n" << program << "\n"
-              << "================================\n\n"
-              << "Learning:\n"
-              << "  --learn FILE [FILE ...]  Ingest text files into the knowledge base.\n"
-              << "  --chunk-size N           Bytes per processed chunk.\n\n"
-              << "Knowledge base:\n"
-              << "  --save-kb FILE           Save the current knowledge base to disk.\n"
-              << "  --load-kb FILE           Load an existing knowledge base.\n"
-              << "  --info-kb FILE           Display metadata about a saved knowledge base.\n\n"
-              << "Retrieval:\n"
-              << "  --top-k N                Number of matching archetypes to retrieve.\n"
-              << "  --retrieve TEXT          Retrieve and display top-k patterns for a prompt.\n"
-              << "  --show-retrieval         Print retrieved patterns before generating a response.\n\n"
-              << "Interaction:\n"
-              << "  --max-length N           Maximum number of tokens to generate.\n"
-              << "  --prompt TEXT            Generate a single response to the provided text.\n"
-              << "  --chat                   Interactive chat mode.\n"
-              << "                             * Multiline input; terminate with /end;\n"
-              << "                             * Exit with /quit\n\n"
-              << "Other:\n"
-              << "  --help                   Display this help menu.\n\n";
-}
-
-static bool parse_command_line(int argc,char**argv,CommandLine&cmd){
-    for(int i=1;i<argc;++i){
-        const std::string a=argv[i];
-        if(a=="--help"||a=="-h"){cmd.help=true;continue;}
-        if(a=="--chat"){cmd.chat=true;continue;}
-        if(a=="--show-retrieval"){cmd.show_retrieval=true;continue;}
-        if(a=="--learn"){if(i+1>=argc)return false;while(i+1<argc&&argv[i+1][0]!='-')cmd.learn_files.push_back(argv[++i]);continue;}
-        if(a=="--load-kb"){if(i+1>=argc)return false;cmd.load_kb=argv[++i];continue;}
-        if(a=="--save-kb"){if(i+1>=argc)return false;cmd.save_kb=argv[++i];continue;}
-        if(a=="--info-kb"){if(i+1>=argc)return false;cmd.info_kb=argv[++i];continue;}
-        if(a=="--retrieve"){if(i+1>=argc)return false;cmd.retrieve=argv[++i];continue;}
-        if(a=="--prompt"){if(i+1>=argc)return false;cmd.prompt=argv[++i];continue;}
-        if(a=="--top-k"){if(i+1>=argc||!parse_size(argv[++i],config.top_k))return false;continue;}
-        if(a=="--max-length"){if(i+1>=argc||!parse_size(argv[++i],config.max_response_tokens))return false;continue;}
-        if(a=="--chunk-size"){if(i+1>=argc||!parse_size(argv[++i],config.chunk_size))return false;continue;}
-        std::cerr<<"Unknown option: "<<a<<'\n';return false;
-    }
-    return true;
-}
-
-static void print_retrieval(const std::vector<MatchScore>&results){
-    std::cout<<"\nRetrieved structural patterns:\n";
-    if(results.empty()){std::cout<<"  None\n";return;}
-    for(std::size_t i=0;i<results.size();++i){
-        const auto&m=results[i];
-        std::cout<<"\nRank "<<i+1<<"\nExact: "<<m.exact<<"\nStructural: "<<m.structural<<"\nRelational: "<<m.relational<<"\nProfile: "<<m.profile<<"\nObservations: "<<m.observations<<"\nPattern:\n"<<string_pool.view(patterns[m.id].text.offset,patterns[m.id].text.length)<<'\n';
-    }
-}
-
-static void process_prompt(const std::string&prompt){
-    if(prompt.empty()||patterns.empty())return;
-    const Query q=make_query(prompt);
-    const auto results=search_matches(q,config.top_k);
-    if(config.show_retrieval)print_retrieval(results);
-    const std::string response=produce_completion(q,results);
-    std::cout<<(response.empty()?"No deterministic completion found.\n":response+'\n');
-}
-
-static void retrieve_only(const std::string&query){if(patterns.empty())return;print_retrieval(search_matches(make_query(query),config.top_k));}
-
-static void chat_loop(){
-    std::cout<<"\nPattern Constructor Chat\n====================================\nFinish prompt with /end\nExit with /quit\n\n";
-    std::string prompt,line;
-    while(true){
-        std::cout<<"> "<<std::flush;
-        if(!std::getline(std::cin,line))break;
-        if(line=="/quit")break;
-        if(line=="/end"){if(!prompt.empty()){process_prompt(prompt);std::cout<<'\n';}prompt.clear();continue;}
-        if(!prompt.empty()) prompt.push_back('\n');
-        prompt+=line;
-    }
-}
-
-int main(int argc,char**argv){
-#ifdef _WIN32
-    SetConsoleCP(65001);SetConsoleOutputCP(65001);
-#endif
-    CommandLine cmd;
-    if(!parse_command_line(argc,argv,cmd)){print_help(argv[0]);return 1;}
-    if(cmd.help){print_help(argv[0]);return 0;}
-    config.show_retrieval=cmd.show_retrieval;
-    config.chat_mode=cmd.chat;
-    if(!cmd.info_kb.empty()){return print_kb_info(cmd.info_kb)?0:1;}
-
-    if(!cmd.load_kb.empty()){
-        std::cerr<<"Loading knowledge base: "<<cmd.load_kb<<'\n';
-        if(!load_knowledge_base(cmd.load_kb))return 1;
-    }
-
-    if(!cmd.learn_files.empty()){
-        std::vector<std::string> dataset;
-        load_learning_files(cmd.learn_files,dataset);
-        const std::size_t before=dataset.size();
-        deduplicate(dataset);
-        std::cerr<<"Removed "<<before-dataset.size()<<" duplicate pattern(s)\n";
-        build_knowledge_base(dataset);
-    }
-
-    std::cout<<"Knowledge base size: "<<patterns.size()<<'\n'
-             <<"Unique token types: "<<token_dict.text.size()<<'\n'
-             <<"Chunk size: "<<config.chunk_size<<" bytes\n"
-             <<"Top-K: "<<config.top_k<<'\n'
-             <<"Maximum generated tokens: "<<config.max_response_tokens<<'\n'
-             <<"OpenMP threads available: "<<omp_get_max_threads()<<'\n';
-
-    if(!cmd.save_kb.empty()){
-        std::cerr<<"Saving knowledge base: "<<cmd.save_kb<<'\n';
-        if(!save_knowledge_base(cmd.save_kb))return 1;
-    }
-    if(!cmd.retrieve.empty()){retrieve_only(cmd.retrieve);return 0;}
-    if(!cmd.prompt.empty()){process_prompt(cmd.prompt);return 0;}
-    if(cmd.chat){chat_loop();return 0;}
-    std::cout<<(patterns.empty()?"\nNo knowledge base loaded. Use --learn, --load-kb, or --help.\n":"\nKnowledge base loaded successfully. Use --retrieve, --prompt, or --chat.\n");
-    return 0;
-}