Delete pattern_constructor_q.cpp

pattern_constructor_q.cpp

@@ -1,1296 +0,0 @@
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <cstdint>
-#include <cmath>
-#include <deque>
-#include <cstdio>
-#include <fstream>
-#include <iostream>
-#include <limits>
-#include <numeric>
-#include <stdexcept>
-#include <string>
-#include <string_view>
-#include <tuple>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-#ifdef _OPENMP
-#include <omp.h>
-#else
-static int omp_get_max_threads(){ return 1; }
-static int omp_get_thread_num(){ return 0; }
-#endif
-
-// ============================================================================
-// Configuration
-// ============================================================================
-struct Config {
-    std::size_t max_response_tokens = 32;
-    std::size_t top_k = 3;
-    std::size_t chunk_size = 8192;
-    bool show_retrieval = false;
-    bool chat_mode = false;
-};
-static Config config;
-
-// ============================================================================
-// Character helpers
-// ============================================================================
-static inline unsigned char uchar(char c){ return static_cast<unsigned char>(c); }
-static inline bool is_alpha(char c){ return std::isalpha(uchar(c)) != 0; }
-static inline bool is_digit(char c){ return std::isdigit(uchar(c)) != 0; }
-static inline bool is_alnum(char c){ return std::isalnum(uchar(c)) != 0; }
-static inline bool is_space(char c){ return std::isspace(uchar(c)) != 0; }
-static inline bool is_lower(char c){ return std::islower(uchar(c)) != 0; }
-static inline bool is_upper(char c){ return std::isupper(uchar(c)) != 0; }
-
-// ============================================================================
-// Deterministic hashes
-// ============================================================================
-static constexpr std::uint64_t FNV_OFFSET = 14695981039346656037ULL;
-static constexpr std::uint64_t FNV_PRIME  = 1099511628211ULL;
-
-static std::uint64_t fnv1a64(std::string_view s){
-    std::uint64_t h = FNV_OFFSET;
-    for(unsigned char c : s){ h ^= c; h *= FNV_PRIME; }
-    return h;
-}
-
-static std::uint64_t mix64(std::uint64_t x){
-    x += 0x9E3779B97F4A7C15ULL;
-    x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL;
-    x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL;
-    return x ^ (x >> 31);
-}
-
-static std::uint64_t combine_hashes(std::uint64_t a,std::uint64_t b){
-    return mix64(a ^ (mix64(b)+0x9E3779B97F4A7C15ULL+(a<<6)+(a>>2)));
-}
-
-// ============================================================================
-// Compact string pool (memory optimization)
-// ============================================================================
-struct StringRef { std::uint32_t offset=0, length=0; };
-
-struct StringPool {
-    std::string bytes;
-    void clear(){ bytes.clear(); }
-    std::uint32_t append(std::string_view s){
-        const std::uint32_t off=static_cast<std::uint32_t>(bytes.size());
-        bytes.append(s);
-        return off;
-    }
-    std::string_view view(std::uint32_t offset,std::uint32_t length) const {
-        if(static_cast<std::size_t>(offset)+length>bytes.size()) return {};
-        return std::string_view(bytes.data()+offset,length);
-    }
-};
-static StringPool string_pool;
-
-// ============================================================================
-// Integer token dictionary
-// ============================================================================
-struct TokenHash {
-    std::size_t operator()(std::string_view s) const noexcept {
-        return static_cast<std::size_t>(fnv1a64(s));
-    }
-};
-
-struct TokenDictionary {
-    std::deque<std::string> text;
-    std::unordered_map<std::string_view,std::uint32_t,TokenHash> to_id;
-
-    void clear(){ text.clear(); to_id.clear(); }
-
-    std::uint32_t intern(std::string_view s){
-        auto it=to_id.find(s);
-        if(it!=to_id.end()) return it->second;
-        const std::uint32_t id=static_cast<std::uint32_t>(text.size());
-        text.emplace_back(s);
-        to_id.emplace(std::string_view(text.back()),id);
-        return id;
-    }
-
-    std::string_view view(std::uint32_t id) const {
-        return id<text.size()?std::string_view(text[id]):std::string_view{};
-    }
-};
-static TokenDictionary token_dict;
-
-// ============================================================================
-// Tokenization
-// ============================================================================
-struct RawToken {
-    std::string_view text;
-    std::string_view space_before;
-    std::size_t start=0,end=0;
-};
-
-static bool is_identifier_char(char c){ return is_alnum(c) || c=='_'; }
-
-static std::vector<RawToken> tokenize_views(std::string_view text){
-    std::vector<RawToken> out;
-    out.reserve(text.size()/4+1);
-    std::size_t i=0,space_begin=0;
-    while(i<text.size()){
-        if(is_space(text[i])){ ++i; continue; }
-        const std::size_t start=i;
-        if(is_identifier_char(text[i])){
-            while(i<text.size() && is_identifier_char(text[i])) ++i;
-        } else {
-            ++i;
-        }
-        out.push_back({text.substr(start,i-start),text.substr(space_begin,start-space_begin),start,i});
-        space_begin=i;
-    }
-    return out;
-}
-
-// ============================================================================
-// Structural classes
-// ============================================================================
-enum class TKind : std::uint8_t { EMPTY, PUNCT, MIXNUM, NUM, MIX, UPP, LOW, OTHER };
-
-enum class ShapeKind : std::uint8_t {
-    EMPTY=0, ALPHA=1, DIGIT=2, ALNUM=3, SPACE=4, PUNCT=5,
-    ALPHA_DIGIT=6, DIGIT_ALPHA=7, MIXED_ALPHA=8, OTHER=9
-};
-
-static TKind token_kind(std::string_view token){
-    if(token.empty()) return TKind::EMPTY;
-    bool alpha=false,digit=false,lower=false,upper=false;
-    for(char c:token){
-        if(is_alpha(c)){ alpha=true; lower|=is_lower(c); upper|=is_upper(c); }
-        else if(is_digit(c)) digit=true;
-    }
-    if(!is_alnum(token.front()) && token.front()!='_') return TKind::PUNCT;
-    if(alpha&&digit) return TKind::MIXNUM;
-    if(digit) return TKind::NUM;
-    if(upper&&lower) return TKind::MIX;
-    if(upper) return TKind::UPP;
-    if(lower) return TKind::LOW;
-    return TKind::OTHER;
-}
-
-static bool variable_candidate(TKind k){
-    return k==TKind::LOW || k==TKind::UPP || k==TKind::MIX || k==TKind::MIXNUM || k==TKind::NUM;
-}
-
-static ShapeKind shape_kind(std::string_view token){
-    if(token.empty()) return ShapeKind::EMPTY;
-    bool alpha=false,digit=false,space=false,punct=false,lower=false,upper=false;
-    for(char c:token){
-        alpha|=is_alpha(c); digit|=is_digit(c); space|=is_space(c); punct|=!is_alnum(c)&&!is_space(c);
-        lower|=is_lower(c); upper|=is_upper(c);
-    }
-    if(punct && !alpha && !digit) return ShapeKind::PUNCT;
-    if(space && !alpha && !digit && !punct) return ShapeKind::SPACE;
-    if(alpha&&digit){
-        bool first_alpha=is_alpha(token.front()), first_digit=is_digit(token.front());
-        return first_alpha ? ShapeKind::ALPHA_DIGIT : (first_digit ? ShapeKind::DIGIT_ALPHA : ShapeKind::ALNUM);
-    }
-    if(alpha){
-        if(lower&&upper) return ShapeKind::MIXED_ALPHA;
-        return ShapeKind::ALPHA;
-    }
-    if(digit) return ShapeKind::DIGIT;
-    return ShapeKind::OTHER;
-}
-
-static std::uint64_t morphology_hash(std::string_view token){
-    std::uint64_t h=FNV_OFFSET;
-    for(char c:token){
-        std::uint8_t k=0;
-        if(is_alpha(c)) k=is_lower(c)?1:2;
-        else if(is_digit(c)) k=3;
-        else if(is_space(c)) k=4;
-        else k=5;
-        h ^= k; h *= FNV_PRIME;
-    }
-    h=combine_hashes(h,static_cast<std::uint64_t>(token.size()));
-    return h;
-}
-
-static std::uint64_t transition_hash(std::string_view token){
-    std::uint64_t h=FNV_OFFSET;
-    std::uint8_t prev=0;
-    bool first=true;
-    for(char c:token){
-        std::uint8_t k=0;
-        if(is_alpha(c)) k=is_lower(c)?1:2;
-        else if(is_digit(c)) k=3;
-        else if(is_space(c)) k=4;
-        else k=5;
-        if(first){ h=combine_hashes(h,k); first=false; }
-        else { h=combine_hashes(h,(static_cast<std::uint64_t>(prev)<<8)|k); }
-        prev=k;
-    }
-    return combine_hashes(h,token.size());
-}
-
-// ============================================================================
-// Profiles
-// ============================================================================
-struct Profile {
-    float distinct_ratio=0.0f;
-    float alpha_ratio=0.0f;
-    float num_ratio=0.0f;
-    float punct_ratio=0.0f;
-    float space_ratio=0.0f;
-};
-
-static Profile extract_profile(std::string_view text){
-    std::array<std::uint8_t,256> seen{};
-    std::size_t distinct=0,alpha=0,num=0,punct=0,space=0;
-    for(char c:text){
-        const auto u=uchar(c);
-        if(!seen[u]){seen[u]=1;++distinct;}
-        if(is_alpha(c)) ++alpha;
-        else if(is_digit(c)) ++num;
-        else if(is_space(c)) ++space;
-        else ++punct;
-    }
-    if(text.empty()) return {};
-    const float total=static_cast<float>(text.size());
-    return {static_cast<float>(distinct)/total,
-            static_cast<float>(alpha)/total,
-            static_cast<float>(num)/total,
-            static_cast<float>(punct)/total,
-            static_cast<float>(space)/total};
-}
-
-static float profile_distance(const Profile&a,const Profile&b){
-    return std::fabs(a.distinct_ratio-b.distinct_ratio)+
-           std::fabs(a.alpha_ratio-b.alpha_ratio)+
-           std::fabs(a.num_ratio-b.num_ratio)+
-           std::fabs(a.punct_ratio-b.punct_ratio)+
-           std::fabs(a.space_ratio-b.space_ratio);
-}
-
-// ============================================================================
-// Generic CSR index
-// ============================================================================
-struct CSRIndex {
-    std::vector<std::uint64_t> keys;
-    std::vector<std::uint32_t> offsets;
-    std::vector<std::uint32_t> postings;
-    void clear(){keys.clear();offsets.clear();postings.clear();}
-    std::pair<const std::uint32_t*,std::size_t> find(std::uint64_t key) const {
-        auto it=std::lower_bound(keys.begin(),keys.end(),key);
-        if(it==keys.end()||*it!=key) return {nullptr,0};
-        const std::size_t idx=static_cast<std::size_t>(it-keys.begin());
-        const std::size_t a=offsets[idx],b=offsets[idx+1];
-        return {postings.data()+a,b-a};
-    }
-};
-
-struct BuildIndex {
-    std::unordered_map<std::uint64_t,std::vector<std::uint32_t>> postings;
-    void clear(){postings.clear();}
-};
-
-static void merge_index(BuildIndex& dst,BuildIndex& src){
-    for(auto& kv:src.postings){
-        auto& v=dst.postings[kv.first];
-        v.insert(v.end(),kv.second.begin(),kv.second.end());
-    }
-}
-
-static void compile_csr(const BuildIndex& src,CSRIndex& dst){
-    std::vector<std::uint64_t> keys;
-    keys.reserve(src.postings.size());
-    for(const auto& kv:src.postings) keys.push_back(kv.first);
-    std::sort(keys.begin(),keys.end());
-    dst.clear();
-    dst.keys=std::move(keys);
-    dst.offsets.reserve(dst.keys.size()+1);
-    std::size_t total=0;
-    for(std::uint64_t k:dst.keys) total+=src.postings.at(k).size();
-    dst.postings.reserve(total);
-    dst.offsets.push_back(0);
-    for(std::uint64_t k:dst.keys){
-        const auto& v=src.postings.at(k);
-        dst.postings.insert(dst.postings.end(),v.begin(),v.end());
-        dst.offsets.push_back(static_cast<std::uint32_t>(dst.postings.size()));
-    }
-}
-
-// ============================================================================
-// Pattern graph / relations
-// ============================================================================
-struct Edge {
-    std::uint32_t from=0,to=0;
-    std::uint8_t type=0;
-};
-
-struct Relation {
-    std::uint32_t a=0,b=0;
-    std::int32_t distance=0;
-    std::uint8_t type=0;
-};
-
-// ============================================================================
-// Canonical and structural pattern representation
-// ============================================================================
-struct Pattern {
-    StringRef text;
-    Profile profile;
-    std::vector<std::uint32_t> token_ids;
-    std::vector<std::uint32_t> canonical_ids;
-    std::vector<std::uint8_t> kinds;
-    std::vector<std::uint8_t> shapes;
-    std::vector<std::uint64_t> token_morph_hashes;
-    std::vector<std::uint64_t> token_transition_hashes;
-    std::vector<Relation> relations;
-    std::vector<Edge> graph;
-    std::uint64_t canonical_hash=0;
-    std::uint64_t graph_hash=0;
-    std::uint64_t relation_hash=0;
-    std::uint64_t transition_hash=0;
-    std::uint32_t observations=1;
-    std::uint32_t structural_variants=1;
-    std::uint32_t exact_matches=0;
-    std::uint32_t parent=std::numeric_limits<std::uint32_t>::max();
-    std::vector<std::uint32_t> children;
-};
-
-static std::vector<std::uint32_t> canonicalize(const std::vector<std::uint32_t>& ids){
-    std::unordered_map<std::uint32_t,std::uint32_t> renumber;
-    renumber.reserve(ids.size()*2+1);
-    std::vector<std::uint32_t> out(ids.size());
-    std::uint32_t next=0;
-    for(std::size_t i=0;i<ids.size();++i){
-        auto it=renumber.find(ids[i]);
-        if(it==renumber.end()) it=renumber.emplace(ids[i],next++).first;
-        out[i]=it->second;
-    }
-    return out;
-}
-
-static std::uint64_t canonical_hash(const std::vector<std::uint32_t>& c,
-                                    const std::vector<std::uint8_t>& k){
-    std::uint64_t h=FNV_OFFSET;
-    for(std::size_t i=0;i<c.size();++i){
-        h=combine_hashes(h,c[i]);
-        h=combine_hashes(h,k[i]);
-    }
-    return h;
-}
-
-static std::vector<Relation> build_relations(const std::vector<std::uint32_t>& c){
-    std::vector<Relation> r;
-    const std::size_t n=c.size();
-    if(n<2) return r;
-    r.reserve(n*2);
-    for(std::size_t i=0;i<n;++i){
-        for(std::size_t j=i+1;j<n;++j){
-            if(c[i]==c[j]) r.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),static_cast<std::int32_t>(j-i),0});
-        }
-    }
-    for(std::size_t i=0;i+1<n;++i)
-        r.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(i+1),1,1});
-    std::sort(r.begin(),r.end(),[](const Relation&a,const Relation&b){
-        return std::tie(a.type,a.distance,a.a,a.b)<std::tie(b.type,b.distance,b.a,b.b);
-    });
-    return r;
-}
-
-static std::uint64_t relation_hash(const std::vector<Relation>& r){
-    std::uint64_t h=FNV_OFFSET;
-    for(const auto& x:r){
-        h=combine_hashes(h,x.type);
-        h=combine_hashes(h,x.a);
-        h=combine_hashes(h,x.b);
-        h=combine_hashes(h,static_cast<std::uint32_t>(x.distance));
-    }
-    return h;
-}
-
-static std::vector<Edge> build_graph(const std::vector<std::uint32_t>& c){
-    std::vector<Edge> g;
-    if(c.size()<2) return g;
-    g.reserve(c.size()*3);
-    for(std::size_t i=0;i+1<c.size();++i) g.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(i+1),0});
-    for(std::size_t i=0;i<c.size();++i){
-        for(std::size_t j=i+1;j<c.size();++j){
-            if(c[i]==c[j]) g.push_back({static_cast<std::uint32_t>(i),static_cast<std::uint32_t>(j),1});
-        }
-    }
-    return g;
-}
-
-static std::uint64_t graph_hash(const std::vector<Edge>& g){
-    std::uint64_t h=FNV_OFFSET;
-    for(const auto&e:g){h=combine_hashes(h,e.from);h=combine_hashes(h,e.to);h=combine_hashes(h,e.type);}
-    return h;
-}
-
-static std::uint64_t sequence_transition_hash(const std::vector<std::uint64_t>& h){
-    std::uint64_t x=FNV_OFFSET;
-    for(std::uint64_t v:h) x=combine_hashes(x,v);
-    return x;
-}
-
-// ============================================================================
-// Generic local token analysis
-// ============================================================================
-static void analyze_token_sequence(std::string_view text,
-                                   std::vector<std::uint32_t>& ids,
-                                   std::vector<std::uint8_t>& kinds,
-                                   std::vector<std::uint8_t>& shapes,
-                                   std::vector<std::uint64_t>& morph,
-                                   std::vector<std::uint64_t>& transitions){
-    auto raw=tokenize_views(text);
-    ids.resize(raw.size()); kinds.resize(raw.size()); shapes.resize(raw.size());
-    morph.resize(raw.size()); transitions.resize(raw.size());
-    for(std::size_t i=0;i<raw.size();++i){
-        ids[i]=token_dict.intern(raw[i].text);
-        kinds[i]=static_cast<std::uint8_t>(token_kind(raw[i].text));
-        shapes[i]=static_cast<std::uint8_t>(shape_kind(raw[i].text));
-        morph[i]=morphology_hash(raw[i].text);
-        transitions[i]=transition_hash(raw[i].text);
-    }
-}
-
-static std::vector<std::uint32_t> longest_common_structure(const std::vector<std::uint32_t>& a,
-                                                            const std::vector<std::uint32_t>& b){
-    if(a.empty()||b.empty()) return {};
-    if(a.size()*b.size()>2000000ULL) return {};
-    std::vector<std::uint32_t> prev(b.size()+1),cur(b.size()+1);
-    for(std::size_t i=1;i<=a.size();++i){
-        for(std::size_t j=1;j<=b.size();++j)
-            cur[j]=(a[i-1]==b[j-1])?prev[j-1]+1:std::max(prev[j],cur[j-1]);
-        prev.swap(cur); std::fill(cur.begin(),cur.end(),0);
-    }
-    std::vector<std::uint32_t> out(prev.back());
-    std::iota(out.begin(),out.end(),0u);
-    return out;
-}
-
-// ============================================================================
-// Pattern construction
-// ============================================================================
-static Pattern construct_pattern(const std::string& text,StringRef ref){
-    Pattern p;
-    p.text=ref;
-    p.profile=extract_profile(text);
-    analyze_token_sequence(text,p.token_ids,p.kinds,p.shapes,p.token_morph_hashes,p.token_transition_hashes);
-    p.canonical_ids=canonicalize(p.token_ids);
-    p.canonical_hash=canonical_hash(p.canonical_ids,p.kinds);
-    p.relations=build_relations(p.canonical_ids);
-    p.relation_hash=relation_hash(p.relations);
-    p.graph=build_graph(p.canonical_ids);
-    p.graph_hash=graph_hash(p.graph);
-    p.transition_hash=sequence_transition_hash(p.token_transition_hashes);
-    return p;
-}
-
-// ============================================================================
-// Efficient Suffix Arrays for variable-length prefix lookups
-// ============================================================================
-enum class SAType : std::uint8_t { TEXT=0, CANONICAL=1, SHAPE=2 };
-
-struct SAElem {
-    std::uint32_t pat_id=0;
-    std::uint32_t offset=0;
-};
-
-static std::vector<SAElem> text_sa;
-static std::vector<SAElem> canonical_sa;
-static std::vector<SAElem> shape_sa;
-
-static std::size_t sa_sequence_size(const Pattern& p,SAType type){
-    if(type==SAType::TEXT) return p.token_ids.size();
-    if(type==SAType::CANONICAL) return p.canonical_ids.size();
-    return p.shapes.size();
-}
-
-static std::uint32_t sa_sequence_value(const Pattern& p,SAType type,std::size_t i){
-    if(type==SAType::TEXT) return p.token_ids[i];
-    if(type==SAType::CANONICAL) return p.canonical_ids[i];
-    return p.shapes[i];
-}
-
-static void build_suffix_array(const std::vector<Pattern>& pats,std::vector<SAElem>& sa,SAType type){
-    std::size_t total=0;
-    for(const Pattern& p:pats) total+=sa_sequence_size(p,type);
-    sa.clear(); sa.resize(total);
-    std::unordered_map<std::uint32_t,std::vector<SAElem>> buckets;
-    buckets.reserve(total?std::min<std::size_t>(total,1u<<20):0);
-    for(std::uint32_t id=0;id<pats.size();++id){
-        const Pattern& p=pats[id];
-        const std::size_t n=sa_sequence_size(p,type);
-        for(std::uint32_t off=0;off<n;++off) buckets[sa_sequence_value(p,type,off)].push_back({id,off});
-    }
-    std::vector<std::vector<SAElem>*> bucket_ptrs;
-    bucket_ptrs.reserve(buckets.size());
-    for(auto& kv:buckets) bucket_ptrs.push_back(&kv.second);
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic)
-#endif
-    for(int i=0;i<static_cast<int>(bucket_ptrs.size());++i){
-        auto* b=bucket_ptrs[static_cast<std::size_t>(i)];
-        std::sort(b->begin(),b->end(),[&](const SAElem&a,const SAElem&z){
-            const Pattern&pa=pats[a.pat_id],&pb=pats[z.pat_id];
-            const std::size_t na=sa_sequence_size(pa,type)-a.offset,nz=sa_sequence_size(pb,type)-z.offset,m=std::min(na,nz);
-            for(std::size_t k=0;k<m;++k){
-                const std::uint32_t va=sa_sequence_value(pa,type,a.offset+k),vz=sa_sequence_value(pb,type,z.offset+k);
-                if(va!=vz) return va<vz;
-            }
-            if(na!=nz) return na<nz;
-            return a.pat_id<z.pat_id || (a.pat_id==z.pat_id && a.offset<z.offset);
-        });
-    }
-    std::vector<std::uint32_t> keys;
-    keys.reserve(buckets.size());
-    for(const auto& kv:buckets) keys.push_back(kv.first);
-    std::sort(keys.begin(),keys.end());
-    std::size_t out=0;
-    for(std::uint32_t key:keys){
-        const auto& b=buckets.find(key)->second;
-        std::copy(b.begin(),b.end(),sa.begin()+static_cast<std::ptrdiff_t>(out));
-        out+=b.size();
-    }
-}
-
-static std::tuple<std::size_t,std::size_t,std::size_t> search_sa_longest(const std::vector<SAElem>&sa,const std::vector<Pattern>&pats,const std::vector<std::uint32_t>&query,std::size_t query_offset,SAType type){
-    if(query_offset>=query.size()||sa.empty()) return {0,0,0};
-    const std::size_t query_size=query.size()-query_offset;
-    auto cmp_lower=[&](const SAElem&a,const std::vector<std::uint32_t>&q){
-        const Pattern&p=pats[a.pat_id];const std::size_t n=sa_sequence_size(p,type)-a.offset,m=std::min(n,query_size);
-        for(std::size_t k=0;k<m;++k){const std::uint32_t v=sa_sequence_value(p,type,a.offset+k);if(v!=q[query_offset+k])return v<q[query_offset+k];}
-        return n<query_size;
-    };
-    auto it_low=std::lower_bound(sa.begin(),sa.end(),query,cmp_lower);
-    auto lcp_at=[&](auto it){
-        if(it==sa.end())return std::size_t(0);
-        const Pattern&p=pats[it->pat_id];const std::size_t n=sa_sequence_size(p,type)-it->offset,m=std::min(n,query_size);std::size_t k=0;
-        while(k<m&&sa_sequence_value(p,type,it->offset+k)==query[query_offset+k]) ++k;
-        return k;
-    };
-    const std::size_t lcp_low=lcp_at(it_low),lcp_prev=(it_low!=sa.begin())?lcp_at(it_low-1):0,max_lcp=std::max(lcp_low,lcp_prev);
-    if(!max_lcp)return {0,0,0};
-    auto cmp_prefix_lower=[&](const SAElem&a,const std::vector<std::uint32_t>&q){
-        const Pattern&p=pats[a.pat_id];const std::size_t n=sa_sequence_size(p,type)-a.offset,m=std::min(n,max_lcp);
-        for(std::size_t k=0;k<m;++k){const std::uint32_t v=sa_sequence_value(p,type,a.offset+k);if(v!=q[query_offset+k])return v<q[query_offset+k];}
-        return n<max_lcp;
-    };
-    auto cmp_prefix_upper=[&](const std::vector<std::uint32_t>&q,const SAElem&a){
-        const Pattern&p=pats[a.pat_id];const std::size_t m=std::min(sa_sequence_size(p,type)-a.offset,max_lcp);
-        for(std::size_t k=0;k<m;++k){const std::uint32_t v=sa_sequence_value(p,type,a.offset+k);if(q[query_offset+k]!=v)return q[query_offset+k]<v;}
-        return false;
-    };
-    auto first=std::lower_bound(sa.begin(),sa.end(),query,cmp_prefix_lower),last=std::upper_bound(first,sa.end(),query,cmp_prefix_upper);
-    return {static_cast<std::size_t>(first-sa.begin()),static_cast<std::size_t>(last-sa.begin()),max_lcp};
-}
-
-// ============================================================================
-// Global indexes
-// ============================================================================
-static std::vector<Pattern> patterns;
-static CSRIndex morph_csr,transition_csr,relation_csr,graph_csr;
-static std::unordered_map<std::uint64_t,std::uint32_t> canonical_pattern_map;
-
-static void build_pattern_indexes(){
-    BuildIndex morph,transition,relation,graph;
-    canonical_pattern_map.clear();
-    const std::size_t n=patterns.size();
-    const int threads=std::max(1,omp_get_max_threads());
-    std::vector<BuildIndex> locals_morph(static_cast<std::size_t>(threads)),locals_transition(static_cast<std::size_t>(threads)),locals_rel(static_cast<std::size_t>(threads)),locals_graph(static_cast<std::size_t>(threads));
-#ifdef _OPENMP
-#pragma omp parallel
-#endif
-    {
-        const int tid=omp_get_thread_num();
-        BuildIndex&lm=locals_morph[static_cast<std::size_t>(tid)],&lt=locals_transition[static_cast<std::size_t>(tid)],&lr=locals_rel[static_cast<std::size_t>(tid)],&lg=locals_graph[static_cast<std::size_t>(tid)];
-#ifdef _OPENMP
-#pragma omp for schedule(dynamic,16)
-#endif
-        for(int ii=0;ii<static_cast<int>(n);++ii){
-            const std::uint32_t id=static_cast<std::uint32_t>(ii); const Pattern&p=patterns[id];
-            for(std::uint64_t h:p.token_morph_hashes) lm.postings[h].push_back(id);
-            if(p.transition_hash) lt.postings[p.transition_hash].push_back(id);
-            if(p.relation_hash) lr.postings[p.relation_hash].push_back(id);
-            if(p.graph_hash) lg.postings[p.graph_hash].push_back(id);
-        }
-    }
-    for(auto&x:locals_morph) merge_index(morph,x);
-    for(auto&x:locals_transition) merge_index(transition,x);
-    for(auto&x:locals_rel) merge_index(relation,x);
-    for(auto&x:locals_graph) merge_index(graph,x);
-    compile_csr(morph,morph_csr); compile_csr(transition,transition_csr); compile_csr(relation,relation_csr); compile_csr(graph,graph_csr);
-#ifdef _OPENMP
-#pragma omp parallel sections
-#endif
-    {
-#ifdef _OPENMP
-#pragma omp section
-#endif
-        build_suffix_array(patterns,text_sa,SAType::TEXT);
-#ifdef _OPENMP
-#pragma omp section
-#endif
-        build_suffix_array(patterns,canonical_sa,SAType::CANONICAL);
-#ifdef _OPENMP
-#pragma omp section
-#endif
-        build_suffix_array(patterns,shape_sa,SAType::SHAPE);
-    }
-    canonical_pattern_map.reserve(n*2+1);
-    for(std::uint32_t i=0;i<n;++i) canonical_pattern_map.emplace(patterns[i].canonical_hash,i);
-}
-
-// ============================================================================
-// Input cleaning and file loading
-// ============================================================================
-static bool read_file(const std::string& filename,std::string& output){
-    std::ifstream is(filename.c_str(),std::ios::binary|std::ios::ate);
-    if(!is) return false;
-    const std::streampos end=is.tellg();
-    if(end<0) return false;
-    output.clear(); output.resize(static_cast<std::size_t>(end));
-    is.seekg(0,std::ios::beg);
-    return output.empty() || static_cast<bool>(is.read(output.data(),static_cast<std::streamsize>(output.size())));
-}
-
-static void split_text(const std::string& text,std::size_t chunk_size,std::vector<std::string>& out){
-    if(text.empty()) return;
-    if(!chunk_size) chunk_size=8192;
-    std::string current; current.reserve(chunk_size);
-    std::size_t line_start=0;
-    while(line_start<text.size()){
-        std::size_t line_end=text.find('\n',line_start);
-        if(line_end==std::string::npos) line_end=text.size();
-        std::string line=text.substr(line_start,line_end-line_start);
-        if(line_end<text.size()) line.push_back('\n');
-        if(line.size()>chunk_size){
-            if(!current.empty()){out.push_back(std::move(current));current.clear();current.reserve(chunk_size);}
-            for(std::size_t pos=0;pos<line.size();){
-                const std::size_t take=std::min(chunk_size,line.size()-pos);
-                out.push_back(line.substr(pos,take)); pos+=take;
-            }
-        } else {
-            if(!current.empty()&&current.size()+line.size()>chunk_size){out.push_back(std::move(current));current.clear();current.reserve(chunk_size);}
-            current+=line;
-        }
-        if(line_end==text.size()) break;
-        line_start=line_end+1;
-    }
-    if(!current.empty()) out.push_back(std::move(current));
-}
-
-static bool load_learning_file(const std::string& filename,std::vector<std::string>& dataset){
-    std::string text;
-    if(!read_file(filename,text)){std::cerr<<"Could not read file: "<<filename<<'\n';return false;}
-    if(text.empty()) return false;
-    const std::size_t before=dataset.size();
-    split_text(text,config.chunk_size,dataset);
-    std::cerr<<"Learn source: "<<filename<<" -> "<<dataset.size()-before<<" pattern(s)\n";
-    return dataset.size()>before;
-}
-
-static bool load_learning_files(const std::vector<std::string>& files,std::vector<std::string>& dataset){
-    if(files.empty()) return false;
-    std::vector<std::vector<std::string>> local(files.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic)
-#endif
-    for(int i=0;i<static_cast<int>(files.size());++i)
-        load_learning_file(files[static_cast<std::size_t>(i)],local[static_cast<std::size_t>(i)]);
-    std::size_t total=dataset.size();
-    for(const auto&v:local) total+=v.size();
-    dataset.reserve(total);
-    for(auto&v:local) dataset.insert(dataset.end(),std::make_move_iterator(v.begin()),std::make_move_iterator(v.end()));
-    return !dataset.empty();
-}
-
-static void deduplicate(std::vector<std::string>& data){
-    if(data.size()<2) return;
-    std::unordered_set<std::string> seen; seen.reserve(data.size()*2);
-    std::vector<std::string> unique; unique.reserve(data.size());
-    for(std::string&s:data) if(seen.insert(s).second) unique.push_back(std::move(s));
-    data.swap(unique);
-}
-
-// ============================================================================
-// Pattern merging / specialization tree
-// ============================================================================
-static std::uint32_t invalid_pattern_id(){ return std::numeric_limits<std::uint32_t>::max(); }
-
-static void insert_pattern_incremental(Pattern&& candidate,std::vector<Pattern>&store,std::unordered_map<std::uint64_t,std::uint32_t>&map){
-    auto it=map.find(candidate.canonical_hash);
-    if(it==map.end()){
-        const std::uint32_t id=static_cast<std::uint32_t>(store.size());
-        map.emplace(candidate.canonical_hash,id);
-        store.push_back(std::move(candidate));
-        return;
-    }
-    Pattern& q=store[it->second];
-    const auto common=longest_common_structure(q.canonical_ids,candidate.canonical_ids);
-    q.observations+=candidate.observations;
-    q.structural_variants+=static_cast<std::uint32_t>(std::max<std::size_t>(1,common.size()));
-    q.exact_matches+=candidate.exact_matches;
-    if(profile_distance(candidate.profile,{})<profile_distance(q.profile,{})) q.profile=candidate.profile;
-    for(std::size_t i=0;i<q.token_morph_hashes.size()&&i<candidate.token_morph_hashes.size();++i)
-        if(q.token_morph_hashes[i]!=candidate.token_morph_hashes[i]) q.exact_matches++;
-}
-
-static void merge_equivalent_patterns(){
-    if(patterns.empty()) return;
-    std::vector<Pattern> merged; merged.reserve(patterns.size());
-    canonical_pattern_map.clear(); canonical_pattern_map.reserve(patterns.size()*2+1);
-    for(Pattern&p:patterns) insert_pattern_incremental(std::move(p),merged,canonical_pattern_map);
-    patterns.swap(merged);
-}
-
-static std::vector<std::uint32_t> generalized_canonical(const Pattern&p){
-    if(p.canonical_ids.empty()) return {};
-    std::vector<std::uint32_t> g=p.canonical_ids;
-    const std::uint32_t target=g.back();
-    g.back()=0;
-    if(target==0) return g;
-    std::unordered_map<std::uint32_t,std::uint32_t> remap; remap.reserve(g.size()*2+1);
-    std::uint32_t next=0;
-    for(std::uint32_t&x:g){auto it=remap.find(x);if(it==remap.end())it=remap.emplace(x,next++).first;x=it->second;}
-    return g;
-}
-
-static void build_specialization_tree(){
-    if(patterns.empty()) return;
-    for(Pattern&p:patterns){p.parent=invalid_pattern_id();p.children.clear();}
-    if(canonical_pattern_map.size()!=patterns.size()){canonical_pattern_map.clear();canonical_pattern_map.reserve(patterns.size()*2+1);for(std::uint32_t i=0;i<patterns.size();++i)canonical_pattern_map.emplace(patterns[i].canonical_hash,i);}
-    for(std::uint32_t i=0;i<patterns.size();++i){
-        const Pattern&p=patterns[i]; const std::vector<std::uint32_t> g=generalized_canonical(p); if(g.empty()) continue;
-        const std::uint64_t h=canonical_hash(g,p.kinds);
-        auto it=canonical_pattern_map.find(h);
-        if(it!=canonical_pattern_map.end()&&it->second!=i){patterns[i].parent=it->second;patterns[it->second].children.push_back(i);}
-    }
-}
-
-static void expand_specialization_candidates(const std::vector<std::uint32_t>& seeds,std::vector<std::uint32_t>& ids){
-    if(seeds.empty()||patterns.empty()) return;
-    std::unordered_set<std::uint32_t> seen;seen.reserve(seeds.size()*4+1);
-    std::vector<std::uint32_t> frontier;frontier.reserve(seeds.size()*2+1);
-    for(std::uint32_t id:seeds) if(id<patterns.size()&&seen.insert(id).second) frontier.push_back(id);
-    constexpr unsigned max_depth=2;
-    for(unsigned depth=0;depth<max_depth&&!frontier.empty();++depth){
-        std::vector<std::uint32_t> next;next.reserve(frontier.size()*2+1);
-        for(std::uint32_t id:frontier){
-            const Pattern&p=patterns[id];
-            if(p.parent<patterns.size()&&seen.insert(p.parent).second) next.push_back(p.parent);
-            for(std::uint32_t child:p.children) if(child<patterns.size()&&seen.insert(child).second) next.push_back(child);
-        }
-        ids.insert(ids.end(),next.begin(),next.end());frontier.swap(next);
-    }
-}
-
-static void build_knowledge_base(const std::vector<std::string>& dataset){
-    patterns.clear(); string_pool.clear(); token_dict.clear();
-    morph_csr.clear(); transition_csr.clear(); relation_csr.clear(); graph_csr.clear();
-    text_sa.clear(); canonical_sa.clear(); shape_sa.clear();
-
-    if(dataset.empty()) return;
-
-    std::vector<StringRef> refs(dataset.size());
-    string_pool.bytes.reserve(std::accumulate(dataset.begin(),dataset.end(),std::size_t(0),[](std::size_t n,const std::string&s){return n+s.size();}));
-    std::unordered_set<std::string_view,TokenHash> unique_tokens;
-    unique_tokens.reserve(dataset.size()*8+1);
-    for(std::size_t i=0;i<dataset.size();++i){
-        refs[i]={string_pool.append(dataset[i]),static_cast<std::uint32_t>(dataset[i].size())};
-        const auto raw=tokenize_views(dataset[i]);
-        for(const auto&t:raw) unique_tokens.insert(t.text);
-    }
-    token_dict.to_id.reserve(unique_tokens.size()*2+1);
-    for(const auto&t:unique_tokens) token_dict.intern(t);
-
-    patterns.resize(dataset.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(dynamic,16)
-#endif
-    for(int i=0;i<static_cast<int>(dataset.size());++i)
-        patterns[static_cast<std::size_t>(i)]=construct_pattern(dataset[static_cast<std::size_t>(i)],refs[static_cast<std::size_t>(i)]);
-    merge_equivalent_patterns();
-    build_specialization_tree();
-    build_pattern_indexes();
-}
-
-// ============================================================================
-// Query representation
-// ============================================================================
-struct Query {
-    std::string text;
-    std::vector<std::uint32_t> token_ids;
-    std::vector<std::uint8_t> kinds;
-    std::vector<std::uint8_t> shapes;
-    std::vector<std::uint64_t> morph;
-    std::vector<std::uint64_t> transitions;
-    std::vector<std::uint32_t> canonical_ids;
-    Profile profile;
-    std::uint64_t relation_hash=0;
-    std::uint64_t graph_hash=0;
-    std::uint64_t transition_hash_value=0;
-};
-
-static Query make_query(const std::string& text){
-    Query q; q.text=text;
-    analyze_token_sequence(text,q.token_ids,q.kinds,q.shapes,q.morph,q.transitions);
-    q.canonical_ids=canonicalize(q.token_ids);
-    q.profile=extract_profile(text);
-    q.relation_hash=relation_hash(build_relations(q.canonical_ids));
-    q.graph_hash=graph_hash(build_graph(q.canonical_ids));
-    q.transition_hash_value=sequence_transition_hash(q.transitions);
-    return q;
-}
-
-// ============================================================================
-// Deterministic scoring
-// ============================================================================
-struct MatchScore {
-    std::uint32_t exact=0,structural=0,relational=0,contextual=0,profile=0,length=0,observations=0,id=0;
-};
-
-static bool better_score(const MatchScore&a,const MatchScore&b){
-    return std::tie(a.exact,a.structural,a.relational,a.contextual,a.profile,a.length,a.observations)>std::tie(b.exact,b.structural,b.relational,b.contextual,b.profile,b.length,b.observations)||
-        (a.exact==b.exact&&a.structural==b.structural&&a.relational==b.relational&&a.contextual==b.contextual&&a.profile==b.profile&&a.length==b.length&&a.observations==b.observations&&a.id<b.id);
-}
-
-static std::uint32_t profile_score(const Profile&a,const Profile&b){
-    const double d=profile_distance(a,b); return static_cast<std::uint32_t>(std::max(0.0,1000000.0*(1.0-d)));
-}
-
-static std::uint32_t sequence_exact_prefix(const std::vector<std::uint32_t>&a,const std::vector<std::uint32_t>&b){
-    const std::size_t n=std::min(a.size(),b.size()); std::uint32_t x=0; for(std::size_t i=0;i<n;++i){if(a[i]!=b[i])break;++x;} return x;
-}
-
-static std::uint32_t common_structure_count(const std::vector<std::uint32_t>&a,const std::vector<std::uint32_t>&b){
-    const std::size_t n=std::min(a.size(),b.size()); std::uint32_t x=0; for(std::size_t i=0;i<n;++i)if(a[i]==b[i])++x; return x;
-}
-
-struct CandidateAccumulator {
-    std::vector<std::uint32_t> stamp;
-    std::vector<MatchScore> score;
-    std::vector<std::uint32_t> touched;
-    std::vector<std::uint32_t> anchor_pattern,anchor_query,anchor_lcp;
-    std::uint32_t epoch=0;
-    void prepare(std::size_t n){
-        if(stamp.size()<n){stamp.resize(n,0);score.resize(n);anchor_pattern.resize(n);anchor_query.resize(n);anchor_lcp.resize(n);}
-        if(++epoch==0){std::fill(stamp.begin(),stamp.end(),0);epoch=1;}
-        touched.clear();
-    }
-    MatchScore& touch(std::uint32_t id){
-        if(stamp[id]!=epoch){stamp[id]=epoch;score[id]={0,0,0,0,0,0,0,id};anchor_pattern[id]=anchor_query[id]=anchor_lcp[id]=0;touched.push_back(id);}
-        return score[id];
-    }
-    void anchor(std::uint32_t id,std::size_t pattern_off,std::size_t query_off,std::size_t lcp,bool exact_channel){
-        touch(id);
-        if(lcp>anchor_lcp[id] || (lcp==anchor_lcp[id]&&exact_channel)){anchor_lcp[id]=static_cast<std::uint32_t>(lcp);anchor_pattern[id]=static_cast<std::uint32_t>(pattern_off);anchor_query[id]=static_cast<std::uint32_t>(query_off);}
-    }
-};
-
-static void add_csr_structural_hits(const CSRIndex&idx,const std::vector<std::uint64_t>&keys,CandidateAccumulator&c){
-    for(std::uint64_t key:keys){auto [ptr,count]=idx.find(key);for(std::size_t i=0;i<count;++i)++c.touch(ptr[i]).structural;}
-}
-
-static std::uint32_t consecutive_equal_left(const std::vector<std::uint32_t>&a,std::size_t ai,const std::vector<std::uint32_t>&b,std::size_t bi){
-    std::uint32_t n=0;while(ai&&bi&&a[ai-1]==b[bi-1]){--ai;--bi;++n;}return n;
-}
-
-static std::uint32_t consecutive_equal_right(const std::vector<std::uint32_t>&a,std::size_t ai,const std::vector<std::uint32_t>&b,std::size_t bi){
-    const std::size_t na=a.size(),nb=b.size();std::uint32_t n=0;while(ai<na&&bi<nb&&a[ai]==b[bi]){++ai;++bi;++n;}return n;
-}
-
-static std::uint32_t structural_context_score(const Query&q,const Pattern&p,std::size_t qo,std::size_t po,std::size_t lcp){
-    const std::size_t ql=std::min(qo,q.canonical_ids.size()),pl=std::min(po,p.canonical_ids.size());
-    const std::uint32_t left=consecutive_equal_left(q.canonical_ids,ql,p.canonical_ids,pl);
-    const std::size_t qr=std::min(q.canonical_ids.size(),qo+lcp),pr=std::min(p.canonical_ids.size(),po+lcp);
-    const std::uint32_t right=consecutive_equal_right(q.canonical_ids,qr,p.canonical_ids,pr);
-    const std::size_t qden=std::max<std::size_t>(1,q.canonical_ids.size()?q.canonical_ids.size()-1:1),pden=std::max<std::size_t>(1,p.canonical_ids.size()?p.canonical_ids.size()-1:1);
-    const std::uint64_t qpos=(1000000ULL*qo)/qden,ppos=(1000000ULL*po)/pden;
-    const std::uint32_t position=static_cast<std::uint32_t>(1000000ULL-(qpos>ppos?qpos-ppos:ppos-qpos));
-    return left*left+right*right+position;
-}
-
-static int shape_distance(std::uint8_t a,std::uint8_t b){ return a==b?0:1; }
-
-static int substitution_level(const Pattern&p,std::size_t pi,const Query&q,std::size_t qi){
-    const std::uint32_t s=p.token_ids[pi],t=q.token_ids[qi];
-    if(s==t) return 3;
-    const TKind ks=token_kind(token_dict.view(s)),kt=token_kind(token_dict.view(t));
-    if(variable_candidate(ks)&&variable_candidate(kt)&&ks==kt) return 3;
-    if(variable_candidate(ks)&&variable_candidate(kt)&&shape_distance(p.shapes[pi],q.shapes[qi])==0) return 2;
-    if(!variable_candidate(ks)||!variable_candidate(kt)) return -1;
-    // Structural role: same edge status and same repetition cardinality.
-    auto role=[&](const std::vector<std::uint32_t>&v,std::size_t i){
-        std::uint32_t count=0;for(std::uint32_t x:v)if(x==v[i])++count;
-        const std::uint8_t edge=static_cast<std::uint8_t>((i==0?1:0)|(i+1==v.size()?2:0));
-        return std::pair<std::uint32_t,std::uint8_t>{count,edge};
-    };
-    return role(p.token_ids,pi)==role(q.token_ids,qi)?1:-1;
-}
-
-struct BindingState {
-    std::unordered_map<std::uint32_t,std::uint32_t> forward;
-    std::unordered_set<std::uint32_t> used_targets;
-    void reserve(std::size_t n){forward.reserve(n*2+1);used_targets.reserve(n*2+1);}
-};
-
-static bool add_binding(BindingState&b,std::uint32_t s,std::uint32_t t){
-    if(s==t) return true;
-    auto it=b.forward.find(s);
-    if(it!=b.forward.end()) return it->second==t;
-    if(!b.used_targets.insert(t).second) return false;
-    b.forward.emplace(s,t);return true;
-}
-
-struct Alignment {
-    bool valid=false;std::size_t position=0;BindingState bindings;std::uint32_t exact=0,structural=0,context=0;
-};
-
-static bool align_at(const Pattern&p,const Query&q,std::size_t pos,Alignment&out){
-    if(pos+q.token_ids.size()>p.token_ids.size()) return false;
-    Alignment a;a.valid=true;a.position=pos;a.bindings.reserve(q.token_ids.size());
-    for(std::size_t i=0;i<q.token_ids.size();++i){
-        const int level=substitution_level(p,pos+i,q,i);
-        if(level<0) return false;
-        if(level==3&&p.token_ids[pos+i]==q.token_ids[i]) ++a.exact;
-        else a.structural+=static_cast<std::uint32_t>(level);
-        if(p.token_ids[pos+i]!=q.token_ids[i]&&!add_binding(a.bindings,p.token_ids[pos+i],q.token_ids[i])) return false;
-    }
-    a.context=static_cast<std::uint32_t>(std::min(pos,p.token_ids.size()-pos-q.token_ids.size()));
-    out=std::move(a);return true;
-}
-
-static bool better_alignment(const Alignment&a,const Alignment&b){
-    if(a.exact!=b.exact)return a.exact>b.exact;
-    if(a.structural!=b.structural)return a.structural>b.structural;
-    if(a.context!=b.context)return a.context>b.context;
-    return a.position<b.position;
-}
-
-static bool infer_best_alignment(const Pattern&p,const Query&q,Alignment&best){
-    if(q.token_ids.empty()||p.token_ids.size()<q.token_ids.size()) return false;
-    std::size_t pivot=0;int selectivity=-1;
-    for(std::size_t i=0;i<q.token_ids.size();++i){const TKind k=token_kind(token_dict.view(q.token_ids[i]));const int s=(k==TKind::PUNCT?4:(variable_candidate(k)?1:3));if(s>selectivity){selectivity=s;pivot=i;}}
-    bool found=false;
-    for(std::size_t s=pivot;s<p.token_ids.size();++s){const int level=substitution_level(p,s,q,pivot);if(level<0)continue;const std::size_t pos=s-pivot;if(pos+q.token_ids.size()>p.token_ids.size())continue;Alignment a;if(align_at(p,q,pos,a)&&(!found||better_alignment(a,best))){best=std::move(a);found=true;}}
-    if(!found){
-        // Shape-aware second pass for unseen token IDs.
-        for(std::size_t s=pivot;s<p.token_ids.size();++s){if(p.shapes[s]!=q.shapes[pivot])continue;const std::size_t pos=s-pivot;if(pos+q.token_ids.size()>p.token_ids.size())continue;Alignment a;if(align_at(p,q,pos,a)&&(!found||better_alignment(a,best))){best=std::move(a);found=true;}}
-    }
-    return found;
-}
-
-static std::vector<MatchScore> search_matches(const Query&q,std::size_t k){
-    if(patterns.empty()||q.token_ids.empty()||!k) return {};
-    CandidateAccumulator acc;acc.prepare(patterns.size());
-    auto collect=[&](const std::vector<SAElem>&sa,SAType type,const std::vector<std::uint32_t>&query,bool exact_channel){
-        for(std::size_t qi=0;qi<query.size();++qi){
-            auto [start,end,lcp]=search_sa_longest(sa,patterns,query,qi,type);
-            for(std::size_t j=start;j<end;++j){const SAElem&e=sa[j];if(exact_channel)acc.touch(e.pat_id).exact+=static_cast<std::uint32_t>(lcp*lcp);else acc.touch(e.pat_id).structural+=static_cast<std::uint32_t>(lcp*lcp);acc.anchor(e.pat_id,e.offset,qi,lcp,exact_channel);}
-        }
-    };
-    collect(text_sa,SAType::TEXT,q.token_ids,true);
-    collect(canonical_sa,SAType::CANONICAL,q.canonical_ids,false);
-    const std::vector<std::uint32_t> query_shapes(q.shapes.begin(),q.shapes.end());
-    collect(shape_sa,SAType::SHAPE,query_shapes,false);
-    add_csr_structural_hits(morph_csr,q.morph,acc);
-    if(q.transition_hash_value){auto [p,n]=transition_csr.find(q.transition_hash_value);for(std::size_t i=0;i<n;++i)++acc.touch(p[i]).structural;}
-    if(q.relation_hash){auto [p,n]=relation_csr.find(q.relation_hash);for(std::size_t i=0;i<n;++i)++acc.touch(p[i]).relational;}
-    if(q.graph_hash){auto [p,n]=graph_csr.find(q.graph_hash);for(std::size_t i=0;i<n;++i)++acc.touch(p[i]).structural;}
-    const std::vector<std::uint32_t> seeds=acc.touched;
-    std::vector<std::uint32_t> candidate_ids=seeds;
-    expand_specialization_candidates(seeds,candidate_ids);
-    for(std::uint32_t id:candidate_ids) acc.touch(id);
-    std::vector<MatchScore> tmp(acc.touched.size());
-#ifdef _OPENMP
-#pragma omp parallel for schedule(static)
-#endif
-    for(int i=0;i<static_cast<int>(acc.touched.size());++i){
-        const std::uint32_t id=acc.touched[static_cast<std::size_t>(i)];MatchScore&s=acc.score[id];const Pattern&p=patterns[id];
-        s.exact+=sequence_exact_prefix(q.token_ids,p.token_ids);s.structural+=common_structure_count(q.canonical_ids,p.canonical_ids);s.relational+=static_cast<std::uint32_t>(std::min<std::size_t>(p.relations.size(),q.canonical_ids.size()));s.profile=profile_score(q.profile,p.profile);s.length=static_cast<std::uint32_t>(std::min(q.token_ids.size(),p.token_ids.size()));s.observations=p.observations;
-        if(id<acc.anchor_pattern.size()&&acc.anchor_lcp[id]) s.contextual=structural_context_score(q,p,acc.anchor_query[id],acc.anchor_pattern[id],acc.anchor_lcp[id]);
-        tmp[static_cast<std::size_t>(i)]=s;
-    }
-    if(tmp.size()>k){std::nth_element(tmp.begin(),tmp.begin()+static_cast<std::ptrdiff_t>(k),tmp.end(),better_score);tmp.resize(k);}
-    std::sort(tmp.begin(),tmp.end(),better_score);return tmp;
-}
-
-// ============================================================================
-// Rendering without repeated tokenization
-// ============================================================================
-static std::string render_suffix(const Pattern&p,std::size_t begin,const BindingState&bindings,std::size_t max_tokens){
-    if(begin>=p.token_ids.size()) return {};
-    const auto raw=tokenize_views(string_pool.view(p.text.offset,p.text.length));
-    const std::size_t end=std::min(p.token_ids.size(),begin+max_tokens);
-    std::size_t reserve=0;
-    for(std::size_t i=begin;i<end;++i) reserve+=raw[i].space_before.size()+raw[i].text.size();
-    std::string result; result.reserve(reserve);
-    for(std::size_t i=begin;i<end;++i){
-        result.append(raw[i].space_before);
-        auto it=bindings.forward.find(p.token_ids[i]);
-        if(it!=bindings.forward.end()){
-            std::string src(token_dict.view(p.token_ids[i]));
-            std::string tgt(token_dict.view(it->second));
-
-            std::cout << "\n[Approval Required] Apply variable binding '" << src << "' -> '" << tgt << "'? (y/n): ";
-            std::string ans;
-            if (std::getline(std::cin, ans) && (ans == "y" || ans == "Y")) {
-                result.append(tgt);
-            } else {
-                result.append(raw[i].text);
-            }
-        }
-        else {
-            result.append(raw[i].text);
-        }
-    }
-    return result;
-}
-
-static std::string produce_completion(const Query&q,const std::vector<MatchScore>&matches){
-    for(const auto&m:matches){
-        const Pattern&p=patterns[m.id];
-        Alignment a;
-        if(!infer_best_alignment(p,q,a)) continue;
-        const std::size_t begin=a.position+q.token_ids.size();
-        if(begin>=p.token_ids.size()) continue;
-        std::string suffix=render_suffix(p,begin,a.bindings,config.max_response_tokens);
-        std::string output=q.text;
-        output.reserve(q.text.size()+suffix.size());
-        output+=suffix;
-        return output;
-    }
-    // Dynamic variable-length suffix fallback utilizing Suffix Array LCP
-    if(!q.token_ids.empty()){
-        for (std::size_t i = 0; i < q.token_ids.size(); ++i) {
-            auto [start,end,lcp]=search_sa_longest(text_sa,patterns,q.token_ids,i,SAType::TEXT);
-            if(lcp==q.token_ids.size()-i){
-                for(std::size_t j = start; j < end; ++j){
-                    const Pattern& p = patterns[text_sa[j].pat_id];
-                    Alignment a;
-                    if(!infer_best_alignment(p, q, a)) continue;
-                    const std::size_t begin = a.position + q.token_ids.size();
-                    if(begin >= p.token_ids.size()) continue;
-                    std::string suffix = render_suffix(p, begin, a.bindings, config.max_response_tokens);
-                    if(!suffix.empty()){
-                        std::string output = q.text;
-                        output += suffix;
-                        return output;
-                    }
-                }
-            }
-        }
-    }
-    return {};
-}
-
-// ============================================================================
-// Binary persistence
-// ============================================================================
-static constexpr std::uint64_t KB_MAGIC=0x5041545445524E55ULL;
-static constexpr std::uint64_t KB_VERSION=12ULL;
-
-static void write_varuint(std::ostream&os,std::uint64_t v){while(v>=0x80){os.put(static_cast<char>((v&0x7F)|0x80));v>>=7;}os.put(static_cast<char>(v));if(!os)throw std::runtime_error("KB write failed");}
-static std::uint64_t read_varuint(std::istream&is){std::uint64_t v=0;unsigned shift=0;for(;;){unsigned char c=0;if(!is.read(reinterpret_cast<char*>(&c),1))throw std::runtime_error("truncated KB");v|=static_cast<std::uint64_t>(c&0x7F)<<shift;if(!(c&0x80))return v;shift+=7;if(shift>=64)throw std::runtime_error("invalid KB varint");}}
-static void write_float(std::ostream&os,float x){os.write(reinterpret_cast<const char*>(&x),sizeof(x));if(!os)throw std::runtime_error("KB write failed");}
-static float read_float(std::istream&is){float x=0;if(!is.read(reinterpret_cast<char*>(&x),sizeof(x)))throw std::runtime_error("truncated KB");return x;}
-static void write_string(std::ostream&os,std::string_view s){write_varuint(os,s.size());if(!s.empty())os.write(s.data(),static_cast<std::streamsize>(s.size()));}
-static std::string read_string(std::istream&is){const auto n=read_varuint(is);if(n>1024ULL*1024ULL*1024ULL)throw std::runtime_error("KB string too large");std::string s(static_cast<std::size_t>(n),'\0');if(n&&!is.read(s.data(),static_cast<std::streamsize>(n)))throw std::runtime_error("truncated KB");return s;}
-
-static void save_u32_vector(std::ostream&os,const std::vector<std::uint32_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u32_vector(std::istream&is,std::vector<std::uint32_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=static_cast<std::uint32_t>(read_varuint(is));}
-static void save_u8_vector(std::ostream&os,const std::vector<std::uint8_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u8_vector(std::istream&is,std::vector<std::uint8_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=static_cast<std::uint8_t>(read_varuint(is));}
-static void save_u64_vector(std::ostream&os,const std::vector<std::uint64_t>&v){write_varuint(os,v.size());for(auto x:v)write_varuint(os,x);}
-static void load_u64_vector(std::istream&is,std::vector<std::uint64_t>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v)x=read_varuint(is);}
-static void save_relations(std::ostream&os,const std::vector<Relation>&v){write_varuint(os,v.size());for(auto x:v){write_varuint(os,x.a);write_varuint(os,x.b);write_varuint(os,static_cast<std::uint32_t>(x.distance));write_varuint(os,x.type);}}
-static void load_relations(std::istream&is,std::vector<Relation>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v){x.a=static_cast<std::uint32_t>(read_varuint(is));x.b=static_cast<std::uint32_t>(read_varuint(is));x.distance=static_cast<std::int32_t>(read_varuint(is));x.type=static_cast<std::uint8_t>(read_varuint(is));}}
-static void save_edges(std::ostream&os,const std::vector<Edge>&v){write_varuint(os,v.size());for(auto x:v){write_varuint(os,x.from);write_varuint(os,x.to);write_varuint(os,x.type);}}
-static void load_edges(std::istream&is,std::vector<Edge>&v){v.resize(static_cast<std::size_t>(read_varuint(is)));for(auto&x:v){x.from=static_cast<std::uint32_t>(read_varuint(is));x.to=static_cast<std::uint32_t>(read_varuint(is));x.type=static_cast<std::uint8_t>(read_varuint(is));}}
-
-static bool save_knowledge_base(const std::string&filename){
-    const std::string tmp=filename+".tmp";
-    try{
-        std::ofstream os(tmp.c_str(),std::ios::binary|std::ios::trunc);
-        if(!os)throw std::runtime_error("cannot open temporary KB");
-        write_varuint(os,KB_MAGIC);write_varuint(os,KB_VERSION);write_varuint(os,config.chunk_size);
-        write_varuint(os,token_dict.text.size());
-        for(const auto&s:token_dict.text) write_string(os,s);
-        write_varuint(os,patterns.size());
-        for(const auto&p:patterns){
-            write_string(os,string_pool.view(p.text.offset,p.text.length));
-            write_float(os,p.profile.distinct_ratio);write_float(os,p.profile.alpha_ratio);write_float(os,p.profile.num_ratio);write_float(os,p.profile.punct_ratio);write_float(os,p.profile.space_ratio);
-            save_u32_vector(os,p.token_ids);save_u32_vector(os,p.canonical_ids);save_u8_vector(os,p.kinds);save_u8_vector(os,p.shapes);save_u64_vector(os,p.token_morph_hashes);save_u64_vector(os,p.token_transition_hashes);
-            save_relations(os,p.relations);save_edges(os,p.graph);
-            write_varuint(os,p.canonical_hash);write_varuint(os,p.graph_hash);write_varuint(os,p.relation_hash);write_varuint(os,p.transition_hash);write_varuint(os,p.observations);write_varuint(os,p.structural_variants);write_varuint(os,p.exact_matches);
-        }
-        os.flush();if(!os)throw std::runtime_error("KB write failed");os.close();std::remove(filename.c_str());
-        if(std::rename(tmp.c_str(),filename.c_str())!=0){std::remove(tmp.c_str());throw std::runtime_error("KB rename failed");}
-        return true;
-    }catch(const std::exception&e){std::cerr<<"Save error: "<<e.what()<<'\n';std::remove(tmp.c_str());return false;}
-}
-
-static bool load_knowledge_base(const std::string&filename){
-    try{
-        std::ifstream is(filename.c_str(),std::ios::binary);if(!is)throw std::runtime_error("cannot open KB");
-        if(read_varuint(is)!=KB_MAGIC)throw std::runtime_error("invalid KB magic");
-        if(read_varuint(is)!=KB_VERSION)throw std::runtime_error("unsupported KB version");
-        config.chunk_size=static_cast<std::size_t>(read_varuint(is));
-        token_dict.clear();string_pool.clear();patterns.clear();
-        const std::size_t token_count=static_cast<std::size_t>(read_varuint(is));token_dict.to_id.reserve(token_count*2+1);
-        for(std::size_t i=0;i<token_count;++i) token_dict.intern(read_string(is));
-        const std::size_t count=static_cast<std::size_t>(read_varuint(is));if(count>100000000ULL)throw std::runtime_error("unreasonable KB pattern count");
-        patterns.resize(count);
-        for(Pattern&p:patterns){
-            const std::string s=read_string(is);const std::uint32_t off=string_pool.append(s);p.text={off,static_cast<std::uint32_t>(s.size())};
-            p.profile.distinct_ratio=read_float(is);p.profile.alpha_ratio=read_float(is);p.profile.num_ratio=read_float(is);p.profile.punct_ratio=read_float(is);p.profile.space_ratio=read_float(is);
-            load_u32_vector(is,p.token_ids);load_u32_vector(is,p.canonical_ids);load_u8_vector(is,p.kinds);load_u8_vector(is,p.shapes);load_u64_vector(is,p.token_morph_hashes);load_u64_vector(is,p.token_transition_hashes);
-            load_relations(is,p.relations);load_edges(is,p.graph);
-            p.canonical_hash=read_varuint(is);p.graph_hash=read_varuint(is);p.relation_hash=read_varuint(is);p.transition_hash=read_varuint(is);p.observations=static_cast<std::uint32_t>(read_varuint(is));p.structural_variants=static_cast<std::uint32_t>(read_varuint(is));p.exact_matches=static_cast<std::uint32_t>(read_varuint(is));
-        }
-        build_pattern_indexes();build_specialization_tree();
-        std::cerr<<"Loaded "<<patterns.size()<<" pattern(s), "<<token_dict.text.size()<<" token type(s)\n";
-        return true;
-    }catch(const std::exception&e){std::cerr<<"Load error: "<<e.what()<<'\n';return false;}
-}
-
-static bool print_kb_info(const std::string&filename){
-    try{std::ifstream is(filename.c_str(),std::ios::binary);if(!is)throw std::runtime_error("cannot open KB");if(read_varuint(is)!=KB_MAGIC)throw std::runtime_error("invalid KB magic");
-        const auto version=read_varuint(is),chunk=read_varuint(is);const auto tokens=read_varuint(is);std::cout<<"KB: "<<filename<<"\nVersion: "<<version<<"\nChunk size: "<<chunk<<"\nToken types: "<<tokens<<'\n';return true;
-    }catch(const std::exception&e){std::cerr<<"KB info error: "<<e.what()<<'\n';return false;}
-}
-
-// ============================================================================
-// CLI
-// ============================================================================
-struct CommandLine{std::vector<std::string>learn_files;std::string load_kb,save_kb,info_kb,prompt,retrieve;bool help=false,chat=false,show_retrieval=false;};
-
-static bool parse_size(const std::string&s,std::size_t&out){try{std::size_t pos=0;const auto n=std::stoull(s,&pos,10);if(pos!=s.size()||!n)return false;out=static_cast<std::size_t>(n);return true;}catch(...){return false;}}
-
-static void print_help(const char* program) {
-    std::cout << "\n" << program << "\n"
-              << "================================\n\n"
-              << "Learning:\n"
-              << "  --learn FILE [FILE ...]  Ingest text files into the knowledge base.\n"
-              << "  --chunk-size N           Bytes per processed chunk.\n\n"
-              << "Knowledge base:\n"
-              << "  --save-kb FILE           Save the current knowledge base to disk.\n"
-              << "  --load-kb FILE           Load an existing knowledge base.\n"
-              << "  --info-kb FILE           Display metadata about a saved knowledge base.\n\n"
-              << "Retrieval:\n"
-              << "  --top-k N                Number of matching archetypes to retrieve.\n"
-              << "  --retrieve TEXT          Retrieve and display top-k patterns for a prompt.\n"
-              << "  --show-retrieval         Print retrieved patterns before generating a response.\n\n"
-              << "Interaction:\n"
-              << "  --max-length N           Maximum number of tokens to generate.\n"
-              << "  --prompt TEXT            Generate a single response to the provided text.\n"
-              << "  --chat                   Interactive chat mode.\n"
-              << "                             * Multiline input; terminate with /end;\n"
-              << "                             * Exit with /quit\n\n"
-              << "Other:\n"
-              << "  --help                   Display this help menu.\n\n";
-}
-
-static bool parse_command_line(int argc,char**argv,CommandLine&cmd){
-    for(int i=1;i<argc;++i){
-        const std::string a=argv[i];
-        if(a=="--help"||a=="-h"){cmd.help=true;continue;}
-        if(a=="--chat"){cmd.chat=true;continue;}
-        if(a=="--show-retrieval"){cmd.show_retrieval=true;continue;}
-        if(a=="--learn"){if(i+1>=argc)return false;while(i+1<argc&&argv[i+1][0]!='-')cmd.learn_files.push_back(argv[++i]);continue;}
-        if(a=="--load-kb"){if(i+1>=argc)return false;cmd.load_kb=argv[++i];continue;}
-        if(a=="--save-kb"){if(i+1>=argc)return false;cmd.save_kb=argv[++i];continue;}
-        if(a=="--info-kb"){if(i+1>=argc)return false;cmd.info_kb=argv[++i];continue;}
-        if(a=="--retrieve"){if(i+1>=argc)return false;cmd.retrieve=argv[++i];continue;}
-        if(a=="--prompt"){if(i+1>=argc)return false;cmd.prompt=argv[++i];continue;}
-        if(a=="--top-k"){if(i+1>=argc||!parse_size(argv[++i],config.top_k))return false;continue;}
-        if(a=="--max-length"){if(i+1>=argc||!parse_size(argv[++i],config.max_response_tokens))return false;continue;}
-        if(a=="--chunk-size"){if(i+1>=argc||!parse_size(argv[++i],config.chunk_size))return false;continue;}
-        std::cerr<<"Unknown option: "<<a<<'\n';return false;
-    }
-    return true;
-}
-
-static void print_retrieval(const std::vector<MatchScore>&results){
-    std::cout<<"\nRetrieved structural patterns:\n";
-    if(results.empty()){std::cout<<"  None\n";return;}
-    for(std::size_t i=0;i<results.size();++i){
-        const auto&m=results[i];
-        std::cout<<"\nRank "<<i+1<<"\nExact: "<<m.exact<<"\nStructural: "<<m.structural<<"\nRelational: "<<m.relational<<"\nContextual: "<<m.contextual<<"\nProfile: "<<m.profile<<"\nObservations: "<<m.observations<<"\nPattern:\n"<<string_pool.view(patterns[m.id].text.offset,patterns[m.id].text.length)<<'\n';
-    }
-}
-
-static void process_prompt(const std::string&prompt){
-    if(prompt.empty()||patterns.empty())return;
-    const Query q=make_query(prompt);
-    const auto results=search_matches(q,config.top_k);
-    if(config.show_retrieval)print_retrieval(results);
-    const std::string response=produce_completion(q,results);
-    std::cout<<(response.empty()?"No deterministic completion found.\n":response+'\n');
-}
-
-static void retrieve_only(const std::string&query){if(patterns.empty())return;print_retrieval(search_matches(make_query(query),config.top_k));}
-
-static void chat_loop(){
-    std::cout<<"\nPattern Constructor Chat\n====================================\nFinish prompt with /end\nExit with /quit\n\n";
-    std::string prompt,line;
-    while(true){
-        std::cout<<"> "<<std::flush;
-        if(!std::getline(std::cin,line))break;
-        if(line=="/quit")break;
-        if(line=="/end"){if(!prompt.empty()){process_prompt(prompt);std::cout<<'\n';}prompt.clear();continue;}
-        if(!prompt.empty()) prompt.push_back('\n');
-        prompt+=line;
-    }
-}
-
-int main(int argc,char**argv){
-#ifdef _WIN32
-    SetConsoleCP(65001);SetConsoleOutputCP(65001);
-#endif
-    CommandLine cmd;
-    if(!parse_command_line(argc,argv,cmd)){print_help(argv[0]);return 1;}
-    if(cmd.help){print_help(argv[0]);return 0;}
-    config.show_retrieval=cmd.show_retrieval;
-    config.chat_mode=cmd.chat;
-    if(!cmd.info_kb.empty()){return print_kb_info(cmd.info_kb)?0:1;}
-
-    if(!cmd.load_kb.empty()){
-        std::cerr<<"Loading knowledge base: "<<cmd.load_kb<<'\n';
-        if(!load_knowledge_base(cmd.load_kb))return 1;
-    }
-
-    if(!cmd.learn_files.empty()){
-        std::vector<std::string> dataset;
-        load_learning_files(cmd.learn_files,dataset);
-        const std::size_t before=dataset.size();
-        deduplicate(dataset);
-        std::cerr<<"Removed "<<before-dataset.size()<<" duplicate pattern(s)\n";
-        build_knowledge_base(dataset);
-    }
-
-    std::cout<<"Knowledge base size: "<<patterns.size()<<'\n'
-             <<"Unique token types: "<<token_dict.text.size()<<'\n'
-             <<"Chunk size: "<<config.chunk_size<<" bytes\n"
-             <<"Top-K: "<<config.top_k<<'\n'
-             <<"Maximum generated tokens: "<<config.max_response_tokens<<'\n'
-             <<"OpenMP threads available: "<<omp_get_max_threads()<<'\n';
-
-    if(!cmd.save_kb.empty()){
-        std::cerr<<"Saving knowledge base: "<<cmd.save_kb<<'\n';
-        if(!save_knowledge_base(cmd.save_kb))return 1;
-    }
-    if(!cmd.retrieve.empty()){retrieve_only(cmd.retrieve);return 0;}
-    if(!cmd.prompt.empty()){process_prompt(cmd.prompt);return 0;}
-    if(cmd.chat){chat_loop();return 0;}
-    std::cout<<(patterns.empty()?"\nNo knowledge base loaded. Use --learn, --load-kb, or --help.\n":"\nKnowledge base loaded successfully. Use --retrieve, --prompt, or --chat.\n");
-    return 0;
-}