Upload 2 files

ChatIPC.cpp

@@ -1,7 +1,3 @@
-// chatipc_modular.cpp
-// Compile: g++ -std=c++17 -O2 -fopenmp -o chatipc_modular chatipc_modular.cpp
-// Requires dictionary.cpp providing: extern unsigned char dictionary_json[]; extern unsigned int dictionary_json_len;
-
 #include <algorithm>
 #include <atomic>
 #include <cctype>
@@ -74,8 +70,15 @@ struct StringInterner {
     }
 };
 
-// --------------------------- Knowledge base (short methods) --------------
+// ---------- Global parsed dictionary (populated once in main) ----------
+static std::unordered_map<std::string,std::string> g_raw_dict;
+
+static std::unordered_set<std::string> def_tokens_from_text(const std::string &s){
+    auto toks = tokenize_non_alnum(s);
+    return std::unordered_set<std::string>(toks.begin(), toks.end());
+}
 
+// --------------------------- Knowledge base (short methods) --------------
 using StrPtr = const std::string*;
 struct PtrHash { size_t operator()(StrPtr p) const noexcept { return std::hash<std::string>()(*p); } };
 struct PtrEq   { bool operator()(StrPtr a, StrPtr b) const noexcept { return *a == *b; } };
@@ -86,17 +89,91 @@ struct KnowledgeBase {
     StringInterner interner;
     std::unordered_map<StrPtr, NextSet, PtrHash, PtrEq> next;
     std::mutex m;
+
+    // def-index: for each interned word pointer -> list of interned tokens (definition expansion)
+    std::unordered_map<StrPtr, std::vector<StrPtr>, PtrHash, PtrEq> def_index;
+    std::mutex def_m;
+    int def_depth = 0;
+
     void add_pair_interned(StrPtr k, StrPtr v){
         std::lock_guard<std::mutex> lk(m);
         auto &vec = next[k];
         for (auto p : vec) if (*p == *v) return;
         vec.push_back(v);
     }
+
+    // set def depth; if changed, drop previously computed def expansions
+    void set_def_depth(int D){
+        std::lock_guard<std::mutex> lk(def_m);
+        if (D != def_depth){
+            def_index.clear();
+            def_depth = D;
+        }
+    }
+
+    // compute definition expansion for a single interned word (if needed)
+    void ensure_def_for_interned(StrPtr wp){
+        // quick no-op checks
+        if (wp == nullptr) return;
+        if (def_depth <= 0) return;
+
+        // double-checked locking
+        {
+            std::lock_guard<std::mutex> lk(def_m);
+            if (def_index.find(wp) != def_index.end()) return;
+        }
+
+        // compute expansion using global parsed dictionary g_raw_dict
+        std::unordered_set<std::string> acc;
+        std::vector<std::string> frontier;
+        auto it_raw = g_raw_dict.find(*wp);
+        if (it_raw != g_raw_dict.end()){
+            auto toks = def_tokens_from_text(it_raw->second);
+            for (auto &t : toks){
+                if (acc.insert(t).second) frontier.push_back(t);
+            }
+        }
+
+        for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
+            std::vector<std::string> nextf;
+            for (auto &w : frontier){
+                auto it2 = g_raw_dict.find(w);
+                if (it2 == g_raw_dict.end()) continue;
+                auto toks2 = def_tokens_from_text(it2->second);
+                for (auto &t : toks2){
+                    if (acc.insert(t).second) nextf.push_back(t);
+                }
+            }
+            frontier.swap(nextf);
+        }
+
+        // intern all accumulated tokens and store pointers
+        std::vector<StrPtr> out;
+        out.reserve(acc.size());
+        for (auto &s : acc){
+            out.push_back(interner.intern(s));
+        }
+
+        // store atomically (prevent double insertion)
+        {
+            std::lock_guard<std::mutex> lk(def_m);
+            // another thread may have inserted meanwhile; do not overwrite
+            if (def_index.find(wp) == def_index.end()){
+                def_index.emplace(wp, std::move(out));
+            }
+        }
+    }
+
+    // existing public add_pair but now ensure def-expansion is built immediately
     void add_pair(const std::string &k, const std::string &v){
         StrPtr kp = interner.intern(k);
         StrPtr vp = interner.intern(v);
+        // ensure definition expansion for both words as soon as they are seen
+        ensure_def_for_interned(kp);
+        ensure_def_for_interned(vp);
         add_pair_interned(kp, vp);
     }
+
     std::optional<NextSet> lookup_by_string(const std::string &k) const {
         for (auto &pr : next) if (*pr.first == k) return pr.second;
         return std::nullopt;
@@ -108,6 +185,21 @@ struct KnowledgeBase {
     }
 };
 
+// thread-safe snapshot of kb.def_index as string-based def-index
+static std::unordered_map<std::string,std::unordered_set<std::string>>
+snapshot_def_index(KnowledgeBase &kb){
+    std::unordered_map<std::string,std::unordered_set<std::string>> out;
+    std::lock_guard<std::mutex> lk(kb.def_m);
+    out.reserve(kb.def_index.size());
+    for (auto &pr : kb.def_index){
+        std::unordered_set<std::string> s;
+        s.reserve(pr.second.size());
+        for (auto p : pr.second) s.insert(*p);
+        out.emplace(*pr.first, std::move(s));
+    }
+    return out;
+}
+
 // --------------------------- Small JSON parse helpers ----------------------
 
 static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
@@ -168,51 +260,6 @@ static std::unordered_map<std::string,std::string> parse_dictionary_json(){
     return dict;
 }
 
-// --------------------------- Build definition index (small funcs) ---------
-
-static std::unordered_set<std::string> def_tokens_from_text(const std::string &s){
-    auto toks = tokenize_non_alnum(s);
-    return std::unordered_set<std::string>(toks.begin(), toks.end());
-}
-
-static void expand_def_index(const std::unordered_map<std::string,std::unordered_set<std::string>> &direct,
-                             std::unordered_map<std::string,std::unordered_set<std::string>> &out,
-                             int depth)
-{
-    for (auto &pr : direct){
-        const std::string &word = pr.first;
-        std::unordered_set<std::string> acc = pr.second;
-        if (depth > 1){
-            std::vector<std::string> frontier(acc.begin(), acc.end());
-            for (int d=1; d<depth; ++d){
-                std::vector<std::string> nextf;
-                for (auto &w : frontier){
-                    auto it = direct.find(w);
-                    if (it==direct.end()) continue;
-                    for (auto &t : it->second){
-                        if (acc.insert(t).second) nextf.push_back(t);
-                    }
-                }
-                if (nextf.empty()) break;
-                frontier.swap(nextf);
-            }
-        }
-        out.emplace(word, std::move(acc));
-    }
-}
-
-static std::unordered_map<std::string,std::unordered_set<std::string>>
-build_definition_index(int depth)
-{
-    std::unordered_map<std::string,std::unordered_set<std::string>> out;
-    if (depth <= 0) return out;
-    auto raw = parse_dictionary_json();
-    std::unordered_map<std::string,std::unordered_set<std::string>> direct;
-    for (auto &pr : raw) direct.emplace(pr.first, def_tokens_from_text(pr.second));
-    expand_def_index(direct, out, depth);
-    return out;
-}
-
 // --------------------------- Similarity helpers (very small) ----------------
 
 static double jaccard_similarity(const std::unordered_set<std::string> &A,
@@ -379,13 +426,15 @@ static void learn_files_parallel(KnowledgeBase &kb, const std::vector<std::strin
 
 // --------------------------- Serialization (short functions) ----------------
 
-// File format documented in comments near functions
 static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
     std::ofstream ofs(fname, std::ios::binary);
     if (!ofs) throw std::runtime_error("cannot open save file");
+
+    // interned strings snapshot (must include all tokens used by def_index)
     std::vector<const std::string*> interned;
     interned.reserve(kb.interner.pool.size());
     for (auto &s : kb.interner.pool) interned.push_back(&s);
+
     uint64_t N = interned.size();
     ofs.write(reinterpret_cast<const char*>(&N), sizeof(N));
     for (auto p : interned){
@@ -393,10 +442,11 @@ static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
         ofs.write(reinterpret_cast<const char*>(&L), sizeof(L));
         ofs.write(p->data(), static_cast<std::streamsize>(L));
     }
+
+    // edges
     uint64_t E = kb.next.size();
     ofs.write(reinterpret_cast<const char*>(&E), sizeof(E));
     for (auto &pr : kb.next){
-        // find index of key
         const std::string &key = *pr.first;
         auto it = std::find_if(interned.begin(), interned.end(), [&](const std::string* s){ return *s == key; });
         if (it == interned.end()) throw std::runtime_error("save index error");
@@ -411,12 +461,40 @@ static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
             ofs.write(reinterpret_cast<const char*>(&v_idx), sizeof(v_idx));
         }
     }
+
+    // --- write definition expansion section ---
+    uint64_t D = static_cast<uint64_t>(kb.def_depth);
+    ofs.write(reinterpret_cast<const char*>(&D), sizeof(D));
+
+    // def entries: number of keys with a stored expansion
+    uint64_t K = kb.def_index.size();
+    ofs.write(reinterpret_cast<const char*>(&K), sizeof(K));
+    for (auto &pr : kb.def_index){
+        // key index
+        const std::string &key = *pr.first;
+        auto it = std::find_if(interned.begin(), interned.end(), [&](const std::string* s){ return *s == key; });
+        if (it == interned.end()) throw std::runtime_error("save def index error");
+        uint64_t key_idx = static_cast<uint64_t>(std::distance(interned.begin(), it));
+        ofs.write(reinterpret_cast<const char*>(&key_idx), sizeof(key_idx));
+
+        // number of tokens
+        uint64_t M = pr.second.size();
+        ofs.write(reinterpret_cast<const char*>(&M), sizeof(M));
+        for (auto tokp : pr.second){
+            auto it2 = std::find_if(interned.begin(), interned.end(), [&](const std::string* s){ return *s == *tokp; });
+            if (it2 == interned.end()) throw std::runtime_error("save def token index error");
+            uint64_t v_idx = static_cast<uint64_t>(std::distance(interned.begin(), it2));
+            ofs.write(reinterpret_cast<const char*>(&v_idx), sizeof(v_idx));
+        }
+    }
+
     safe_flush(ofs);
 }
 
-static void load_kb_binary(KnowledgeBase &kb, const std::string &fname){
+static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_dict_depth){
     std::ifstream ifs(fname, std::ios::binary);
     if (!ifs) throw std::runtime_error("cannot open load file");
+
     uint64_t N;
     ifs.read(reinterpret_cast<char*>(&N), sizeof(N));
     std::vector<std::string> strings; strings.reserve((size_t)N);
@@ -428,6 +506,7 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname){
     }
     std::vector<StrPtr> ptrs; ptrs.reserve(strings.size());
     for (auto &s : strings) ptrs.push_back(kb.interner.intern(s));
+
     uint64_t E; ifs.read(reinterpret_cast<char*>(&E), sizeof(E));
     for (uint64_t i=0;i<E;++i){
         uint64_t key_idx; ifs.read(reinterpret_cast<char*>(&key_idx), sizeof(key_idx));
@@ -440,12 +519,69 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname){
         }
         kb.next.emplace(key_ptr, std::move(vec));
     }
+
+    // read def-expansion section (new-format)
+    uint64_t file_def_depth;
+    ifs.read(reinterpret_cast<char*>(&file_def_depth), sizeof(file_def_depth));
+    uint64_t K; ifs.read(reinterpret_cast<char*>(&K), sizeof(K));
+    // populate kb.def_index from file
+    {
+        std::lock_guard<std::mutex> lk(kb.def_m);
+        kb.def_index.clear();
+        kb.def_depth = static_cast<int>(file_def_depth);
+    }
+    for (uint64_t i=0;i<K;++i){
+        uint64_t key_idx; ifs.read(reinterpret_cast<char*>(&key_idx), sizeof(key_idx));
+        uint64_t M; ifs.read(reinterpret_cast<char*>(&M), sizeof(M));
+        std::vector<StrPtr> tokens; tokens.reserve((size_t)M);
+        for (uint64_t j=0;j<M;++j){
+            uint64_t v_idx; ifs.read(reinterpret_cast<char*>(&v_idx), sizeof(v_idx));
+            tokens.push_back(ptrs.at((size_t)v_idx));
+        }
+        kb.def_index.emplace(ptrs.at((size_t)key_idx), std::move(tokens));
+    }
+
+    // If CLI requested a different dict depth, clear and recompute expansion for loaded words only
+    if (cli_dict_depth != kb.def_depth){
+        kb.set_def_depth(cli_dict_depth);
+        // --- build deduplicated union of "words present" = saved strings (ptrs) ∪ KB words (keys and neighbors)
+        std::vector<StrPtr> targets;
+        targets.reserve(ptrs.size() + kb.next.size()*2);
+
+        {
+            std::unordered_set<StrPtr, PtrHash, PtrEq> seen;
+            // include all strings from the saved file
+            for (auto p : ptrs) {
+                if (seen.insert(p).second) targets.push_back(p);
+            }
+            // include all words present in KB edges (keys and their neighbors)
+            for (auto &pr : kb.next) {
+                if (seen.insert(pr.first).second) targets.push_back(pr.first);
+                for (auto v : pr.second) {
+                    if (seen.insert(v).second) targets.push_back(v);
+                }
+            }
+        }
+
+        // --- recompute definition expansion for each target in parallel
+        #pragma omp parallel for schedule(dynamic)
+        for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(targets.size()); ++i) {
+            kb.ensure_def_for_interned(targets[(size_t)i]);
+        }
+    }
 }
 
 // --------------------------- CLI + Interactive loop (shorters) -----------
 
 static void print_usage(const char *p){
-    std::cout << "Usage: " << p << " [--maxlen N] [--save FILE] [--load-kb FILE] [--dict-depth D] [--learn f1 f2 ...]\n";
+    std::cout << "Usage: " << p << " [--maxlen N] [--save FILE] [--load-kb FILE] [--dict-depth D] [--learn f1 f2 ...] [--repeat-penalty P] [--help]\n";
+    std::cout << "  --maxlen N          Maximum number of tokens generated in a response.\n";
+    std::cout << "  --save FILE         Save the knowledge-base and dictionary expansions to a binary file.\n";
+    std::cout << "  --load-kb FILE      Load a previously saved knowledge-base (and dictionary expansions) from a binary file.\n";
+    std::cout << "  --dict-depth D      Depth of dictionary-definition expansion used during learning.\n";
+    std::cout << "  --learn f1 f2 ...   Learn from one or more text files to update the knowledge base.\n";
+    std::cout << "  --repeat-penalty P  Penalize repeated tokens during response generation (higher values discourage repetition).\n";
+    std::cout << "  --help              Show command-line interface options for ChatIPC usage.\n";
 }
 
 int main(int argc, char **argv){
@@ -480,8 +616,14 @@ int main(int argc, char **argv){
 
     KnowledgeBase kb;
 
+    // parse the embedded dictionary once for use by per-word expansion
+    g_raw_dict = parse_dictionary_json();
+    // set KB def depth (clears any previous expansion)
+    kb.set_def_depth(dict_depth);
+
+
     if (!load_kb.empty()){
-        try { load_kb_binary(kb, load_kb); std::cerr << "Loaded KB: " << load_kb << "\n"; }
+        try { load_kb_binary(kb, load_kb, dict_depth); std::cerr << "Loaded KB: " << load_kb << "\n"; }
         catch (const std::exception &e){ std::cerr << "Load KB error: " << e.what() << "\n"; }
     }
 
@@ -490,15 +632,13 @@ int main(int argc, char **argv){
         learn_files_parallel(kb, learn_files);
     }
 
-    auto def_index = build_definition_index(dict_depth);
-    if (!def_index.empty()) std::cerr << "Dictionary depth " << dict_depth << " loaded (" << def_index.size() << " words)\n";
-
     std::string line;
     std::cout << "Ready. Enter prompts.\n";
     while (std::cout << "> " , std::getline(std::cin, line)){
         if (line.empty()){ std::cout << "\n"; continue; }
         auto prompt_toks = tokenize_whitespace(line);
         for (size_t i=1;i<prompt_toks.size();++i) kb.add_pair(prompt_toks[i-1], prompt_toks[i]);
+        auto def_index = snapshot_def_index(kb);
         auto resp = generate_response(kb, prompt_toks, maxlen, def_index, repeat_penalty);
         for (size_t i=0;i<resp.size();++i){ std::cout << resp[i]; if (i+1<resp.size()) std::cout << ' '; }
         std::cout << "\n";

ChatIPC.exe

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f67f38cd598779cf67847c62456380856ded672c2a93c8b57a1ae53900a6bef3
-size 24020480
+oid sha256:cebe8014f2651597ff43d04dc0a27089ada505868be13eda19ce52089baafc8b
+size 24027648