Upload 2 files

ChatIPC.cpp

@@ -1281,15 +1281,27 @@ static void learn_files_parallel(KnowledgeBase &kb, const std::vector<std::strin
 static constexpr std::uint64_t KB_MAGIC   = 0x434850434B535641ULL;
 static constexpr std::uint64_t KB_VERSION = 2ULL;
 
-static void write_u64(std::ostream &os, std::uint64_t v){
-    os.write(reinterpret_cast<const char*>(&v), sizeof(v));
-    if(!os) throw std::runtime_error("write_u64 failed");
+static void write_u64(std::ostream &os, std::uint64_t v) {
+    do {
+        unsigned char byte = v & 0x7F;
+        v >>= 7;
+        if (v != 0) byte |= 0x80;
+        os.put(static_cast<char>(byte));
+    } while (v != 0);
+    if (!os) throw std::runtime_error("write_u64 failed");
 }
 
-static std::uint64_t read_u64(std::istream &is){
+static std::uint64_t read_u64(std::istream &is) {
     std::uint64_t v = 0;
-    is.read(reinterpret_cast<char*>(&v), sizeof(v));
-    if(!is) throw std::runtime_error("read_u64 failed");
+    unsigned int shift = 0;
+    unsigned char byte;
+    do {
+        if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
+            throw std::runtime_error("read_u64 failed");
+        }
+        v |= (static_cast<std::uint64_t>(byte & 0x7F) << shift);
+        shift += 7;
+    } while (byte & 0x80);
     return v;
 }
 
@@ -1521,7 +1533,7 @@ static void print_kb_info(const std::string &fname) {
         return;
     }
 
-    std::cout << "Reckoning knowledge-base "<< fname << " info.\n";
+    std::cout << "Reckoning knowledge-base " << fname << " info.\n";
 
     try {
         const std::uint64_t magic = read_u64(ifs);
@@ -1535,7 +1547,7 @@ static void print_kb_info(const std::string &fname) {
         const std::uint64_t N = read_u64(ifs);
         if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
 
-        // Fast-skip string pool to reach the n-gram graph data
+        // Dynamically read and skip strings since lengths are now variable-length encoded
         for (std::uint64_t i = 0; i < N; ++i){
             std::uint64_t len = read_u64(ifs);
             if (len > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
@@ -1550,17 +1562,20 @@ static void print_kb_info(const std::string &fname) {
             const std::uint64_t ctx_size = read_u64(ifs);
             if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
 
-            // The maximum context size stored represents the n-gram size
             if (ctx_size > max_ngram) max_ngram = ctx_size;
 
-            // Skip the context keys
-            ifs.seekg(static_cast<std::streamoff>(ctx_size * sizeof(std::uint64_t)), std::ios_base::cur);
+            // Sequentially read (and skip) context keys using LEB128 stream parsing
+            for(std::uint64_t c = 0; c < ctx_size; ++c) {
+                read_u64(ifs);
+            }
 
             const std::uint64_t M = read_u64(ifs);
             if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
 
-            // Skip the next-nodes values
-            ifs.seekg(static_cast<std::streamoff>(M * sizeof(std::uint64_t)), std::ios_base::cur);
+            // Sequentially read (and skip) next-node values
+            for(std::uint64_t j = 0; j < M; ++j){
+                read_u64(ifs);
+            }
         }
 
         std::cout << fname << " dictionary depth: " << file_def_depth << "\n";