Upload 2 files

ChatIPC.cpp

@@ -1514,17 +1514,75 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_
     }
 }
 
+static void print_kb_info(const std::string &fname) {
+    std::ifstream ifs(fname, std::ios::binary);
+    if (!ifs) {
+        std::cerr << "Error: cannot open knowledge-base file " << fname << "\n";
+        return;
+    }
+
+    std::cout << "Reckoning knowledge-base "<< fname << " info.\n";
+
+    try {
+        const std::uint64_t magic = read_u64(ifs);
+        if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
+
+        const std::uint64_t version = read_u64(ifs);
+        if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
+
+        const std::uint64_t file_def_depth = read_u64(ifs);
+
+        const std::uint64_t N = read_u64(ifs);
+        if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
+
+        // Fast-skip string pool to reach the n-gram graph data
+        for (std::uint64_t i = 0; i < N; ++i){
+            std::uint64_t len = read_u64(ifs);
+            if (len > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
+            if (len > 0) ifs.seekg(static_cast<std::streamoff>(len), std::ios_base::cur);
+        }
+
+        const std::uint64_t E = read_u64(ifs);
+        if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
+
+        std::uint64_t max_ngram = 0;
+        for (std::uint64_t i = 0; i < E; ++i){
+            const std::uint64_t ctx_size = read_u64(ifs);
+            if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
+
+            // The maximum context size stored represents the n-gram size
+            if (ctx_size > max_ngram) max_ngram = ctx_size;
+
+            // Skip the context keys
+            ifs.seekg(static_cast<std::streamoff>(ctx_size * sizeof(std::uint64_t)), std::ios_base::cur);
+
+            const std::uint64_t M = read_u64(ifs);
+            if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
+
+            // Skip the next-nodes values
+            ifs.seekg(static_cast<std::streamoff>(M * sizeof(std::uint64_t)), std::ios_base::cur);
+        }
+
+        std::cout << fname << " dictionary depth: " << file_def_depth << "\n";
+        std::cout << fname << " n-gram max size: " << max_ngram << "\n";
+
+    } catch (const std::exception &e) {
+        std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
+    }
+}
+
 static void print_commands(const char *p){
-    std::cout << "CLI options: [--response-max-length N] [--save FILE] [--load-kb FILE] [--dictionary-depth D] [--learn f1 f2 ...] [--repeat-penalty P] [--activate-agi] [--help]\n";
-    std::cout << "  --response-max-length N     Maximum number of tokens in a response.\n";
+    std::cout << p << " command-line interface options:\n";
+    std::cout << "  --response-max-length N     Set maximum number of tokens in a response.\n";
     std::cout << "  --save-kb FILE              Save the knowledge-base to a binary file.\n";
     std::cout << "  --load-kb FILE              Load a previously saved knowledge-base from a binary file.\n";
-    std::cout << "  --dictionary-depth D        Depth of dictionary-definition expansion used during learning.\n";
-    std::cout << "  --n-gram N                  Size of the n-gram where N is the size.\n";
+    std::cout << "  --info-kb FILE              Show the dictionary-depth and n-gram max size of a saved knowledge-base.\n";
+    std::cout << "  --dictionary-depth D        Set depth of dictionary-definition expansion used during learning.\n";
+    std::cout << "  --n-gram-max-size N         Set maximum size of the n-gram where N is the size.\n";
     std::cout << "  --learn f1 f2 ...           Learn from one or more text files to update the knowledge-base.\n";
-    std::cout << "  --repeat-penalty P          Penalize repeated tokens when constructing response (higher values reduce repetition).\n";
+    std::cout << "  --repeat-penalty P          Set penalty for repeated tokens when constructing response (higher values reduce repetition).\n";
     std::cout << "  --activate-agi              Activate the Artificial General Intelligence (AGI) features.\n";
-    std::cout << "  --help                      Show all the Command-Line Interface (CLI) options for using " << p << "\n";
+    std::cout << "  --help                      Show " << p << " command-line interface options.\n";
 }
 
 int main(int argc, char **argv){
@@ -1543,12 +1601,13 @@ int main(int argc, char **argv){
         if (a=="--response-max-length" && i+1<argc){ response_maxlen = std::stoul(argv[++i]); continue; }
         if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
         if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
+        if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
         if (a=="--dictionary-depth" && i+1<argc){ def_depth = std::stoi(argv[++i]); continue; }
-        if (a=="--n-gram" && i+1<argc){ n_gram_size = std::max(1, std::stoi(argv[++i])); continue; }
+        if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = std::max(1, std::stoi(argv[++i])); continue; }
         if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = std::stod(argv[++i]); continue; }
         if (a=="--activate-agi"){
                 std::cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
-                continue;
+                return 0;
         }
         if (a=="--learn"){
             while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]);