Upload ChatIPC.cpp

ChatIPC.cpp

@@ -265,16 +265,51 @@ static std::vector<std::string> tokenize_whitespace(const std::string &s){
     return out;
 }
 
-static std::vector<std::string> tokenize_non_alnum(const std::string &s){
-    std::vector<std::string> out; std::string cur;
-    for (char ch : s){
-        if (std::isalnum(static_cast<unsigned char>(ch)) || ch=='-' || ch=='\''){
-            cur.push_back(to_low(ch));
+static std::vector<std::string> tokenize_dictionary_expansion(const std::string &s) {
+    std::vector<std::string> out;
+    std::string cur;
+
+    auto flush = [&]() {
+        if (!cur.empty()) {
+            out.push_back(cur);
+            cur.clear();
+        }
+    };
+
+    for (size_t i = 0; i < s.size(); ++i) {
+        unsigned char uc = static_cast<unsigned char>(s[i]);
+        char ch = static_cast<char>(uc);
+
+        if (ch == '_' || ch == '-' || ch == '/' || std::isspace(uc)) {
+            flush();
+            continue;
+        }
+
+        const bool is_upper = std::isupper(uc) != 0;
+        bool camel_boundary = false;
+
+        if (is_upper && !cur.empty()) {
+            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
+            const bool prev_lower_or_digit = (std::islower(prev) != 0) || (std::isdigit(prev) != 0);
+            const bool prev_upper = std::isupper(prev) != 0;
+            const bool next_lower =
+                (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
+
+            camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
+        }
+
+        if (camel_boundary) {
+            flush();
+        }
+
+        if (std::isalnum(uc) != 0) {
+            cur.push_back(static_cast<char>(std::tolower(uc)));
         } else {
-            if (!cur.empty()){ out.push_back(cur); cur.clear(); }
+            flush();
         }
     }
-    if (!cur.empty()) out.push_back(cur);
+
+    flush();
     return out;
 }
 
@@ -367,7 +402,7 @@ static void build_def_tokens_cache(){
 
         auto &defs = global_def_tokens_cache[key];
         for (const auto &def : entry.definitions){
-            auto toks = tokenize_non_alnum(def);
+            auto toks = tokenize_dictionary_expansion(def);
             defs.insert(defs.end(), toks.begin(), toks.end());
         }
     }