chore(deps): update tokenizers to 0.23

Cargo.lock

@@ -1018,7 +1018,7 @@ dependencies = [
  "rayon",
  "safetensors 0.7.0",
  "thiserror 2.0.18",
- "tokenizers",
+ "tokenizers 0.22.2",
  "yoke 0.8.2",
  "zip 7.2.0",
 ]
@@ -1705,6 +1705,12 @@ dependencies = [
  "libloading 0.9.0",
 ]
 
+[[package]]
+name = "daachorse"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"
+
 [[package]]
 name = "darling"
 version = "0.20.11"
@@ -4419,7 +4425,7 @@ dependencies = [
  "rustfft",
  "serde",
  "serde_json",
- "tokenizers",
+ "tokenizers 0.23.1",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -8877,6 +8883,39 @@ dependencies = [
  "unicode_categories",
 ]
 
+[[package]]
+name = "tokenizers"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"
+dependencies = [
+ "ahash",
+ "compact_str",
+ "daachorse",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.3.4",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.9.4",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror 2.0.18",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
 [[package]]
 name = "tokio"
 version = "1.52.1"

Cargo.toml

@@ -71,7 +71,7 @@ tauri-plugin-updater = "2.10"
 blake3 = "1.8"
 rayon = "1.12"
 # disables `esaxx_fast` feature which is conflicting with `ort`
-tokenizers = { version = "0.22", default-features = false, features = ["onig"] }
+tokenizers = { version = "0.23", default-features = false, features = ["onig"] }
 strum = { version = "0.28", features = ["derive"] }
 zip = { version = "8.6", default-features = false, features = ["deflate"] }
 dirs = "6.0"

koharu-ml/src/manga_ocr/tokenizer.rs

@@ -32,7 +32,9 @@ pub fn load_tokenizer(
         }
     }
     if !added.is_empty() {
-        tokenizer.add_special_tokens(&added);
+        tokenizer
+            .add_special_tokens(added)
+            .map_err(|e| anyhow!(e))?;
     }
 
     Ok(tokenizer)