feat: Implement alternative solution achieving functional goals

This commit represents a mixed outcome. While the implementation successfully achieves the intended functionality, it diverges from the original plan. The solution, though unconventional, meets the necessary requirements and ensures operational effectiveness. Future revisions may align it more closely with the initial strategy.

Cargo.lock

@@ -37,6 +37,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -62,7 +71,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -73,7 +82,18 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
 ]
 
 [[package]]
@@ -511,6 +531,29 @@ dependencies = [
  "vsimd",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da379dbebc0b76ef63ca68d8fc6e71c0f13e59432e0987e508c1820e6ab5239"
+dependencies = [
+ "bitflags 1.3.2",
+ "cexpr 0.4.0",
+ "clang-sys",
+ "clap",
+ "env_logger",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex 0.1.1",
+ "which 3.1.1",
+]
+
 [[package]]
 name = "bindgen"
 version = "0.68.1"
@@ -518,7 +561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
 dependencies = [
  "bitflags 2.4.1",
- "cexpr",
+ "cexpr 0.6.0",
  "clang-sys",
  "lazy_static",
  "lazycell",
@@ -529,9 +572,9 @@ dependencies = [
  "quote",
  "regex",
  "rustc-hash",
- "shlex",
- "syn",
- "which",
+ "shlex 1.2.0",
+ "syn 2.0.39",
+ "which 4.4.2",
 ]
 
 [[package]]
@@ -586,13 +629,22 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cexpr"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
+dependencies = [
+ "nom 5.1.3",
+]
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
 dependencies = [
- "nom",
+ "nom 7.1.3",
 ]
 
 [[package]]
@@ -612,6 +664,21 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags 1.3.2",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.50"
@@ -630,7 +697,7 @@ dependencies = [
  "async-trait",
  "json5",
  "lazy_static",
- "nom",
+ "nom 7.1.3",
  "pathdiff",
  "ron",
  "rust-ini",
@@ -722,6 +789,19 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "env_logger"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
+dependencies = [
+ "atty",
+ "humantime",
+ "log",
+ "regex",
+ "termcolor",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.1"
@@ -788,7 +868,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -818,6 +898,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "fvad"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b8e04cf8731da968d9456575a0ae44cb8760dee46169a5289a0e87d4cc4743a"
+dependencies = [
+ "libfvad-sys",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -909,6 +998,15 @@ dependencies = [
  "http",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.3"
@@ -939,6 +1037,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "hound"
+version = "3.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
+
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -973,6 +1077,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "0.14.27"
@@ -1078,6 +1188,16 @@ version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
+[[package]]
+name = "libfvad-sys"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "473b5389760c65fab561600c78f609ee5779799ae8d29818eccea95d8a8c94d8"
+dependencies = [
+ "bindgen 0.56.0",
+ "cc",
+]
+
 [[package]]
 name = "libloading"
 version = "0.7.4"
@@ -1173,6 +1293,16 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "nom"
+version = "5.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08959a387a676302eebf4ddbcbc611da04285579f76f88ee0506c63b1a61dd4b"
+dependencies = [
+ "memchr",
+ "version_check",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -1218,7 +1348,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.3",
  "libc",
 ]
 
@@ -1337,7 +1467,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1405,7 +1535,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1420,6 +1550,8 @@ dependencies = [
  "aws-sdk-translate",
  "config",
  "futures-util",
+ "fvad",
+ "hound",
  "once_cell",
  "poem",
  "serde",
@@ -1429,6 +1561,7 @@ dependencies = [
  "tokio-stream",
  "tracing",
  "tracing-subscriber",
+ "tracing-test",
  "whisper-rs",
  "whisper-rs-sys",
 ]
@@ -1452,7 +1585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
 dependencies = [
  "proc-macro2",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1764,7 +1897,7 @@ checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1834,6 +1967,12 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "shlex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+
 [[package]]
 name = "shlex"
 version = "1.2.0"
@@ -1890,12 +2029,29 @@ version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
 
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.39"
@@ -1907,6 +2063,24 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.50"
@@ -1924,7 +2098,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2006,7 +2180,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2107,7 +2281,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2149,6 +2323,29 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "tracing-test"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4"
+dependencies = [
+ "lazy_static",
+ "tracing-core",
+ "tracing-subscriber",
+ "tracing-test-macro",
+]
+
+[[package]]
+name = "tracing-test-macro"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08"
+dependencies = [
+ "lazy_static",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "try-lock"
 version = "0.2.4"
@@ -2225,6 +2422,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-width"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.9"
@@ -2272,6 +2475,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -2299,6 +2508,15 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "which"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "which"
 version = "4.4.2"
@@ -2324,7 +2542,7 @@ name = "whisper-rs-sys"
 version = "0.7.3"
 source = "git+https://github.com/mingyang91/whisper-rs.git#f8e424a19b13cc348395afd862f0dcb864fcb1fc"
 dependencies = [
- "bindgen",
+ "bindgen 0.68.1",
  "cfg-if",
  "cmake",
  "fs_extra",
@@ -2346,6 +2564,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

Cargo.toml

@@ -18,8 +18,9 @@ serde_json = "1.0"
 serde_yaml = "0.9"
 tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "sync", "signal"] }
 tokio-stream = "0.1"
-tracing = "0.1"
+tracing = { version = "0.1", features = [] }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+fvad = "0.1"
 
 [dependencies.poem]
 version = "1.3"
@@ -27,6 +28,11 @@ features = ["websocket", "static-files"]
 
 [dependencies.whisper-rs]
 git = "https://github.com/mingyang91/whisper-rs.git"
+features = ["coreml", "metal"]
 [dependencies.whisper-rs-sys]
 git = "https://github.com/mingyang91/whisper-rs.git"
 package = "whisper-rs-sys"
+
+[dev-dependencies]
+hound = "3.5.1"
+tracing-test = "*"

config/dev.yaml

@@ -2,24 +2,32 @@ server:
     port: 8080
     host: 0.0.0.0
 whisper:
-    length_ms: 10000
+    length_ms: 5000
     keep_ms: 200
     step_ms: 5000
-    model: "models/ggml-large-q5_0.bin" #"models/ggml-base.bin"
-    max_prompt_tokens: 128
+    model: "models/ggml-large-v3.bin"
+#    model: "models/ggml-base.bin"
+#    model: "models/ggml-medium.en.bin"
+    max_prompt_tokens: 32
+    context_confidence_threshold: 0.5
     params:
-        #n_threads: 4
-        max_tokens: 0
-        audio_ctx: 0
+#        n_threads: 8
+#        max_tokens: 0
+#        audio_ctx: 0
         speed_up: false
-        single_segment: false
+        single_segment: true
         translate: false
-        no_fallback: false
-        temperature_inc: -1.0
+#        temperature_inc: 0.2 #0.4
+#        temperature_inc: 0
+#        entropy_threshold: 2.5
+#        entropy_threshold: 2.8
+#        n_max_text_ctx: 64 #16384
         print_special: false
         print_progress: false
         print_realtime: false
+        token_timestamps: false
         no_context: false
         no_timestamps: false
+        suppress_non_speech_tokens: false
         tinydiarize: false
         language: "en"

src/config.rs

@@ -3,7 +3,7 @@ use std::{env, ffi::c_int, net::IpAddr};
 use config::{Config, Environment, File};
 use once_cell::sync::Lazy;
 use serde::Deserialize;
-use whisper_rs::FullParams;
+use whisper_rs::{FullParams};
 use tracing::debug;
 
 pub(crate) static SETTINGS: Lazy<Settings> =
@@ -12,28 +12,33 @@ pub(crate) static SETTINGS: Lazy<Settings> =
 #[derive(Debug, Deserialize, Clone)]
 pub(crate) struct WhisperConfig {
     pub(crate) params: WhisperParams,
-    pub(crate) step_ms: u32,
-    pub(crate) length_ms: u32,
-    pub(crate) keep_ms: u32,
+    pub(crate) step_ms: usize,
+    pub(crate) length_ms: usize,
+    pub(crate) keep_ms: usize,
     pub(crate) model: String,
     pub(crate) max_prompt_tokens: usize,
+    pub(crate) context_confidence_threshold: f32,
 }
 
 #[allow(dead_code)]
 #[derive(Debug, Deserialize, Clone)]
 pub(crate) struct WhisperParams {
     pub(crate) n_threads: Option<usize>,
-    pub(crate) max_tokens: u32,
-    pub(crate) audio_ctx: u32,
-    pub(crate) speed_up: bool,
-    pub(crate) translate: bool,
-    pub(crate) no_fallback: bool,
-    pub(crate) print_special: bool,
-    pub(crate) print_realtime: bool,
-    pub(crate) print_progress: bool,
-    pub(crate) no_timestamps: bool,
-    pub(crate) temperature_inc: f32,
-    pub(crate) single_segment: bool,
+    pub(crate) max_tokens: Option<u32>,
+    pub(crate) audio_ctx: Option<u32>,
+    pub(crate) speed_up: Option<bool>,
+    pub(crate) translate: Option<bool>,
+    pub(crate) no_context: Option<bool>,
+    pub(crate) print_special: Option<bool>,
+    pub(crate) print_realtime: Option<bool>,
+    pub(crate) print_progress: Option<bool>,
+    pub(crate) token_timestamps: Option<bool>,
+    pub(crate) no_timestamps: Option<bool>,
+    pub(crate) temperature_inc: Option<f32>,
+    pub(crate) entropy_threshold: Option<f32>,
+    pub(crate) single_segment: Option<bool>,
+    pub(crate) suppress_non_speech_tokens: Option<bool>,
+    pub(crate) n_max_text_ctx: Option<usize>,
     // pub(crate) tinydiarize: bool,
     pub(crate) language: Option<String>,
 }
@@ -41,25 +46,58 @@ pub(crate) struct WhisperParams {
 impl WhisperParams {
     pub(crate) fn to_full_params<'a, 'b>(&'a self, tokens: &'b [c_int]) -> FullParams<'a, 'b> {
         let mut param = FullParams::new(Default::default());
-        param.set_print_progress(self.print_progress);
-        param.set_print_special(self.print_special);
-        param.set_print_realtime(self.print_realtime);
-        param.set_print_timestamps(!self.no_timestamps);
-        param.set_translate(self.translate);
-        param.set_single_segment(false);
-        param.set_max_tokens(self.max_tokens as i32);
-        let lang = self.language.as_deref();
-        param.set_language(lang);
-        let num_cpus = std::thread::available_parallelism()
-            .map(|c| c.get())
-            .unwrap_or(4);
-        param.set_n_threads(self.n_threads.unwrap_or(num_cpus) as c_int);
-        param.set_audio_ctx(self.audio_ctx as i32);
-        param.set_speed_up(self.speed_up);
+        if let Some(print_progress) = self.print_progress.as_ref() {
+            param.set_print_progress(*print_progress);
+        }
+        if let Some(print_special) = self.print_special.as_ref() {
+            param.set_print_special(*print_special);
+        }
+        if let Some(print_realtime) = self.print_realtime.as_ref() {
+            param.set_print_realtime(*print_realtime);
+        }
+        if let Some(single_segment) = self.single_segment.as_ref() {
+            param.set_single_segment(*single_segment);
+        }
+        if let Some(no_timestamps) = self.no_timestamps.as_ref() {
+            param.set_print_timestamps(!no_timestamps);
+        }
+        if let Some(token_timestamps) = self.token_timestamps.as_ref() {
+            param.set_token_timestamps(*token_timestamps);
+        }
+        if let Some(translate) = self.translate.as_ref() {
+            param.set_translate(*translate);
+        }
+        if let Some(max_tokens) = self.max_tokens.as_ref() {
+            param.set_max_tokens(*max_tokens as i32);
+        }
+        param.set_language(self.language.as_deref());
+        if let Some(n_threads) = self.n_threads.as_ref() {
+            param.set_n_threads(*n_threads as i32);
+        }
+        if let Some(audio_ctx) = self.audio_ctx.as_ref() {
+            param.set_audio_ctx(*audio_ctx as i32);
+        }
+        if let Some(speed_up) = self.speed_up.as_ref() {
+            param.set_speed_up(*speed_up);
+        }
         // param.set_tdrz_enable(self.tinydiarize);
-        param.set_temperature_inc(self.temperature_inc);
-        param.set_tokens(tokens);
+        if let Some(temperature_inc) = self.temperature_inc.as_ref() {
+            param.set_temperature_inc(*temperature_inc);
+        }
+        if let Some(suppress_non_speech_tokens) = self.suppress_non_speech_tokens.as_ref() {
+            param.set_suppress_non_speech_tokens(*suppress_non_speech_tokens);
+        }
+        if let Some(no_context) = self.no_context.as_ref() {
+            param.set_no_context(*no_context);
+        }
+        if let Some(entropy_threshold) = self.entropy_threshold.as_ref() {
+            param.set_entropy_thold(*entropy_threshold);
+        }
+        if let Some(n_max_text_ctx) = self.n_max_text_ctx.as_ref() {
+            param.set_n_max_text_ctx(*n_max_text_ctx as i32);
+        }
 
+        param.set_tokens(tokens);
         param
     }
 }

src/main.rs

@@ -104,7 +104,7 @@ async fn stream_speaker(
     ws.on_upgrade(|mut socket| async move {
         let _origin_tx = lesson.voice_channel();
         let mut transcribe_rx = lesson.transcript_channel();
-        let whisper = WhisperHandler::new(SETTINGS.whisper.clone(), prompt)
+        let mut whisper = WhisperHandler::new(SETTINGS.whisper.clone(), prompt)
             .expect("failed to create whisper");
         let mut whisper_transcribe_rx = whisper.subscribe();
         loop {
@@ -118,7 +118,7 @@ async fn stream_speaker(
                 msg = socket.next() => {
                     match msg.as_ref() {
                         Some(Ok(Message::Binary(bin))) => {
-                            let _ = whisper.send(bin.to_vec()).await; // whisper test
+                            let _ = whisper.send_bytes(bin.to_vec()).await; // whisper test
                             // if let Err(e) = origin_tx.send(bin.to_vec()).await {
                             //     tracing::warn!("failed to send voice: {}", e);
                             //     break;
@@ -173,7 +173,7 @@ async fn stream_listener(
     ws: WebSocket,
 ) -> impl IntoResponse {
     let lesson_opt = ctx.lessons_manager.get_lesson(query.id).await;
-    tracing::debug!("listener param = {:?}", query);
+    debug!("listener param = {:?}", query);
 
     ws.on_upgrade(|mut socket| async move {
         let voice_id = match query.voice.parse() {

src/whisper.rs

@@ -1,24 +1,26 @@
 use std::{
     collections::VecDeque,
-    ffi::c_int,
     fmt::{Debug, Display, Formatter},
     thread::sleep,
     time::Duration,
 };
+use fvad::SampleRate;
 
 use once_cell::sync::Lazy;
 use tokio::sync::{broadcast, mpsc, oneshot};
-use tracing::{debug, trace};
-use whisper_rs::{convert_integer_to_float_audio, WhisperContext, WhisperState, WhisperToken};
-use whisper_rs_sys::WHISPER_SAMPLE_RATE;
+use tokio::time::Instant;
+use tracing::{debug, trace, warn};
+use whisper_rs::{convert_integer_to_float_audio, WhisperContext, WhisperState, WhisperToken, WhisperTokenData};
 
 use crate::config::{Settings, SETTINGS};
 use crate::{config::WhisperConfig, group::GroupedWithin};
 
+const WHISPER_SAMPLE_RATE: usize = whisper_rs_sys::WHISPER_SAMPLE_RATE as usize;
+
 static WHISPER_CONTEXT: Lazy<WhisperContext> = Lazy::new(|| {
     let settings = Settings::new().expect("Failed to initialize settings.");
     if tracing::enabled!(tracing::Level::DEBUG) {
-        let info = print_system_info();
+        let info = whisper_rs::print_system_info();
         debug!("system_info: n_threads = {} / {} | {}\n",
             settings.whisper.params.n_threads.unwrap_or(0),
             std::thread::available_parallelism().map(|c| c.get()).unwrap_or(0),
@@ -27,13 +29,6 @@ static WHISPER_CONTEXT: Lazy<WhisperContext> = Lazy::new(|| {
     WhisperContext::new(&settings.whisper.model).expect("failed to create WhisperContext")
 });
 
-fn print_system_info() -> String {
-    unsafe {
-        let raw_info = whisper_rs_sys::whisper_print_system_info();
-        let info = std::ffi::CStr::from_ptr(raw_info);
-        info.to_str().unwrap_or("failed to get system info").to_string()
-    }
-}
 
 #[derive(Debug)]
 pub(crate) enum Error {
@@ -70,16 +65,21 @@ impl std::error::Error for Error {
     }
 }
 
-fn pcm_i16_to_f32(input: &[u8]) -> Vec<f32> {
-    let pcm_i16 = input
+fn u8_to_i16(input: &[u8]) -> Vec<i16> {
+    input
         .chunks_exact(2)
         .map(|chunk| {
             let mut buf = [0u8; 2];
             buf.copy_from_slice(chunk);
             i16::from_le_bytes(buf)
         })
-        .collect::<Vec<i16>>();
-    convert_integer_to_float_audio(pcm_i16.as_slice())
+        .collect::<Vec<i16>>()
+}
+
+#[derive(Clone, Debug)]
+pub enum Output {
+    Unstable(Segment),
+    Stable(Segment),
 }
 
 #[derive(Clone, Debug)]
@@ -87,20 +87,21 @@ pub struct Segment {
     pub start_timestamp: i64,
     pub end_timestamp: i64,
     pub text: String,
-    tokens: Vec<c_int>,
+    tokens: Vec<WhisperTokenData>,
 }
 
 pub struct WhisperHandler {
-    tx: mpsc::Sender<Vec<u8>>,
-    transcription_tx: broadcast::Sender<Vec<Segment>>,
+    tx: mpsc::Sender<Vec<i16>>,
+    transcription_tx: broadcast::Sender<Vec<Output>>,
     stop_handle: Option<oneshot::Sender<()>>,
 }
 
 impl WhisperHandler {
     pub(crate) fn new(config: WhisperConfig, prompt: String) -> Result<Self, Error> {
+        let vad_slice_size = WHISPER_SAMPLE_RATE / 100 * 3;
         let (stop_handle, mut stop_signal) = oneshot::channel();
-        let (pcm_tx, pcm_rx) = mpsc::channel::<Vec<u8>>(128);
-        let (transcription_tx, _) = broadcast::channel::<Vec<Segment>>(128);
+        let (pcm_tx, pcm_rx) = mpsc::channel::<Vec<i16>>(128);
+        let (transcription_tx, _) = broadcast::channel::<Vec<Output>>(128);
         let shared_transcription_tx = transcription_tx.clone();
         let state = WHISPER_CONTEXT
             .create_state()
@@ -109,21 +110,46 @@ impl WhisperHandler {
             .tokenize(prompt.as_str(), SETTINGS.whisper.max_prompt_tokens)
             .map_err(|e| Error::whisper_error("failed to tokenize prompt", e))?;
         tokio::task::spawn_blocking(move || {
+            let mut vad = fvad::Fvad::new().expect("failed to create VAD")
+                .set_sample_rate(SampleRate::Rate16kHz);
             let mut detector = Detector::new(state, &SETTINGS.whisper, preset_prompt_tokens);
             let mut grouped = GroupedWithin::new(
-                detector.n_samples_step * 2,
+                detector.n_samples_step,
                 Duration::from_millis(config.step_ms as u64),
                 pcm_rx,
                 u16::MAX as usize,
             );
             while let Err(oneshot::error::TryRecvError::Empty) = stop_signal.try_recv() {
+                if detector.has_crossed_next_line() {
+                    if let Some(segment) = detector.next_line() {
+                        let segments = vec![Output::Stable(segment)];
+                        if let Err(e) = shared_transcription_tx.send(segments) {
+                            tracing::error!("failed to send transcription: {}", e);
+                            break;
+                        };
+                    }
+                }
                 let new_pcm_f32 = match grouped.next() {
                     Err(mpsc::error::TryRecvError::Disconnected) => break,
                     Err(mpsc::error::TryRecvError::Empty) => {
                         sleep(Duration::from_millis(10));
                         continue;
                     }
-                    Ok(data) => pcm_i16_to_f32(&data),
+                    Ok(data) => {
+                        let active_voice = data
+                            .chunks(vad_slice_size)
+                            .filter(|frame| {
+                                if frame.len() != vad_slice_size {
+                                    true
+                                } else {
+                                    vad.is_voice_frame(frame).unwrap_or(true)
+                                }
+                                // true
+                            })
+                            .collect::<Vec<_>>()
+                            .concat();
+                        convert_integer_to_float_audio(&active_voice)
+                    },
                 };
 
                 detector.feed(new_pcm_f32);
@@ -135,26 +161,22 @@ impl WhisperHandler {
                         result
                     }
                     Err(err) => {
-                        tracing::warn!("failed to inference: {}", err);
+                        warn!("failed to inference: {}", err);
                         continue;
                     }
                 };
 
-                for segment in segments.iter() {
-                    trace!(
-                        "[{}-{}]s SEGMENT: {}",
-                        segment.start_timestamp as f32 / 1000.0,
-                        segment.end_timestamp as f32 / 1000.0,
-                        segment.text
-                    );
-                }
-
-                if let Err(e) = shared_transcription_tx.send(segments) {
+                let outputs = segments
+                    .iter()
+                    .map(|segment| Output::Unstable(segment.clone()))
+                    .collect::<Vec<_>>();
+                if let Err(e) = shared_transcription_tx.send(outputs) {
                     tracing::error!("failed to send transcription: {}", e);
                     break;
                 };
             }
         });
+
         Ok(Self {
             tx: pcm_tx,
             transcription_tx,
@@ -162,27 +184,34 @@ impl WhisperHandler {
         })
     }
 
-    pub fn subscribe(&self) -> broadcast::Receiver<Vec<Segment>> {
+    pub fn subscribe(&self) -> broadcast::Receiver<Vec<Output>> {
         self.transcription_tx.subscribe()
     }
 
-    pub async fn send(&self, data: Vec<u8>) -> Result<(), mpsc::error::SendError<Vec<u8>>> {
+    pub async fn send_i16(&mut self, data: Vec<i16>) -> Result<(), mpsc::error::SendError<Vec<i16>>> {
         self.tx.send(data).await
     }
+
+    pub async fn send_bytes(&mut self, data: Vec<u8>) -> Result<(), mpsc::error::SendError<Vec<i16>>> {
+        let i16_data = u8_to_i16(&data);
+        self.send_i16(i16_data).await
+    }
 }
 
 #[allow(dead_code)]
 struct Detector {
     state: WhisperState<'static>,
     config: &'static WhisperConfig,
+    start_time: Instant,
+    segment: Option<Segment>,
+    line_num: usize,
     preset_prompt_tokens: Vec<WhisperToken>,
     n_samples_keep: usize,
     n_samples_step: usize,
     n_samples_len: usize,
-    prompt_tokens: Vec<c_int>,
+    prompt_tokens: Vec<WhisperToken>,
     pcm_f32: VecDeque<f32>,
     offset: usize,
-    stable_offset: usize,
 }
 
 impl Detector {
@@ -194,14 +223,16 @@ impl Detector {
         Detector {
             state,
             config,
+            start_time: Instant::now(),
+            segment: None,
+            line_num: 0,
             preset_prompt_tokens,
-            n_samples_keep: (config.keep_ms * WHISPER_SAMPLE_RATE / 1000) as usize,
-            n_samples_step: (config.step_ms * WHISPER_SAMPLE_RATE / 1000) as usize,
-            n_samples_len: (config.length_ms * WHISPER_SAMPLE_RATE / 1000) as usize,
+            n_samples_keep: config.keep_ms * WHISPER_SAMPLE_RATE / 1000,
+            n_samples_step: config.step_ms * WHISPER_SAMPLE_RATE / 1000,
+            n_samples_len: config.length_ms * WHISPER_SAMPLE_RATE / 1000,
             prompt_tokens: Default::default(),
-            pcm_f32: VecDeque::from(vec![0f32; 30 * WHISPER_SAMPLE_RATE as usize]),
+            pcm_f32: VecDeque::with_capacity(config.length_ms * WHISPER_SAMPLE_RATE / 1000),
             offset: 0,
-            stable_offset: 0,
         }
     }
 
@@ -210,20 +241,16 @@ impl Detector {
         if self.pcm_f32.len() < self.n_samples_len {
             return;
         }
-        let len_to_drain = self
-            .pcm_f32
-            .drain(0..(self.pcm_f32.len() - self.n_samples_len))
-            .len();
-        self.offset += len_to_drain;
+        // let len_to_drain = self
+        //     .pcm_f32
+        //     .drain(0..(self.pcm_f32.len() - self.n_samples_len))
+        //     .len();
+        // warn!("ASR too slow, drain {} samples", len_to_drain);
+        // self.offset += len_to_drain;
     }
 
     fn inference(&mut self) -> Result<Vec<Segment>, Error> {
-        let prompt_tokens = [
-            self.preset_prompt_tokens.as_slice(),
-            self.prompt_tokens.as_slice(),
-        ]
-        .concat();
-        let params = self.config.params.to_full_params(prompt_tokens.as_slice());
+        let params = self.config.params.to_full_params(self.prompt_tokens.as_slice());
         let start = std::time::Instant::now();
         let _ = self
             .state
@@ -231,35 +258,32 @@ impl Detector {
             .map_err(|e| Error::whisper_error("failed to initialize WhisperState", e))?;
         let end = std::time::Instant::now();
         if end - start > Duration::from_millis(self.config.step_ms as u64) {
-            tracing::warn!(
-                "full([{}]) took {} ms too slow",
-                self.pcm_f32.len(),
-                (end - start).as_millis()
-            );
+            // warn!(
+            //     "full([{}]) took {} ms too slow",
+            //     self.pcm_f32.len(),
+            //     (end - start).as_millis()
+            // );
         }
 
-        let timestamp_offset: i64 = (self.offset * 1000 / WHISPER_SAMPLE_RATE as usize) as i64;
-        let stable_offset: i64 = (self.stable_offset * 1000 / WHISPER_SAMPLE_RATE as usize) as i64;
+        let timestamp_offset: i64 = (self.offset * 1000 / WHISPER_SAMPLE_RATE) as i64;
         let num_segments = self
             .state
             .full_n_segments()
             .map_err(|e| Error::whisper_error("failed to get number of segments", e))?;
         let mut segments: Vec<Segment> = Vec::with_capacity(num_segments as usize);
         for i in 0..num_segments {
-            let end_timestamp: i64 = timestamp_offset
-                + 10 * self
-                    .state
-                    .full_get_segment_t1(i)
-                    .map_err(|e| Error::whisper_error("failed to get end timestamp", e))?;
-            if end_timestamp <= stable_offset {
-                continue;
-            }
-
             let start_timestamp: i64 = timestamp_offset
                 + 10 * self
                     .state
                     .full_get_segment_t0(i)
                     .map_err(|e| Error::whisper_error("failed to get start timestamp", e))?;
+
+            let end_timestamp: i64 = timestamp_offset
+                + 10 * self
+                .state
+                .full_get_segment_t1(i)
+                .map_err(|e| Error::whisper_error("failed to get end timestamp", e))?;
+
             let segment = self
                 .state
                 .full_get_segment_text(i)
@@ -270,11 +294,9 @@ impl Detector {
                 .map_err(|e| Error::whisper_error("failed to get segment tokens", e))?;
             let mut segment_tokens = Vec::with_capacity(num_tokens as usize);
             for j in 0..num_tokens {
-                segment_tokens.push(
-                    self.state
-                        .full_get_token_id(i, j)
-                        .map_err(|e| Error::whisper_error("failed to get token", e))?,
-                );
+                let token_data = self.state.full_get_token_data(i, j)
+                    .map_err(|e| Error::whisper_error("failed to get token data", e))?;
+                segment_tokens.push(token_data);
             }
 
             segments.push(Segment {
@@ -285,52 +307,132 @@ impl Detector {
             });
         }
 
-        let Some((_last, init)) = segments.split_last() else {
-            return Ok(Vec::default());
-        };
+        self.segment = segments.first().cloned();
+        Ok(segments.to_vec())
+    }
 
-        let Some((last_2_seg, _)) = init.split_last() else {
-            return Ok(Vec::default());
+    fn remember_prompt(&mut self) {
+        let Some(segment) = self.segment.as_ref() else {
+            return
         };
 
-        let offset = (last_2_seg.end_timestamp - timestamp_offset) as usize / 1000
-            * WHISPER_SAMPLE_RATE as usize;
-        self.stable_offset = offset;
-        self.drop_stable_by_segments(init);
-        Ok(init.into())
-    }
+        let tokens = segment
+            .tokens
+            .iter()
+            .map(|td| td.tid)
+            .collect::<Vec<WhisperToken>>();
 
-    fn drop_stable_by_segments(&mut self, stable_segments: &[Segment]) {
-        let Some(last) = stable_segments.last() else {
-            return;
-        };
-        let drop_offset: usize =
-            last.end_timestamp as usize / 1000 * WHISPER_SAMPLE_RATE as usize - self.offset;
-        if drop_offset > self.pcm_f32.len() {
-            return; // Arithmetic overflow
+        self.prompt_tokens.extend(tokens);
+        if self.prompt_tokens.len() > self.config.max_prompt_tokens {
+            let _ = self.prompt_tokens.drain(0..(self.prompt_tokens.len() - self.config.max_prompt_tokens)).len();
         }
-        let len_to_drain = self.pcm_f32.drain(0..drop_offset).len();
-        self.offset += len_to_drain;
+    }
 
-        for segment in stable_segments.iter() {
-            self.prompt_tokens.extend(&segment.tokens);
-        }
-        if self.prompt_tokens.len() > self.config.max_prompt_tokens {
-            let _ = self
-                .prompt_tokens
-                .drain(0..(self.prompt_tokens.len() - self.config.max_prompt_tokens))
-                .len();
+    fn has_crossed_next_line(&self) -> bool {
+        let now = Instant::now();
+        let elapsed = now - self.start_time;
+        let line_number: usize = (elapsed.as_millis() / self.config.length_ms as u128) as usize;
+        line_number > self.line_num
+    }
+
+    fn next_line(&mut self) -> Option<Segment> {
+        if self.pcm_f32.len() > self.n_samples_keep {
+            let drain_size = self.pcm_f32.drain(0..(self.pcm_f32.len() - self.n_samples_keep)).len();
+            self.offset += drain_size;
+        } else {
+            let size_will_clear = self.pcm_f32.len();
+            self.pcm_f32.clear();
+            self.offset += size_will_clear;
         }
+
+        self.line_num += 1;
+        self.remember_prompt();
+        self.segment.take()
     }
 }
 
 impl Drop for WhisperHandler {
     fn drop(&mut self) {
         let Some(stop_handle) = self.stop_handle.take() else {
-            return tracing::warn!("WhisperHandler::drop() called without stop_handle");
+            return warn!("WhisperHandler::drop() called without stop_handle");
         };
         if stop_handle.send(()).is_err() {
-            tracing::warn!("WhisperHandler::drop() failed to send stop signal");
+            warn!("WhisperHandler::drop() failed to send stop signal");
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::io::{stdout, Write};
+    use hound;
+    use tracing_test;
+    use tracing::info;
+
+    async fn print_output(output: Output) {
+        match output {
+            Output::Stable(stable) => {
+                print!("\x1b[2K\r");
+                print!("{}\n", stable.text);
+            },
+            Output::Unstable(unstable) => {
+                // back to previous line of console
+                print!("\x1b[2K\r");
+                print!("{}", " ".repeat(100));
+                print!("\x1b[2K\r");
+                print!("{} ...", unstable.text);
+            }
+        }
+        stdout().flush().unwrap();
+    }
+    #[tokio::test]
+    #[tracing_test::traced_test]
+    async fn test_whisper_handler() {
+        let mut whisper_handler = WhisperHandler::new(
+            SETTINGS.whisper.clone(),
+            "Harry Potter and the Philosopher's Stone".to_string(),
+        ).expect("failed to create WhisperHandler");
+
+        let wav = hound::WavReader::open("samples/ADHD_1A.wav")
+            .expect("failed to open wav");
+        let spec = wav.spec();
+        println!("{:?}", spec);
+        let samples = wav
+            .into_samples::<i16>()
+            .map(|s| s.unwrap())
+            .collect::<Vec<i16>>();
+        let chunks = samples.chunks(1600)
+            .map(|chunk| chunk.to_vec())
+            .into_iter();
+
+        let mut rx = whisper_handler.subscribe();
+        let send_fut = async {
+            // tokio::time::sleep(Duration::from_secs(5)).await;
+            for chunk in chunks {
+                let _ = whisper_handler.send_i16(chunk).await.expect("failed to send sample");
+                tokio::time::sleep(Duration::from_millis(100)).await;
+            }
+        };
+
+        let recv_fut = async {
+            while let Ok(outputs) = rx.recv().await {
+                let Some(output) = outputs.first() else {
+                    continue
+                };
+
+                match output {
+                    Output::Stable(stable) => {
+                        println!("{}", stable.text);
+                    },
+                    Output::Unstable(unstable) => {
+
+                    }
+                }
+
+            }
+        };
+
+        tokio::join!(send_fut, recv_fut);
+    }
+}