refactor: koharu-runtime

.cargo/config.toml

@@ -3,5 +3,6 @@
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
 TS_RS_EXPORT_DIR = { value = "ui/lib/generated/protocol", relative = true }
 TS_RS_LARGE_INT = "number"
+LLAMA_CPP_TAG = "b8233"
 # CUDA 13.1 requires C++17
 NVCC_PREPEND_FLAGS = "-std=c++17 -Xcompiler /Zc:preprocessor"

Cargo.lock

@@ -1297,27 +1297,6 @@ dependencies = [
  "syn 2.0.117",
 ]
 
-[[package]]
-name = "csv"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde_core",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ctor"
 version = "0.2.9"
@@ -3921,7 +3900,6 @@ dependencies = [
  "koharu-rpc",
  "koharu-runtime",
  "koharu-types",
- "libloading 0.9.0",
  "once_cell",
  "rfd",
  "serde_json",
@@ -3971,11 +3949,11 @@ dependencies = [
  "hf-hub",
  "keyring",
  "koharu-http",
+ "koharu-runtime",
  "koharu-types",
  "libloading 0.8.9",
  "minijinja",
  "num_cpus",
- "once_cell",
  "proc-macro2",
  "quote",
  "reqwest 0.13.2",
@@ -4010,7 +3988,7 @@ dependencies = [
  "keyring",
  "koharu-http",
  "koharu-types",
- "libloading 0.9.0",
+ "libloading 0.8.9",
  "minijinja",
  "num_cpus",
  "objc2",
@@ -4116,19 +4094,15 @@ name = "koharu-runtime"
 version = "0.40.1"
 dependencies = [
  "anyhow",
- "csv",
+ "dirs",
  "flate2",
- "futures",
  "koharu-http",
- "libloading 0.9.0",
- "num_cpus",
- "once_cell",
- "rayon",
- "reqwest 0.13.2",
+ "libloading 0.8.9",
  "serde_json",
+ "tar",
  "tempfile",
  "tokio",
- "tracing",
+ "windows-sys 0.61.2",
  "zip 8.2.0",
 ]
 

Cargo.toml

@@ -77,7 +77,7 @@ flate2 = { version = "1.0", default-features = false, features = [
 csv = "1.3"
 tempfile = "3.24"
 once_cell = "1.21"
-libloading = "0.9"
+libloading = "0.8.9"
 num_cpus = "1.17"
 rustfft = "6.4"
 cudarc = { version = "0.19.4", features = [

koharu-llm/Cargo.toml

@@ -30,11 +30,11 @@ flate2 = { workspace = true }
 futures = { workspace = true }
 hf-hub = { workspace = true }
 koharu-http = { workspace = true }
+koharu-runtime = { workspace = true }
 koharu-types = { workspace = true }
-libloading = "0.8.9"
+libloading = { workspace = true }
 minijinja = { workspace = true }
 num_cpus = { workspace = true }
-once_cell = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

koharu-llm/bin/llama.rs

@@ -160,20 +160,13 @@ fn main() -> Result<()> {
         list_devices,
     } = Args::parse();
 
-    let runtime_dir = dirs::cache_dir()
-        .unwrap_or_else(std::env::temp_dir)
-        .join("Koharu")
-        .join("llama.cpp")
-        .join(env!("KOHARU_LLM_LLAMA_CPP_TAG"));
     tokio::runtime::Builder::new_current_thread()
         .enable_all()
         .build()
         .with_context(|| "unable to create tokio runtime")?
         .block_on(async {
-            koharu_llm::safe::runtime::ensure_dylibs(&runtime_dir)
+            koharu_runtime::initialize()
                 .await
-                .context("failed to download runtime libraries")?;
-            koharu_llm::safe::runtime::initialize(&runtime_dir)
                 .context("failed to initialize runtime libraries")?;
             Ok::<(), anyhow::Error>(())
         })?;

koharu-llm/bin/llm.rs

@@ -10,11 +10,11 @@ struct Args {
     #[arg(
         long,
         default_value = r#"「え、マジで!?」
-俺、田中太郎は気がつくと見知らぬ森の中にいた。目の前には巨大な魔獣が牙を剥いている。
-「やばい、死ぬ!」
-その瞬間、俺の手から青白い光が放たれた。魔獣は一瞬で消滅する。
-『レベルアップしました。新スキル「爆炎魔法」を習得』
-頭の中に謎の声が響く。どうやら俺、チート能力を手に入れたらしい。
+俺、田ä¸å¤ªéƒŽã¯æ°—がつくと見知らぬ森のä¸ã«ã„た。目の前には巨大なé”獣が牙を剥いている。
+「やばい、æ»ã¬!」
+その瞬間、俺の手から青白い光が放たれた。é”獣は一瞬で消滅する。
+『レベルアップしました。新スã‚ル「爆炎é”法」を習得』
+é ã®ä¸ã«è¬Žã®å£°ãŒéŸ¿ãã€‚どうやら俺、チート能力を手に入れたらしい。
 「この世界で、俺は最強になる!」
 こうして俺の異世界ライフが始まった。美少女との出会いも、もちろん待っている。"#
     )]
@@ -71,9 +71,7 @@ fn init_tracing() {
 }
 
 async fn initialize_runtime() -> anyhow::Result<()> {
-    let runtime_dir = koharu_llm::runtime_dir();
-    koharu_llm::safe::runtime::ensure_dylibs(&runtime_dir).await?;
-    koharu_llm::safe::runtime::initialize(&runtime_dir)?;
+    koharu_runtime::initialize().await?;
     Ok(())
 }
 

koharu-llm/build.rs

@@ -11,10 +11,6 @@ use reqwest::blocking::Client;
 use syn::{FnArg, ImplItem, Item, Pat, Signature};
 use tar::Archive;
 
-const LLAMA_CPP_TAG: &str = "b8233";
-const LLAMA_CPP_COMMIT: &str = "c5a778891ba0ddbd4cbb507c823f970595b1adc2";
-const SOURCE_URL: &str = "https://github.com/ggml-org/llama.cpp/archive/refs/tags/b8233.tar.gz";
-
 const GGML_FUNCTIONS: &[&str] = &[
     "ggml_backend_dev_count",
     "ggml_backend_dev_get",
@@ -45,6 +41,7 @@ const MTMD_FUNCTIONS: &[&str] = &["mtmd_.*", "mtmd_helper_.*"];
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=LLAMA_CPP_TAG");
 
     if let Err(err) = run() {
         panic!("{err:#}");
@@ -54,8 +51,9 @@ fn main() {
 fn run() -> Result<()> {
     validate_target()?;
 
+    let llama_cpp_tag = env::var("LLAMA_CPP_TAG").context("missing LLAMA_CPP_TAG")?;
     let out_dir = PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
-    let source_root = ensure_source_tree(&out_dir)?;
+    let source_root = ensure_source_tree(&out_dir, &llama_cpp_tag)?;
     let header_path = write_wrapper_header(&out_dir)?;
     let include_dirs = include_dirs(&source_root);
 
@@ -106,8 +104,7 @@ fn run() -> Result<()> {
         ],
     )?;
 
-    println!("cargo:rustc-env=KOHARU_LLM_LLAMA_CPP_TAG={LLAMA_CPP_TAG}");
-    println!("cargo:rustc-env=KOHARU_LLM_LLAMA_CPP_COMMIT={LLAMA_CPP_COMMIT}");
+    println!("cargo:rustc-env=KOHARU_LLM_LLAMA_CPP_TAG={llama_cpp_tag}");
 
     Ok(())
 }
@@ -125,10 +122,12 @@ fn validate_target() -> Result<()> {
     Ok(())
 }
 
-fn ensure_source_tree(out_dir: &Path) -> Result<PathBuf> {
+fn ensure_source_tree(out_dir: &Path, llama_cpp_tag: &str) -> Result<PathBuf> {
     let cache_dir = out_dir.join("llama.cpp-source");
-    let tarball_path = cache_dir.join(format!("llama.cpp-{LLAMA_CPP_TAG}.tar.gz"));
-    let source_root = cache_dir.join(format!("llama.cpp-{LLAMA_CPP_TAG}"));
+    let tarball_path = cache_dir.join(format!("llama.cpp-{llama_cpp_tag}.tar.gz"));
+    let source_root = cache_dir.join(format!("llama.cpp-{llama_cpp_tag}"));
+    let source_url =
+        format!("https://github.com/ggml-org/llama.cpp/archive/refs/tags/{llama_cpp_tag}.tar.gz");
 
     if source_root.join("include/llama.h").exists() {
         return Ok(source_root);
@@ -142,7 +141,7 @@ fn ensure_source_tree(out_dir: &Path) -> Result<PathBuf> {
             .build()
             .context("failed to build reqwest client")?;
         let mut response = client
-            .get(SOURCE_URL)
+            .get(&source_url)
             .send()
             .context("failed to download llama.cpp source tarball")?
             .error_for_status()

koharu-llm/src/lib.rs

@@ -15,14 +15,6 @@ pub use language::{Language, language_from_tag, supported_locales};
 pub use model::{GenerateOptions, Llm};
 pub use prompt::{ChatMessage, ChatRole};
 
-pub fn runtime_dir() -> PathBuf {
-    dirs::cache_dir()
-        .unwrap_or_else(std::env::temp_dir)
-        .join("Koharu")
-        .join("llama.cpp")
-        .join(env!("KOHARU_LLM_LLAMA_CPP_TAG"))
-}
-
 #[derive(
     Debug,
     Clone,

koharu-llm/src/model.rs

@@ -59,7 +59,6 @@ impl Default for GenerateOptions {
 
 impl Llm {
     pub async fn load(id: ModelId, cpu: bool) -> Result<Self> {
-        ensure_runtime_initialized()?;
         let model_path = id.get().await?;
 
         tokio::task::spawn_blocking(move || Self::load_from_path(id, cpu, model_path))
@@ -68,6 +67,9 @@ impl Llm {
     }
 
     fn load_from_path(id: ModelId, cpu: bool, model_path: PathBuf) -> Result<Self> {
+        crate::sys::initialize()
+            .context("failed to initialize llama.cpp runtime bindings")?;
+
         LOGGING_READY.call_once(|| {
             send_logs_to_tracing(LogOptions::default().with_logs_enabled(true));
         });
@@ -239,14 +241,6 @@ impl Llm {
     }
 }
 
-fn ensure_runtime_initialized() -> Result<()> {
-    anyhow::ensure!(
-        crate::sys::is_initialized(),
-        "llama.cpp runtime is not initialized; initialize it before calling `koharu_llm::Llm::load`"
-    );
-    Ok(())
-}
-
 fn model_params(cpu: bool, backend: &LlamaBackend) -> LlamaModelParams {
     if !cpu && backend.supports_gpu_offload() {
         LlamaModelParams::default().with_n_gpu_layers(DEFAULT_GPU_LAYERS)

koharu-llm/src/safe/mod.rs

@@ -27,7 +27,6 @@ pub mod llama_backend;
 pub mod llama_batch;
 pub mod model;
 pub mod mtmd;
-pub mod runtime;
 pub mod sampling;
 pub mod timing;
 pub mod token;

koharu-llm/src/safe/runtime.rs

@@ -1,884 +0,0 @@
-use std::env;
-use std::ffi::OsStr;
-use std::fs;
-use std::io;
-use std::path::{Path, PathBuf};
-
-use anyhow::{Context, Result, bail};
-use flate2::read::GzDecoder;
-use futures::StreamExt;
-use libloading::Library;
-use tokio::io::AsyncWriteExt;
-use zip::read::ZipArchive;
-
-const LLAMA_CPP_TAG: &str = env!("KOHARU_LLM_LLAMA_CPP_TAG");
-const RELEASE_BASE_URL: &str = "https://github.com/ggml-org/llama.cpp/releases/download";
-const STAMP_FILE: &str = ".koharu-llm-runtime-stamp";
-const DOWNLOADS_DIR: &str = ".downloads";
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum ArchiveKind {
-    Zip,
-    TarGz,
-}
-
-#[allow(dead_code)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum Backend {
-    Cuda,
-    Vulkan,
-    Default,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) enum CoreLibraryKind {
-    GgmlBase,
-    Ggml,
-    Llama,
-    Mtmd,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct LoadStep {
-    pub(crate) path: PathBuf,
-    pub(crate) core: Option<CoreLibraryKind>,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct ReleaseAsset {
-    name: &'static str,
-    kind: ArchiveKind,
-}
-
-#[derive(Debug, Clone, Copy)]
-struct Manifest {
-    id: &'static str,
-    support_libraries: &'static [&'static str],
-    support_prefixes: &'static [&'static str],
-    core_ggml_base: &'static str,
-    core_ggml: &'static str,
-    core_llama: &'static str,
-    core_mtmd: &'static str,
-    plugin_prefixes: &'static [&'static str],
-    library_extension: &'static str,
-    assets: &'static [ReleaseAsset],
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct TargetTriple {
-    os: &'static str,
-    arch: &'static str,
-    env: &'static str,
-}
-
-const WINDOWS_CUDA_ASSETS: &[ReleaseAsset] = &[
-    ReleaseAsset {
-        name: "llama-b8233-bin-win-cuda-13.1-x64.zip",
-        kind: ArchiveKind::Zip,
-    },
-    ReleaseAsset {
-        name: "cudart-llama-bin-win-cuda-13.1-x64.zip",
-        kind: ArchiveKind::Zip,
-    },
-];
-
-const WINDOWS_VULKAN_ASSETS: &[ReleaseAsset] = &[ReleaseAsset {
-    name: "llama-b8233-bin-win-vulkan-x64.zip",
-    kind: ArchiveKind::Zip,
-}];
-
-const LINUX_VULKAN_ASSETS: &[ReleaseAsset] = &[ReleaseAsset {
-    name: "llama-b8233-bin-ubuntu-vulkan-x64.tar.gz",
-    kind: ArchiveKind::TarGz,
-}];
-
-const MACOS_ARM64_ASSETS: &[ReleaseAsset] = &[ReleaseAsset {
-    name: "llama-b8233-bin-macos-arm64.tar.gz",
-    kind: ArchiveKind::TarGz,
-}];
-
-const WINDOWS_CUDA_MANIFEST: Manifest = Manifest {
-    id: "windows-cuda13-x64",
-    support_libraries: &["cudart64_13.dll", "cublasLt64_13.dll", "cublas64_13.dll"],
-    support_prefixes: &["libomp"],
-    core_ggml_base: "ggml-base.dll",
-    core_ggml: "ggml.dll",
-    core_llama: "llama.dll",
-    core_mtmd: "mtmd.dll",
-    plugin_prefixes: &["ggml-"],
-    library_extension: ".dll",
-    assets: WINDOWS_CUDA_ASSETS,
-};
-
-const WINDOWS_VULKAN_MANIFEST: Manifest = Manifest {
-    id: "windows-vulkan-x64",
-    support_libraries: &[],
-    support_prefixes: &["libomp"],
-    core_ggml_base: "ggml-base.dll",
-    core_ggml: "ggml.dll",
-    core_llama: "llama.dll",
-    core_mtmd: "mtmd.dll",
-    plugin_prefixes: &["ggml-"],
-    library_extension: ".dll",
-    assets: WINDOWS_VULKAN_ASSETS,
-};
-
-const LINUX_VULKAN_MANIFEST: Manifest = Manifest {
-    id: "linux-vulkan-x64",
-    support_libraries: &[],
-    support_prefixes: &[],
-    core_ggml_base: "libggml-base.so",
-    core_ggml: "libggml.so",
-    core_llama: "libllama.so",
-    core_mtmd: "libmtmd.so",
-    plugin_prefixes: &["libggml-"],
-    library_extension: ".so",
-    assets: LINUX_VULKAN_ASSETS,
-};
-
-const MACOS_ARM64_MANIFEST: Manifest = Manifest {
-    id: "macos-arm64",
-    support_libraries: &[],
-    support_prefixes: &[],
-    core_ggml_base: "libggml-base.dylib",
-    core_ggml: "libggml.dylib",
-    core_llama: "libllama.dylib",
-    core_mtmd: "libmtmd.dylib",
-    plugin_prefixes: &["libggml-"],
-    library_extension: ".dylib",
-    assets: MACOS_ARM64_ASSETS,
-};
-
-pub async fn ensure_dylibs(path: impl AsRef<Path>) -> Result<()> {
-    let root = path.as_ref().to_path_buf();
-    let manifest = detect_manifest()?;
-    let runtime_dir = resolve_runtime_dir(&root, manifest)?;
-
-    tokio::fs::create_dir_all(&runtime_dir)
-        .await
-        .with_context(|| {
-            format!(
-                "failed to create runtime directory `{}`",
-                runtime_dir.display()
-            )
-        })?;
-
-    if stamp_matches(&runtime_dir, manifest)? && core_libraries_present(&runtime_dir, manifest) {
-        return Ok(());
-    }
-
-    let downloads_dir = root.join(DOWNLOADS_DIR);
-    tokio::fs::create_dir_all(&downloads_dir)
-        .await
-        .with_context(|| {
-            format!(
-                "failed to create download cache `{}`",
-                downloads_dir.display()
-            )
-        })?;
-
-    for asset in manifest.assets {
-        let archive_path = download_asset(asset, &downloads_dir).await?;
-        let extract_root = runtime_dir.clone();
-        let asset_name = asset.name;
-        tokio::task::spawn_blocking(move || extract_archive(asset, &archive_path, &extract_root))
-            .await
-            .with_context(|| format!("failed to join extraction task for `{asset_name}`"))??;
-    }
-
-    write_stamp(&runtime_dir, manifest)?;
-    Ok(())
-}
-
-pub fn initialize(path: impl AsRef<Path>) -> Result<()> {
-    let manifest = detect_manifest()?;
-    let runtime_dir = resolve_runtime_dir(path.as_ref(), manifest)?;
-    crate::sys::initialize(&runtime_dir)
-}
-
-pub(crate) fn load_plan(dir: &Path) -> Result<Vec<LoadStep>> {
-    let manifest = manifest_for_runtime_dir(dir)?;
-    let mut steps = Vec::new();
-
-    for library_name in manifest.support_libraries {
-        let library_path = dir.join(library_name);
-        if library_path.exists() {
-            steps.push(LoadStep {
-                path: library_path,
-                core: None,
-            });
-        }
-    }
-
-    steps.extend(scan_prefixed_libraries(
-        dir,
-        manifest,
-        manifest.support_prefixes,
-        core_library_names(manifest),
-        manifest.support_libraries,
-    )?);
-
-    steps.push(required_core_step(
-        dir,
-        manifest.core_ggml_base,
-        CoreLibraryKind::GgmlBase,
-    )?);
-    steps.push(required_core_step(
-        dir,
-        manifest.core_ggml,
-        CoreLibraryKind::Ggml,
-    )?);
-
-    steps.extend(scan_prefixed_libraries(
-        dir,
-        manifest,
-        manifest.plugin_prefixes,
-        core_library_names(manifest),
-        manifest.support_libraries,
-    )?);
-
-    steps.push(required_core_step(
-        dir,
-        manifest.core_llama,
-        CoreLibraryKind::Llama,
-    )?);
-    steps.push(required_core_step(
-        dir,
-        manifest.core_mtmd,
-        CoreLibraryKind::Mtmd,
-    )?);
-
-    Ok(steps)
-}
-
-fn detect_manifest() -> Result<&'static Manifest> {
-    let target = current_target();
-    validate_target(target)?;
-    let backend = detect_backend(target)?;
-    select_manifest(target.os, target.arch, target.env, backend)
-}
-
-fn current_target() -> TargetTriple {
-    TargetTriple {
-        os: env::consts::OS,
-        arch: env::consts::ARCH,
-        env: target_env(),
-    }
-}
-
-fn target_env() -> &'static str {
-    #[cfg(target_env = "msvc")]
-    {
-        "msvc"
-    }
-
-    #[cfg(not(target_env = "msvc"))]
-    {
-        ""
-    }
-}
-
-fn validate_target(target: TargetTriple) -> Result<()> {
-    match (target.os, target.arch, target.env) {
-        ("windows", "x86_64", "msvc") | ("linux", "x86_64", _) | ("macos", "aarch64", _) => Ok(()),
-        _ => bail!(
-            "unsupported koharu-llm target: target_os={}, target_arch={}, target_env={}",
-            target.os,
-            target.arch,
-            target.env
-        ),
-    }
-}
-
-fn detect_backend(target: TargetTriple) -> Result<Backend> {
-    let has_cuda_driver = matches!(
-        (target.os, target.arch, target.env),
-        ("windows", "x86_64", "msvc")
-    ) && windows_has_cuda_driver();
-    detect_backend_for_target(target, has_cuda_driver)
-}
-
-fn detect_backend_for_target(target: TargetTriple, has_cuda_driver: bool) -> Result<Backend> {
-    match (target.os, target.arch, target.env) {
-        ("windows", "x86_64", "msvc") => Ok(if has_cuda_driver {
-            Backend::Cuda
-        } else {
-            Backend::Vulkan
-        }),
-        ("linux", "x86_64", _) => Ok(Backend::Vulkan),
-        ("macos", "aarch64", _) => Ok(Backend::Default),
-        _ => bail!(
-            "unsupported koharu-llm target/backend detection: target_os={}, target_arch={}, target_env={}",
-            target.os,
-            target.arch,
-            target.env
-        ),
-    }
-}
-
-fn windows_has_cuda_driver() -> bool {
-    unsafe { Library::new("nvcuda.dll") }.is_ok()
-}
-
-fn select_manifest(
-    target_os: &str,
-    target_arch: &str,
-    target_env: &str,
-    backend: Backend,
-) -> Result<&'static Manifest> {
-    match (target_os, target_arch, target_env, backend) {
-        ("windows", "x86_64", "msvc", Backend::Cuda) => Ok(&WINDOWS_CUDA_MANIFEST),
-        ("windows", "x86_64", "msvc", Backend::Vulkan) => Ok(&WINDOWS_VULKAN_MANIFEST),
-        ("linux", "x86_64", _, Backend::Vulkan) => Ok(&LINUX_VULKAN_MANIFEST),
-        ("macos", "aarch64", _, Backend::Default) => Ok(&MACOS_ARM64_MANIFEST),
-        _ => bail!(
-            "unsupported koharu-llm target/backend combination: target_os={target_os}, target_arch={target_arch}, target_env={target_env}, backend={backend:?}"
-        ),
-    }
-}
-
-fn resolve_runtime_dir(root: &Path, manifest: &Manifest) -> Result<PathBuf> {
-    if is_runtime_dir(root, manifest)? {
-        Ok(root.to_path_buf())
-    } else {
-        Ok(root.join(manifest.id))
-    }
-}
-
-fn is_runtime_dir(dir: &Path, manifest: &Manifest) -> Result<bool> {
-    if dir.file_name().and_then(OsStr::to_str) == Some(manifest.id) {
-        return Ok(true);
-    }
-
-    if let Some(existing_manifest) = manifest_from_stamp(dir)? {
-        return Ok(existing_manifest.id == manifest.id);
-    }
-
-    Ok(core_libraries_present(dir, manifest))
-}
-
-fn manifest_for_runtime_dir(dir: &Path) -> Result<&'static Manifest> {
-    if let Some(manifest) = manifest_from_stamp(dir)? {
-        return Ok(manifest);
-    }
-
-    if let Some(manifest) = dir
-        .file_name()
-        .and_then(OsStr::to_str)
-        .and_then(manifest_by_id)
-    {
-        return Ok(manifest);
-    }
-
-    let manifest = detect_manifest()?;
-    if core_libraries_present(dir, manifest) {
-        return Ok(manifest);
-    }
-
-    bail!(
-        "failed to determine the koharu-llm runtime manifest for `{}`",
-        dir.display()
-    )
-}
-
-fn manifest_from_stamp(dir: &Path) -> Result<Option<&'static Manifest>> {
-    let stamp_path = dir.join(STAMP_FILE);
-    if !stamp_path.exists() {
-        return Ok(None);
-    }
-
-    let contents = fs::read_to_string(&stamp_path)
-        .with_context(|| format!("failed to read `{}`", stamp_path.display()))?;
-    Ok(parse_manifest_from_stamp(&contents))
-}
-
-fn parse_manifest_from_stamp(contents: &str) -> Option<&'static Manifest> {
-    let mut lines = contents.lines();
-    let _tag = lines.next()?;
-    let id = lines.next()?;
-    manifest_by_id(id)
-}
-
-fn manifest_by_id(id: &str) -> Option<&'static Manifest> {
-    if id == WINDOWS_CUDA_MANIFEST.id {
-        Some(&WINDOWS_CUDA_MANIFEST)
-    } else if id == WINDOWS_VULKAN_MANIFEST.id {
-        Some(&WINDOWS_VULKAN_MANIFEST)
-    } else if id == LINUX_VULKAN_MANIFEST.id {
-        Some(&LINUX_VULKAN_MANIFEST)
-    } else if id == MACOS_ARM64_MANIFEST.id {
-        Some(&MACOS_ARM64_MANIFEST)
-    } else {
-        None
-    }
-}
-
-fn core_libraries_present(dir: &Path, manifest: &Manifest) -> bool {
-    core_library_names(manifest)
-        .iter()
-        .all(|name| dir.join(name).exists())
-}
-
-fn core_library_names(manifest: &Manifest) -> [&'static str; 4] {
-    [
-        manifest.core_ggml_base,
-        manifest.core_ggml,
-        manifest.core_llama,
-        manifest.core_mtmd,
-    ]
-}
-
-fn required_core_step(
-    dir: &Path,
-    library_name: &'static str,
-    core: CoreLibraryKind,
-) -> Result<LoadStep> {
-    let path = dir.join(library_name);
-    if !path.exists() {
-        bail!(
-            "required runtime library `{}` is missing from `{}`; run `koharu_llm::runtime::ensure_dylibs` first",
-            library_name,
-            dir.display()
-        );
-    }
-
-    Ok(LoadStep {
-        path,
-        core: Some(core),
-    })
-}
-
-fn scan_prefixed_libraries(
-    dir: &Path,
-    manifest: &Manifest,
-    prefixes: &[&str],
-    core_names: [&str; 4],
-    explicit_support_names: &[&str],
-) -> Result<Vec<LoadStep>> {
-    let mut paths = Vec::new();
-
-    for entry in fs::read_dir(dir)
-        .with_context(|| format!("failed to read runtime directory `{}`", dir.display()))?
-    {
-        let entry = entry.context("failed to read runtime directory entry")?;
-        let path = entry.path();
-        if !path.is_file() {
-            continue;
-        }
-
-        let Some(file_name) = path.file_name().and_then(OsStr::to_str) else {
-            continue;
-        };
-        if !file_name.ends_with(manifest.library_extension) {
-            continue;
-        }
-        if core_names.contains(&file_name) || explicit_support_names.contains(&file_name) {
-            continue;
-        }
-        if prefixes.iter().any(|prefix| file_name.starts_with(prefix)) {
-            paths.push(LoadStep { path, core: None });
-        }
-    }
-
-    paths.sort_by(|left, right| left.path.cmp(&right.path));
-    Ok(paths)
-}
-
-fn release_asset_url(asset_name: &str) -> String {
-    format!("{RELEASE_BASE_URL}/{LLAMA_CPP_TAG}/{asset_name}")
-}
-
-async fn download_asset(asset: &ReleaseAsset, downloads_dir: &Path) -> Result<PathBuf> {
-    let archive_path = downloads_dir.join(asset.name);
-    if archive_path.exists() {
-        return Ok(archive_path);
-    }
-
-    let partial_path = downloads_dir.join(format!("{}.partial", asset.name));
-    let url = release_asset_url(asset.name);
-    let response = reqwest::Client::builder()
-        .user_agent("koharu-llm-runtime")
-        .build()
-        .context("failed to build reqwest client")?
-        .get(&url)
-        .send()
-        .await
-        .with_context(|| format!("failed to download `{url}`"))?
-        .error_for_status()
-        .with_context(|| format!("download failed for `{url}`"))?;
-
-    let mut file = tokio::fs::File::create(&partial_path)
-        .await
-        .with_context(|| format!("failed to create `{}`", partial_path.display()))?;
-    let mut stream = response.bytes_stream();
-
-    while let Some(chunk) = stream.next().await {
-        let chunk = chunk.with_context(|| format!("failed while downloading `{url}`"))?;
-        file.write_all(&chunk)
-            .await
-            .with_context(|| format!("failed to write `{}`", partial_path.display()))?;
-    }
-
-    file.flush()
-        .await
-        .with_context(|| format!("failed to flush `{}`", partial_path.display()))?;
-    drop(file);
-
-    tokio::fs::rename(&partial_path, &archive_path)
-        .await
-        .with_context(|| format!("failed to finalize `{}`", archive_path.display()))?;
-
-    Ok(archive_path)
-}
-
-fn extract_archive(asset: &ReleaseAsset, archive_path: &Path, output_dir: &Path) -> Result<()> {
-    match asset.kind {
-        ArchiveKind::Zip => extract_zip_archive(archive_path, output_dir),
-        ArchiveKind::TarGz => extract_tar_archive(archive_path, output_dir),
-    }
-}
-
-fn extract_zip_archive(archive_path: &Path, output_dir: &Path) -> Result<()> {
-    let file = fs::File::open(archive_path)
-        .with_context(|| format!("failed to open `{}`", archive_path.display()))?;
-    let mut archive = ZipArchive::new(file)
-        .with_context(|| format!("failed to read zip archive `{}`", archive_path.display()))?;
-
-    for index in 0..archive.len() {
-        let mut entry = archive.by_index(index).with_context(|| {
-            format!(
-                "failed to read zip entry {index} from `{}`",
-                archive_path.display()
-            )
-        })?;
-        if entry.is_dir() {
-            continue;
-        }
-
-        let Some(file_name) = Path::new(entry.name())
-            .file_name()
-            .and_then(OsStr::to_str)
-            .map(ToOwned::to_owned)
-        else {
-            continue;
-        };
-        if !looks_like_runtime_library(&file_name) {
-            continue;
-        }
-
-        let out_path = output_dir.join(&file_name);
-        let mut out_file = fs::File::create(&out_path)
-            .with_context(|| format!("failed to create `{}`", out_path.display()))?;
-        io::copy(&mut entry, &mut out_file)
-            .with_context(|| format!("failed to extract `{}`", out_path.display()))?;
-    }
-
-    Ok(())
-}
-
-fn extract_tar_archive(archive_path: &Path, output_dir: &Path) -> Result<()> {
-    let file = fs::File::open(archive_path)
-        .with_context(|| format!("failed to open `{}`", archive_path.display()))?;
-    let decoder = GzDecoder::new(file);
-    let mut archive = tar::Archive::new(decoder);
-    let mut aliases = Vec::new();
-
-    for entry in archive.entries().with_context(|| {
-        format!(
-            "failed to read tar entries from `{}`",
-            archive_path.display()
-        )
-    })? {
-        let mut entry = entry.context("failed to read tar entry")?;
-        let path = entry.path().context("failed to read tar entry path")?;
-        let Some(file_name) = path
-            .file_name()
-            .and_then(OsStr::to_str)
-            .map(ToOwned::to_owned)
-        else {
-            continue;
-        };
-        if !looks_like_runtime_library(&file_name) {
-            continue;
-        }
-
-        let entry_type = entry.header().entry_type();
-        if entry_type.is_symlink() {
-            let Some(target_name) = entry
-                .link_name()
-                .context("failed to read tar symlink target")?
-                .and_then(|target| target.file_name().map(ToOwned::to_owned))
-                .and_then(|name| name.to_str().map(ToOwned::to_owned))
-            else {
-                continue;
-            };
-            aliases.push((output_dir.join(file_name), output_dir.join(target_name)));
-            continue;
-        }
-
-        if !entry_type.is_file() {
-            continue;
-        }
-
-        let out_path = output_dir.join(&file_name);
-        let mut out_file = fs::File::create(&out_path)
-            .with_context(|| format!("failed to create `{}`", out_path.display()))?;
-        io::copy(&mut entry, &mut out_file)
-            .with_context(|| format!("failed to extract `{}`", out_path.display()))?;
-    }
-
-    materialize_aliases(&aliases)?;
-    Ok(())
-}
-
-fn materialize_aliases(aliases: &[(PathBuf, PathBuf)]) -> Result<()> {
-    let mut pending = aliases.to_vec();
-
-    while !pending.is_empty() {
-        let mut progressed = false;
-        let mut next = Vec::new();
-
-        for (alias_path, target_path) in pending {
-            if alias_path.exists() {
-                progressed = true;
-                continue;
-            }
-            if !target_path.exists() {
-                next.push((alias_path, target_path));
-                continue;
-            }
-
-            match fs::hard_link(&target_path, &alias_path) {
-                Ok(()) => {}
-                Err(_) => {
-                    fs::copy(&target_path, &alias_path).with_context(|| {
-                        format!(
-                            "failed to materialize alias `{}` from `{}`",
-                            alias_path.display(),
-                            target_path.display()
-                        )
-                    })?;
-                }
-            }
-            progressed = true;
-        }
-
-        if !progressed {
-            let unresolved = next
-                .into_iter()
-                .map(|(alias, target)| format!("{} -> {}", alias.display(), target.display()))
-                .collect::<Vec<_>>()
-                .join(", ");
-            bail!("failed to materialize runtime library aliases: {unresolved}");
-        }
-
-        pending = next;
-    }
-
-    Ok(())
-}
-
-fn looks_like_runtime_library(file_name: &str) -> bool {
-    file_name.ends_with(".dll")
-        || file_name.ends_with(".so")
-        || file_name.contains(".so.")
-        || file_name.ends_with(".dylib")
-        || file_name.contains(".dylib.")
-}
-
-fn stamp_matches(root: &Path, manifest: &Manifest) -> Result<bool> {
-    let stamp_path = root.join(STAMP_FILE);
-    if !stamp_path.exists() {
-        return Ok(false);
-    }
-
-    let contents = fs::read_to_string(&stamp_path)
-        .with_context(|| format!("failed to read `{}`", stamp_path.display()))?;
-    Ok(contents == manifest_stamp(manifest))
-}
-
-fn write_stamp(root: &Path, manifest: &Manifest) -> Result<()> {
-    let stamp_path = root.join(STAMP_FILE);
-    fs::write(&stamp_path, manifest_stamp(manifest))
-        .with_context(|| format!("failed to write `{}`", stamp_path.display()))?;
-    Ok(())
-}
-
-fn manifest_stamp(manifest: &Manifest) -> String {
-    let mut stamp = format!("{LLAMA_CPP_TAG}\n{}\n", manifest.id);
-    for asset in manifest.assets {
-        stamp.push_str(asset.name);
-        stamp.push('\n');
-    }
-    stamp
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn touch(path: &Path) {
-        fs::write(path, b"ok").unwrap();
-    }
-
-    #[test]
-    fn selects_supported_manifests() {
-        assert_eq!(
-            select_manifest("windows", "x86_64", "msvc", Backend::Cuda)
-                .unwrap()
-                .id,
-            WINDOWS_CUDA_MANIFEST.id
-        );
-        assert_eq!(
-            select_manifest("windows", "x86_64", "msvc", Backend::Vulkan)
-                .unwrap()
-                .id,
-            WINDOWS_VULKAN_MANIFEST.id
-        );
-        assert_eq!(
-            select_manifest("linux", "x86_64", "", Backend::Vulkan)
-                .unwrap()
-                .id,
-            LINUX_VULKAN_MANIFEST.id
-        );
-        assert_eq!(
-            select_manifest("macos", "aarch64", "", Backend::Default)
-                .unwrap()
-                .id,
-            MACOS_ARM64_MANIFEST.id
-        );
-    }
-
-    #[test]
-    fn detects_backends_for_supported_targets() {
-        let windows = TargetTriple {
-            os: "windows",
-            arch: "x86_64",
-            env: "msvc",
-        };
-        let linux = TargetTriple {
-            os: "linux",
-            arch: "x86_64",
-            env: "",
-        };
-        let macos = TargetTriple {
-            os: "macos",
-            arch: "aarch64",
-            env: "",
-        };
-
-        assert_eq!(
-            detect_backend_for_target(windows, true).unwrap(),
-            Backend::Cuda
-        );
-        assert_eq!(
-            detect_backend_for_target(windows, false).unwrap(),
-            Backend::Vulkan
-        );
-        assert_eq!(
-            detect_backend_for_target(linux, true).unwrap(),
-            Backend::Vulkan
-        );
-        assert_eq!(
-            detect_backend_for_target(macos, true).unwrap(),
-            Backend::Default
-        );
-    }
-
-    #[test]
-    fn rejects_unsupported_manifest() {
-        assert!(select_manifest("linux", "x86_64", "", Backend::Cuda).is_err());
-        assert!(select_manifest("windows", "aarch64", "msvc", Backend::Cuda).is_err());
-    }
-
-    #[test]
-    fn resolves_manifest_from_stamp() {
-        assert_eq!(
-            parse_manifest_from_stamp(&manifest_stamp(&WINDOWS_VULKAN_MANIFEST))
-                .unwrap()
-                .id,
-            WINDOWS_VULKAN_MANIFEST.id
-        );
-    }
-
-    #[test]
-    fn manifest_stamp_is_stable() {
-        let stamp = manifest_stamp(&WINDOWS_CUDA_MANIFEST);
-        assert!(stamp.contains(LLAMA_CPP_TAG));
-        assert!(stamp.contains("windows-cuda13-x64"));
-        assert!(stamp.contains("llama-b8233-bin-win-cuda-13.1-x64.zip"));
-    }
-
-    #[test]
-    fn materializes_aliases_by_copy() {
-        let tempdir = tempfile::tempdir().unwrap();
-        let target = tempdir.path().join("libllama.so.0.0.8233");
-        let alias = tempdir.path().join("libllama.so");
-
-        touch(&target);
-        materialize_aliases(&[(alias.clone(), target.clone())]).unwrap();
-
-        assert!(alias.exists());
-        assert_eq!(fs::read(alias).unwrap(), fs::read(target).unwrap());
-    }
-
-    #[test]
-    fn load_plan_orders_core_and_plugins() {
-        let tempdir = tempfile::tempdir().unwrap();
-        let root = tempdir.path();
-        let manifest = &WINDOWS_CUDA_MANIFEST;
-
-        for core in core_library_names(manifest) {
-            touch(&root.join(core));
-        }
-        for support in manifest.support_libraries {
-            touch(&root.join(support));
-        }
-        touch(&root.join("libomp140.x86_64.dll"));
-        touch(&root.join("ggml-cuda.dll"));
-        touch(&root.join("ggml-rpc.dll"));
-        write_stamp(root, manifest).unwrap();
-
-        let steps = load_plan(root).unwrap();
-        let names = steps
-            .iter()
-            .map(|step| {
-                step.path
-                    .file_name()
-                    .unwrap()
-                    .to_string_lossy()
-                    .into_owned()
-            })
-            .collect::<Vec<_>>();
-
-        assert_eq!(names[0], "cudart64_13.dll");
-        assert_eq!(names[1], "cublasLt64_13.dll");
-        assert_eq!(names[2], "cublas64_13.dll");
-        assert_eq!(names[3], "libomp140.x86_64.dll");
-        assert_eq!(names[4], "ggml-base.dll");
-        assert_eq!(names[5], "ggml.dll");
-        assert!(names[6..names.len() - 2].contains(&"ggml-cuda.dll".to_string()));
-        assert_eq!(names[names.len() - 2], "llama.dll");
-        assert_eq!(names[names.len() - 1], "mtmd.dll");
-    }
-
-    #[tokio::test]
-    #[ignore]
-    async fn downloads_runtime_assets() -> Result<()> {
-        let tempdir = tempfile::tempdir()?;
-        ensure_dylibs(tempdir.path()).await?;
-        let manifest = detect_manifest()?;
-        let runtime_dir = resolve_runtime_dir(tempdir.path(), manifest)?;
-        assert!(core_libraries_present(&runtime_dir, manifest));
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[ignore]
-    async fn initializes_runtime_assets() -> Result<()> {
-        let tempdir = tempfile::tempdir()?;
-        ensure_dylibs(tempdir.path()).await?;
-        initialize(tempdir.path())?;
-        Ok(())
-    }
-}

koharu-llm/src/sys/mod.rs

@@ -1,11 +1,8 @@
 use std::ffi::CString;
 use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
 
 use anyhow::{Context, Result, anyhow, bail};
-use libloading::Library;
-use once_cell::sync::OnceCell;
-
-use crate::safe::runtime::{CoreLibraryKind, LoadStep};
 
 #[allow(warnings)]
 mod generated {
@@ -17,28 +14,24 @@ mod generated {
     #[allow(warnings)]
     pub mod llama {
         use super::types::*;
-
         include!(concat!(env!("OUT_DIR"), "/llama_loader.rs"));
     }
 
     #[allow(warnings)]
     pub mod ggml {
         use super::types::*;
-
         include!(concat!(env!("OUT_DIR"), "/ggml_loader.rs"));
     }
 
     #[allow(warnings)]
     pub mod ggml_base {
         use super::types::*;
-
         include!(concat!(env!("OUT_DIR"), "/ggml_base_loader.rs"));
     }
 
     #[allow(warnings)]
     pub mod mtmd {
         use super::types::*;
-
         include!(concat!(env!("OUT_DIR"), "/mtmd_loader.rs"));
     }
 }
@@ -47,37 +40,56 @@ pub use generated::types::*;
 
 struct LoadedLibraries {
     path: PathBuf,
-    auxiliary: Vec<Library>,
     llama: generated::llama::llama,
     ggml: generated::ggml::ggml,
     ggml_base: generated::ggml_base::ggml_base,
     mtmd: generated::mtmd::mtmd,
 }
 
-static LIBRARIES: OnceCell<LoadedLibraries> = OnceCell::new();
+#[cfg(target_os = "windows")]
+const LIB_NAMES: [&str; 4] = ["ggml-base.dll", "ggml.dll", "llama.dll", "mtmd.dll"];
 
-pub(crate) fn is_initialized() -> bool {
-    LIBRARIES.get().is_some()
-}
+#[cfg(target_os = "linux")]
+const LIB_NAMES: [&str; 4] = ["libggml-base.so", "libggml.so", "libllama.so", "libmtmd.so"];
 
-pub(crate) fn initialize(dir: &Path) -> Result<()> {
-    let canonical_dir = canonical_dir(dir)?;
+#[cfg(target_os = "macos")]
+const LIB_NAMES: [&str; 4] = [
+    "libggml-base.dylib",
+    "libggml.dylib",
+    "libllama.dylib",
+    "libmtmd.dylib",
+];
+
+static LIBRARIES: OnceLock<LoadedLibraries> = OnceLock::new();
+
+pub(crate) fn initialize() -> Result<()> {
+    let runtime_dir = koharu_runtime::llama_runtime_dir()
+        .context("failed to resolve the llama runtime directory; call `koharu_runtime::initialize()` first")?;
+
+    if !runtime_dir.exists() {
+        bail!(
+            "runtime directory `{}` does not exist; call `koharu_runtime::initialize()` first",
+            runtime_dir.display()
+        );
+    }
 
-    if let Some(libraries) = LIBRARIES.get() {
-        if libraries.path == canonical_dir {
+    let dir = runtime_dir.canonicalize().with_context(|| {
+        format!("failed to canonicalize `{}`", runtime_dir.display())
+    })?;
+
+    if let Some(existing) = LIBRARIES.get() {
+        if existing.path == dir {
             return Ok(());
         }
-
         bail!(
             "koharu-llm is already initialized with `{}` and cannot be reinitialized with `{}`",
-            libraries.path.display(),
-            canonical_dir.display()
+            existing.path.display(),
+            dir.display()
         );
     }
 
-    let load_plan = crate::safe::runtime::load_plan(&canonical_dir)?;
-    let libraries = load_libraries(&canonical_dir, &load_plan)?;
-    register_backends(&libraries.ggml, &canonical_dir)?;
+    let libraries = load_libraries(&dir)?;
+    register_backends(&libraries.ggml, &dir)?;
 
     LIBRARIES
         .set(libraries)
@@ -86,80 +98,44 @@ pub(crate) fn initialize(dir: &Path) -> Result<()> {
     Ok(())
 }
 
-fn canonical_dir(dir: &Path) -> Result<PathBuf> {
-    if dir.exists() {
-        dir.canonicalize().with_context(|| {
-            format!(
-                "failed to canonicalize runtime directory `{}`",
-                dir.display()
-            )
-        })
-    } else {
-        bail!(
-            "runtime directory `{}` does not exist; call `koharu_llm::runtime::ensure_dylibs` first",
-            dir.display()
-        )
-    }
-}
-
-fn load_libraries(dir: &Path, load_plan: &[LoadStep]) -> Result<LoadedLibraries> {
-    let mut auxiliary = Vec::new();
-    let mut llama = None;
-    let mut ggml = None;
-    let mut ggml_base = None;
-    let mut mtmd = None;
-
-    for step in load_plan {
-        let library = load_library(&step.path)?;
-
-        match step.core {
-            Some(CoreLibraryKind::Llama) => {
-                llama = Some(
-                    unsafe { generated::llama::llama::from_library(library) }
-                        .with_context(|| format!("failed to bind `{}`", step.path.display()))?,
-                );
-            }
-            Some(CoreLibraryKind::Ggml) => {
-                ggml = Some(
-                    unsafe { generated::ggml::ggml::from_library(library) }
-                        .with_context(|| format!("failed to bind `{}`", step.path.display()))?,
-                );
-            }
-            Some(CoreLibraryKind::GgmlBase) => {
-                ggml_base = Some(
-                    unsafe { generated::ggml_base::ggml_base::from_library(library) }
-                        .with_context(|| format!("failed to bind `{}`", step.path.display()))?,
-                );
-            }
-            Some(CoreLibraryKind::Mtmd) => {
-                mtmd = Some(
-                    unsafe { generated::mtmd::mtmd::from_library(library) }
-                        .with_context(|| format!("failed to bind `{}`", step.path.display()))?,
-                );
-            }
-            None => auxiliary.push(library),
-        }
-    }
+fn load_libraries(dir: &Path) -> Result<LoadedLibraries> {
+    let [ggml_base_name, ggml_name, llama_name, mtmd_name] = LIB_NAMES;
+
+    let ggml_base = load_and_bind(ggml_base_name, |lib| unsafe {
+        generated::ggml_base::ggml_base::from_library(lib)
+    })?;
+    let ggml = load_and_bind(ggml_name, |lib| unsafe {
+        generated::ggml::ggml::from_library(lib)
+    })?;
+    let llama = load_and_bind(llama_name, |lib| unsafe {
+        generated::llama::llama::from_library(lib)
+    })?;
+    let mtmd = load_and_bind(mtmd_name, |lib| unsafe {
+        generated::mtmd::mtmd::from_library(lib)
+    })?;
 
     Ok(LoadedLibraries {
         path: dir.to_path_buf(),
-        auxiliary,
-        llama: llama.ok_or_else(|| anyhow!("core llama runtime library was not loaded"))?,
-        ggml: ggml.ok_or_else(|| anyhow!("core ggml runtime library was not loaded"))?,
-        ggml_base: ggml_base
-            .ok_or_else(|| anyhow!("core ggml-base runtime library was not loaded"))?,
-        mtmd: mtmd.ok_or_else(|| anyhow!("core mtmd runtime library was not loaded"))?,
+        llama,
+        ggml,
+        ggml_base,
+        mtmd,
     })
 }
 
-fn load_library(path: &Path) -> Result<Library> {
-    unsafe { Library::new(path) }.with_context(|| format!("failed to load `{}`", path.display()))
+fn load_and_bind<T>(
+    name: &str,
+    bind: impl FnOnce(libloading::Library) -> std::result::Result<T, libloading::Error>,
+) -> Result<T> {
+    let library = koharu_runtime::load_library_by_name(name)
+        .with_context(|| format!("failed to load `{name}`"))?;
+    bind(library).with_context(|| format!("failed to bind `{name}`"))
 }
 
 fn register_backends(ggml: &generated::ggml::ggml, dir: &Path) -> Result<()> {
     let dir = dir
         .to_str()
-        .ok_or_else(|| anyhow!("runtime directory `{}` is not valid UTF-8", dir.display()))?;
+        .ok_or_else(|| anyhow!("runtime directory is not valid UTF-8"))?;
     let dir = CString::new(dir).context("runtime directory contains an interior null byte")?;
 
     unsafe {
@@ -171,12 +147,11 @@ fn register_backends(ggml: &generated::ggml::ggml, dir: &Path) -> Result<()> {
 
 fn libraries() -> &'static LoadedLibraries {
     LIBRARIES.get().expect(
-        "koharu-llm runtime libraries are not initialized; call `koharu_llm::runtime::initialize` first",
+        "koharu-llm runtime libraries are not initialized; call `koharu_runtime::initialize()` first",
     )
 }
 
 fn llama_lib() -> &'static generated::llama::llama {
-    let _ = libraries().auxiliary.len();
     &libraries().llama
 }
 
@@ -195,7 +170,6 @@ fn mtmd_lib() -> &'static generated::mtmd::mtmd {
 #[allow(warnings)]
 mod wrappers {
     use super::*;
-
     include!(concat!(env!("OUT_DIR"), "/wrappers.rs"));
 }
 

koharu-llm/tests/llm.rs

@@ -5,9 +5,7 @@ use strum::IntoEnumIterator;
 use koharu_llm::{GenerateOptions, Language, Llm, ModelId};
 
 async fn initialize_runtime() -> anyhow::Result<()> {
-    let runtime_dir = koharu_llm::runtime_dir();
-    koharu_llm::safe::runtime::ensure_dylibs(&runtime_dir).await?;
-    koharu_llm::safe::runtime::initialize(&runtime_dir)?;
+    koharu_runtime::initialize().await?;
     Ok(())
 }
 

koharu-runtime/Cargo.toml

@@ -14,23 +14,16 @@ publish.workspace = true
 [dependencies]
 koharu-http = { workspace = true }
 anyhow = { workspace = true }
+dirs = { workspace = true }
 serde_json = { workspace = true }
 zip = { workspace = true }
-rayon = { workspace = true }
-tracing = { workspace = true }
-reqwest = { workspace = true }
 tokio = { workspace = true }
-futures = { workspace = true }
-once_cell = { workspace = true }
 libloading = { workspace = true }
-num_cpus = { workspace = true }
-csv = { workspace = true }
 flate2 = { workspace = true }
+tar = "0.4"
+
+[target.'cfg(target_os = "windows")'.dependencies]
+windows-sys = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }
-
-[features]
-default = ["cuda", "cudnn"]
-cuda = []
-cudnn = []

koharu-runtime/src/archive.rs

@@ -0,0 +1,263 @@
+use std::ffi::OsStr;
+use std::fs;
+use std::io::{self, Write};
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result, bail};
+use flate2::read::GzDecoder;
+use koharu_http::download;
+
+const RUNTIME_LIB_EXTENSIONS: &[&str] = &[".dll", ".so", ".dylib"];
+
+pub(crate) async fn download_cached(
+    url: &str,
+    file_name: &str,
+    downloads_dir: &Path,
+) -> Result<PathBuf> {
+    let archive_path = downloads_dir.join(file_name);
+    if archive_path.exists() {
+        return Ok(archive_path);
+    }
+
+    let partial_path = downloads_dir.join(format!("{file_name}.partial"));
+    let bytes = download::bytes(url)
+        .await
+        .with_context(|| format!("failed to download `{url}`"))?;
+
+    let mut file = fs::File::create(&partial_path)
+        .with_context(|| format!("failed to create `{}`", partial_path.display()))?;
+    file.write_all(&bytes)
+        .with_context(|| format!("failed to write `{}`", partial_path.display()))?;
+    file.flush()?;
+
+    fs::rename(&partial_path, &archive_path)
+        .with_context(|| format!("failed to finalize `{}`", archive_path.display()))?;
+
+    Ok(archive_path)
+}
+
+/// Extract files matching known runtime library extensions from a zip archive.
+pub(crate) fn extract_zip(archive_path: &Path, output_dir: &Path) -> Result<()> {
+    let file = fs::File::open(archive_path)
+        .with_context(|| format!("failed to open `{}`", archive_path.display()))?;
+    let mut archive = zip::ZipArchive::new(file)
+        .with_context(|| format!("failed to read zip `{}`", archive_path.display()))?;
+
+    for index in 0..archive.len() {
+        let mut entry = archive.by_index(index)?;
+        if entry.is_dir() {
+            continue;
+        }
+
+        let Some(file_name) = Path::new(entry.name())
+            .file_name()
+            .and_then(OsStr::to_str)
+            .map(ToOwned::to_owned)
+        else {
+            continue;
+        };
+        if !looks_like_runtime_library(&file_name) {
+            continue;
+        }
+
+        let out_path = output_dir.join(file_name);
+        let mut out_file = fs::File::create(&out_path)
+            .with_context(|| format!("failed to create `{}`", out_path.display()))?;
+        io::copy(&mut entry, &mut out_file)
+            .with_context(|| format!("failed to extract `{}`", out_path.display()))?;
+    }
+
+    Ok(())
+}
+
+/// Extract specific files by name from a zip archive (case-insensitive match on basename).
+pub(crate) fn extract_zip_selected(
+    archive_path: &Path,
+    output_dir: &Path,
+    wanted_names: &[&str],
+) -> Result<()> {
+    let file = fs::File::open(archive_path)
+        .with_context(|| format!("failed to open `{}`", archive_path.display()))?;
+    let mut archive = zip::ZipArchive::new(file)
+        .with_context(|| format!("failed to read zip `{}`", archive_path.display()))?;
+
+    for index in 0..archive.len() {
+        let mut entry = archive.by_index(index)?;
+        if entry.is_dir() {
+            continue;
+        }
+
+        let entry_name = Path::new(entry.name())
+            .file_name()
+            .and_then(OsStr::to_str)
+            .unwrap_or(entry.name());
+        if !wanted_names
+            .iter()
+            .any(|name| entry_name.eq_ignore_ascii_case(name))
+        {
+            continue;
+        }
+
+        let out_path = output_dir.join(entry_name);
+        let mut out_file = fs::File::create(&out_path)
+            .with_context(|| format!("failed to create `{}`", out_path.display()))?;
+        io::copy(&mut entry, &mut out_file)
+            .with_context(|| format!("failed to extract `{}`", out_path.display()))?;
+    }
+
+    Ok(())
+}
+
+/// Extract files matching known runtime library extensions from a tar.gz archive.
+pub(crate) fn extract_tar_gz(archive_path: &Path, output_dir: &Path) -> Result<()> {
+    let file = fs::File::open(archive_path)
+        .with_context(|| format!("failed to open `{}`", archive_path.display()))?;
+    let mut archive = tar::Archive::new(GzDecoder::new(file));
+    let mut aliases = Vec::new();
+
+    for entry in archive
+        .entries()
+        .with_context(|| format!("failed to read tar `{}`", archive_path.display()))?
+    {
+        let mut entry = entry.context("failed to read tar entry")?;
+        let path = entry.path().context("failed to read tar entry path")?;
+        let Some(file_name) = path
+            .file_name()
+            .and_then(OsStr::to_str)
+            .map(ToOwned::to_owned)
+        else {
+            continue;
+        };
+        if !looks_like_runtime_library(&file_name) {
+            continue;
+        }
+
+        let entry_type = entry.header().entry_type();
+        if entry_type.is_symlink() {
+            let Some(target_name) = entry
+                .link_name()
+                .context("failed to read tar symlink target")?
+                .and_then(|t| t.file_name().map(ToOwned::to_owned))
+                .and_then(|n| n.to_str().map(ToOwned::to_owned))
+            else {
+                continue;
+            };
+            aliases.push((output_dir.join(file_name), output_dir.join(target_name)));
+            continue;
+        }
+
+        if !entry_type.is_file() {
+            continue;
+        }
+
+        let out_path = output_dir.join(&file_name);
+        let mut out_file = fs::File::create(&out_path)
+            .with_context(|| format!("failed to create `{}`", out_path.display()))?;
+        io::copy(&mut entry, &mut out_file)
+            .with_context(|| format!("failed to extract `{}`", out_path.display()))?;
+    }
+
+    materialize_aliases(&aliases)
+}
+
+fn looks_like_runtime_library(file_name: &str) -> bool {
+    RUNTIME_LIB_EXTENSIONS
+        .iter()
+        .any(|ext| file_name.ends_with(ext) || file_name.contains(&format!("{ext}.")))
+}
+
+fn materialize_aliases(aliases: &[(PathBuf, PathBuf)]) -> Result<()> {
+    let mut pending = aliases.to_vec();
+
+    while !pending.is_empty() {
+        let mut progressed = false;
+        let mut next = Vec::new();
+
+        for (alias_path, target_path) in pending {
+            if alias_path.exists() {
+                progressed = true;
+                continue;
+            }
+            if !target_path.exists() {
+                next.push((alias_path, target_path));
+                continue;
+            }
+
+            fs::hard_link(&target_path, &alias_path)
+                .or_else(|_| fs::copy(&target_path, &alias_path).map(|_| ()))
+                .with_context(|| {
+                    format!(
+                        "failed to create alias `{}` -> `{}`",
+                        alias_path.display(),
+                        target_path.display()
+                    )
+                })?;
+            progressed = true;
+        }
+
+        if !progressed {
+            let unresolved: Vec<_> = next
+                .iter()
+                .map(|(a, t)| format!("{} -> {}", a.display(), t.display()))
+                .collect();
+            bail!("unresolvable aliases: {}", unresolved.join(", "));
+        }
+
+        pending = next;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+
+    use super::*;
+
+    #[test]
+    fn materializes_aliases_by_copy() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let target = tempdir.path().join("libllama.so.0.0.8233");
+        let alias = tempdir.path().join("libllama.so");
+
+        fs::write(&target, b"ok").unwrap();
+        materialize_aliases(&[(alias.clone(), target.clone())]).unwrap();
+
+        assert!(alias.exists());
+        assert_eq!(fs::read(&alias).unwrap(), fs::read(&target).unwrap());
+    }
+
+    #[test]
+    fn extract_zip_selected_filters_by_name() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let archive_path = tempdir.path().join("test.zip");
+        let output_dir = tempdir.path().join("out");
+        fs::create_dir_all(&output_dir).unwrap();
+
+        let file = fs::File::create(&archive_path).unwrap();
+        let mut zip = zip::ZipWriter::new(file);
+        let options = zip::write::SimpleFileOptions::default();
+        zip.start_file("bin/cudart64_13.dll", options).unwrap();
+        zip.write_all(b"cuda").unwrap();
+        zip.start_file("bin/ignored.txt", options).unwrap();
+        zip.write_all(b"ignore").unwrap();
+        zip.finish().unwrap();
+
+        extract_zip_selected(&archive_path, &output_dir, &["cudart64_13.dll"]).unwrap();
+
+        assert_eq!(
+            fs::read(output_dir.join("cudart64_13.dll")).unwrap(),
+            b"cuda"
+        );
+        assert!(!output_dir.join("ignored.txt").exists());
+    }
+
+    #[test]
+    fn runtime_library_detection() {
+        assert!(looks_like_runtime_library("ggml.dll"));
+        assert!(looks_like_runtime_library("libllama.so.0.0.8233"));
+        assert!(looks_like_runtime_library("libggml-metal.0.9.7.dylib"));
+        assert!(!looks_like_runtime_library("README.md"));
+    }
+}

koharu-runtime/src/cuda.rs

@@ -0,0 +1,281 @@
+use std::fmt;
+use std::path::Path;
+
+use anyhow::{Context, Result, anyhow, bail};
+use koharu_http::http::http_client;
+use libloading::Library;
+
+use crate::archive;
+use crate::loader::{add_runtime_search_path, preload_library};
+
+const CUDA_SUCCESS: i32 = 0;
+const CUDA_13_1_DRIVER_VERSION: i32 = 13010;
+
+type CuInit = unsafe extern "C" fn(flags: u32) -> i32;
+type CuDriverGetVersion = unsafe extern "C" fn(driver_version: *mut i32) -> i32;
+
+// ── Public driver API ────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct CudaDriverVersion {
+    raw: i32,
+}
+
+impl CudaDriverVersion {
+    pub const fn from_raw(raw: i32) -> Self {
+        Self { raw }
+    }
+
+    pub const fn raw(self) -> i32 {
+        self.raw
+    }
+
+    pub const fn major(self) -> i32 {
+        self.raw / 1000
+    }
+
+    pub const fn minor(self) -> i32 {
+        (self.raw % 1000) / 10
+    }
+
+    pub const fn supports_cuda_13_1(self) -> bool {
+        self.raw >= CUDA_13_1_DRIVER_VERSION
+    }
+}
+
+impl fmt::Display for CudaDriverVersion {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}.{}", self.major(), self.minor())
+    }
+}
+
+pub fn driver_version() -> Result<CudaDriverVersion> {
+    let lib_name = if cfg!(target_os = "windows") {
+        "nvcuda.dll"
+    } else {
+        "libcuda.so"
+    };
+
+    unsafe {
+        let library = Library::new(lib_name)
+            .with_context(|| format!("failed to load NVIDIA driver library {lib_name}"))?;
+        let cu_init = *library
+            .get::<CuInit>(b"cuInit\0")
+            .context("failed to load cuInit from NVIDIA driver")?;
+        let cu_driver_get_version = *library
+            .get::<CuDriverGetVersion>(b"cuDriverGetVersion\0")
+            .context("failed to load cuDriverGetVersion from NVIDIA driver")?;
+
+        let status = cu_init(0);
+        if status != CUDA_SUCCESS {
+            bail!("cuInit failed with CUDA driver error code {status}");
+        }
+
+        let mut raw = 0;
+        let status = cu_driver_get_version(&mut raw);
+        if status != CUDA_SUCCESS {
+            bail!("cuDriverGetVersion failed with CUDA driver error code {status}");
+        }
+
+        Ok(CudaDriverVersion::from_raw(raw))
+    }
+}
+
+pub(crate) fn is_available() -> bool {
+    #[cfg(target_os = "windows")]
+    return unsafe { Library::new("nvcuda.dll") }.is_ok();
+
+    #[cfg(target_os = "linux")]
+    return unsafe { Library::new("libcuda.so.1") }.is_ok();
+
+    #[allow(unreachable_code)]
+    false
+}
+
+// ── Wheel installation ───────────────────────────────────────────────
+
+#[allow(dead_code)]
+struct CudaWheel {
+    name: &'static str,
+    windows_dylibs: &'static [&'static str],
+    linux_dylibs: &'static [&'static str],
+}
+
+const WHEELS: &[CudaWheel] = &[
+    CudaWheel {
+        name: "nvidia-cuda-runtime/13.1.80",
+        windows_dylibs: &["cudart64_13.dll"],
+        linux_dylibs: &["libcudart.so.13"],
+    },
+    CudaWheel {
+        name: "nvidia-cublas/13.2.1.1",
+        windows_dylibs: &["cublasLt64_13.dll", "cublas64_13.dll"],
+        linux_dylibs: &["libcublasLt.so.13", "libcublas.so.13"],
+    },
+    CudaWheel {
+        name: "nvidia-cufft/12.1.0.78",
+        windows_dylibs: &["cufft64_12.dll"],
+        linux_dylibs: &["libcufft.so.12"],
+    },
+    CudaWheel {
+        name: "nvidia-curand/10.4.1.81",
+        windows_dylibs: &["curand64_10.dll"],
+        linux_dylibs: &["libcurand.so.10"],
+    },
+];
+
+fn platform_tags() -> Result<&'static [&'static str]> {
+    #[cfg(all(target_os = "windows", target_arch = "x86_64"))]
+    return Ok(&["win_amd64"]);
+
+    #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
+    return Ok(&["manylinux_2_27_x86_64", "manylinux_2_17_x86_64"]);
+
+    #[cfg(not(any(
+        all(target_os = "windows", target_arch = "x86_64"),
+        all(target_os = "linux", target_arch = "x86_64")
+    )))]
+    bail!(
+        "CUDA wheels unsupported on {}/{}",
+        std::env::consts::OS,
+        std::env::consts::ARCH
+    )
+}
+
+impl CudaWheel {
+    fn dylibs(&self) -> &'static [&'static str] {
+        #[cfg(target_os = "windows")]
+        return self.windows_dylibs;
+
+        #[cfg(target_os = "linux")]
+        return self.linux_dylibs;
+
+        #[allow(unreachable_code)]
+        &[]
+    }
+}
+
+fn source_id() -> Result<String> {
+    let wheels: Vec<&str> = WHEELS.iter().map(|w| w.name).collect();
+    Ok(format!(
+        "cuda;platform={};wheels={}",
+        platform_tags()?.join(","),
+        wheels.join(",")
+    ))
+}
+
+pub(crate) async fn ensure_ready(root: &Path, downloads_dir: &Path) -> Result<()> {
+    let install_dir = root.join("cuda");
+    let source_id = source_id()?;
+
+    if !crate::is_up_to_date(&install_dir, &source_id) {
+        crate::reset_dir(&install_dir)?;
+
+        let tags = platform_tags()?;
+        for wheel in WHEELS {
+            let (url, filename) = select_wheel(wheel.name, tags).await?;
+            let archive = archive::download_cached(&url, &filename, downloads_dir).await?;
+            archive::extract_zip_selected(&archive, &install_dir, wheel.dylibs())?;
+        }
+
+        crate::mark_installed(&install_dir, &source_id)?;
+    }
+
+    add_runtime_search_path(&install_dir)?;
+    for wheel in WHEELS {
+        for dylib in wheel.dylibs() {
+            let path = install_dir.join(dylib);
+            if path.exists() {
+                preload_library(&path)?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn select_wheel(package: &str, tags: &[&str]) -> Result<(String, String)> {
+    let (dist, version) = package
+        .split_once('/')
+        .ok_or_else(|| anyhow!("invalid wheel package `{package}`"))?;
+
+    let meta_url = format!("https://pypi.org/pypi/{dist}/{version}/json");
+    let json: serde_json::Value = http_client()
+        .get(&meta_url)
+        .send()
+        .await
+        .with_context(|| format!("failed to fetch `{meta_url}`"))?
+        .json()
+        .await
+        .with_context(|| format!("failed to parse metadata for `{dist}`"))?;
+
+    let files = json
+        .get("urls")
+        .and_then(|v| v.as_array())
+        .ok_or_else(|| anyhow!("bad PyPI json for `{dist}`"))?;
+
+    for file in files {
+        let filename = file.get("filename").and_then(|v| v.as_str()).unwrap_or("");
+        let url = file.get("url").and_then(|v| v.as_str()).unwrap_or("");
+
+        if filename.ends_with(".whl") && tags.iter().any(|tag| filename.contains(tag)) {
+            return Ok((url.to_string(), filename.to_string()));
+        }
+    }
+
+    bail!("no wheel found for `{dist}` {version} on {tags:?}")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn source_id_includes_platform() {
+        let id = source_id().unwrap();
+        assert!(id.contains("cuda"));
+        assert!(id.contains("platform="));
+    }
+
+    #[test]
+    fn wheels_have_dylibs_for_current_platform() {
+        for wheel in WHEELS {
+            #[cfg(any(target_os = "windows", target_os = "linux"))]
+            assert!(!wheel.dylibs().is_empty(), "{} has no dylibs", wheel.name);
+        }
+    }
+
+    #[test]
+    fn preload_order_follows_wheel_declaration() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let root = tempdir.path();
+
+        for wheel in WHEELS {
+            for dylib in wheel.dylibs() {
+                std::fs::write(root.join(dylib), b"ok").unwrap();
+            }
+        }
+
+        let all_dylibs: Vec<&str> =
+            WHEELS.iter().flat_map(|w| w.dylibs().iter().copied()).collect();
+        for dylib in &all_dylibs {
+            assert!(root.join(dylib).exists());
+        }
+    }
+
+    #[test]
+    fn parses_major_minor_from_driver_version() {
+        let version = CudaDriverVersion::from_raw(13010);
+        assert_eq!(version.major(), 13);
+        assert_eq!(version.minor(), 1);
+        assert_eq!(version.to_string(), "13.1");
+    }
+
+    #[test]
+    fn checks_cuda_13_1_threshold() {
+        assert!(CudaDriverVersion::from_raw(13010).supports_cuda_13_1());
+        assert!(CudaDriverVersion::from_raw(13020).supports_cuda_13_1());
+        assert!(!CudaDriverVersion::from_raw(13000).supports_cuda_13_1());
+        assert!(!CudaDriverVersion::from_raw(12080).supports_cuda_13_1());
+    }
+}

koharu-runtime/src/dylib.rs

@@ -1,394 +0,0 @@
-use std::{
-    fs,
-    io::{BufWriter, Read, Write},
-    path::{Path, PathBuf},
-    sync::Arc,
-};
-
-use anyhow::{Context, Result};
-use futures::stream::{self, StreamExt, TryStreamExt};
-use koharu_http::download;
-use koharu_http::http::http_client;
-use once_cell::sync::OnceCell;
-use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
-use tokio::task;
-use tracing::debug;
-
-use crate::zip::fetch_record;
-
-/// Keep handles to loaded dynamic libraries alive for process lifetime
-static DYLIB_HANDLES: OnceCell<Vec<libloading::Library>> = OnceCell::new();
-
-#[derive(Clone, Copy)]
-struct DylibSpec {
-    /// Basename as shipped inside the wheel/archive
-    archive_filename: &'static str,
-    /// Basename to write locally (may differ to accommodate preload expectations)
-    alias_filename: &'static str,
-    /// Whether we should preload the library eagerly
-    preload: bool,
-}
-
-#[allow(unused)]
-const fn dylib(name: &'static str) -> DylibSpec {
-    DylibSpec {
-        archive_filename: name,
-        alias_filename: name,
-        preload: true,
-    }
-}
-
-#[allow(unused)]
-const fn dylib_with_alias(
-    archive_filename: &'static str,
-    alias_filename: &'static str,
-) -> DylibSpec {
-    DylibSpec {
-        archive_filename,
-        alias_filename,
-        preload: true,
-    }
-}
-
-#[allow(unused)]
-const fn skip_preload(spec: DylibSpec) -> DylibSpec {
-    DylibSpec {
-        preload: false,
-        ..spec
-    }
-}
-
-/// CUDA packages to pull wheels for
-pub const PACKAGES: &[&str] = &[
-    #[cfg(feature = "cuda")]
-    "nvidia-cuda-runtime/13.1.80",
-    #[cfg(feature = "cuda")]
-    "nvidia-cublas/13.2.1.1",
-    #[cfg(feature = "cuda")]
-    "nvidia-cufft/12.1.0.78",
-    #[cfg(feature = "cuda")]
-    "nvidia-curand/10.4.1.81",
-    #[cfg(feature = "cudnn")]
-    "nvidia-cudnn-cu13/9.19.0.56",
-];
-
-/// Hard-coded load list by platform
-#[cfg(target_os = "windows")]
-const DYLIBS: &[DylibSpec] = &[
-    // Core CUDA runtime and BLAS/FFT
-    #[cfg(feature = "cuda")]
-    dylib("cudart64_13.dll"),
-    #[cfg(feature = "cuda")]
-    dylib("cublasLt64_13.dll"),
-    #[cfg(feature = "cuda")]
-    dylib("cublas64_13.dll"),
-    #[cfg(feature = "cuda")]
-    dylib("cufft64_12.dll"),
-    #[cfg(feature = "cuda")]
-    dylib("curand64_10.dll"),
-    // cuDNN core and dependency chain (graph -> ops -> adv/cnn)
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_graph64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_ops64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_heuristic64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_adv64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_cnn64_9.dll"),
-    // cuDNN engine packs (may require NVRTC/NVJITLINK; load last, ignore failures)
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_engines_precompiled64_9.dll"),
-    #[cfg(feature = "cudnn")]
-    dylib("cudnn_engines_runtime_compiled64_9.dll"),
-];
-
-#[cfg(target_os = "macos")]
-const DYLIBS: &[DylibSpec] = &[];
-
-#[cfg(all(not(target_os = "windows"), not(target_os = "macos")))]
-const DYLIBS: &[DylibSpec] = &[
-    // Core CUDA runtime and BLAS/FFT (sonames)
-    #[cfg(feature = "cuda")]
-    dylib("libcudart.so.13"),
-    #[cfg(feature = "cuda")]
-    dylib("libcublasLt.so.13"),
-    #[cfg(feature = "cuda")]
-    dylib("libcublas.so.13"),
-    #[cfg(feature = "cuda")]
-    dylib("libcufft.so.12"),
-    #[cfg(feature = "cuda")]
-    dylib("libcurand.so.10"),
-    // cuDNN core and dependency chain
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_graph.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_ops.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_heuristic.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_adv.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_cnn.so.9"),
-    // cuDNN engine packs
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_engines_precompiled.so.9"),
-    #[cfg(feature = "cudnn")]
-    dylib("libcudnn_engines_runtime_compiled.so.9"),
-];
-
-pub async fn ensure_dylibs(path: impl AsRef<Path>) -> Result<()> {
-    let path = path.as_ref().to_owned();
-
-    fs::create_dir_all(&path)?;
-
-    let platform_tags = current_platform_tags()?;
-    debug!("ensure_dylibs: start -> {}", path.display());
-
-    let out_dir = Arc::new(path);
-    let packages: Vec<String> = PACKAGES.iter().map(|s| s.to_string()).collect();
-
-    let fetches = packages.into_iter().map(|pkg| {
-        let out_dir = Arc::clone(&out_dir);
-        async move { fetch_and_extract(pkg.as_str(), platform_tags, out_dir).await }
-    });
-
-    stream::iter(fetches)
-        .buffer_unordered(num_cpus::get())
-        .try_collect::<Vec<_>>()
-        .await?;
-
-    debug!("ensure_dylibs: done");
-    Ok(())
-}
-
-/// Preload runtime dynamic libraries with a dependency-friendly order.
-/// Keeps the library handles alive for the process lifetime.
-pub fn preload_dylibs(dir: impl AsRef<Path>) -> Result<()> {
-    let dir = dir.as_ref();
-
-    let mut libs = Vec::new();
-
-    // Load exactly in our hard-coded order; skip names that are not present.
-    for spec in DYLIBS {
-        let path = dir.join(spec.alias_filename);
-        if !path.exists() {
-            continue;
-        }
-
-        if !spec.preload {
-            continue;
-        }
-
-        unsafe {
-            match libloading::Library::new(&path) {
-                Ok(lib) => libs.push(lib),
-                Err(err) => {
-                    anyhow::bail!("preload_dylibs: failed {}: {}", path.display(), err);
-                }
-            }
-        }
-    }
-
-    DYLIB_HANDLES
-        .set(libs)
-        .map_err(|_| anyhow::anyhow!("preload_dylibs: already initialized"))?;
-    Ok(())
-}
-
-fn wanted_spec(path: &str) -> Option<&'static DylibSpec> {
-    let base = Path::new(path)
-        .file_name()
-        .and_then(|s| s.to_str())
-        .unwrap_or(path);
-
-    DYLIBS
-        .iter()
-        .find(|&want| base.eq_ignore_ascii_case(want.archive_filename))
-        .map(|v| v as _)
-}
-
-fn current_platform_tags() -> Result<&'static [&'static str]> {
-    if cfg!(target_os = "windows") {
-        Ok(&["win_amd64"])
-    } else if cfg!(all(target_os = "linux", target_arch = "x86_64")) {
-        Ok(&["manylinux_2_27_x86_64", "manylinux_2_17_x86_64"])
-    } else if cfg!(target_os = "macos") {
-        Ok(&["macosx_13_0_universal2"])
-    } else {
-        anyhow::bail!("unsupported platform for runtime bundling");
-    }
-}
-
-async fn fetch_and_extract(pkg: &str, platform_tags: &[&str], out_dir: Arc<PathBuf>) -> Result<()> {
-    // 1) Query PyPI JSON
-    let meta_url = format!("https://pypi.org/pypi/{pkg}/json");
-    let resp = http_client()
-        .get(&meta_url)
-        .send()
-        .await
-        .context("failed to fetch package metadata")?;
-    let json: serde_json::Value = resp
-        .json()
-        .await
-        .context("failed to parse package metadata")?;
-
-    // 2) Choose a wheel
-    let files = json
-        .get("urls")
-        .and_then(|v| v.as_array())
-        .ok_or_else(|| anyhow::anyhow!("bad json: urls"))?;
-    let mut chosen: Option<(String, String)> = None; // (url, filename)
-    for f in files {
-        let filename = f.get("filename").and_then(|v| v.as_str()).unwrap_or("");
-        let file_url = f.get("url").and_then(|v| v.as_str()).unwrap_or("");
-
-        if !filename.ends_with(".whl") {
-            continue;
-        }
-        if platform_tags.iter().any(|tag| filename.contains(tag)) {
-            chosen = Some((file_url.to_string(), filename.to_string()));
-            break;
-        }
-    }
-    let (wheel_url, wheel_name) = chosen.ok_or_else(|| anyhow::anyhow!("no suitable wheel"))?;
-    debug!("{pkg}: selected wheel {wheel_name}");
-
-    // 3) Use RECORD to check local dylibs; download only if needed
-    let entries = fetch_record(&wheel_url)
-        .await
-        .context("failed to fetch RECORD")?;
-
-    // Fast path: existence + size-only check; no hashing.
-    // If size is None and file exists, treat as OK (no further verification).
-    let needs_download = entries
-        .par_iter()
-        .filter_map(|e| wanted_spec(&e.path).map(|spec| (spec.alias_filename, e.size)))
-        .any(|(base, rec_size)| {
-            let local = out_dir.as_ref().join(base);
-            if !local.exists() {
-                return true;
-            }
-            match (local.metadata(), rec_size) {
-                (Ok(meta), Some(sz)) => meta.len() != sz,
-                _ => false,
-            }
-        });
-
-    if needs_download {
-        debug!("{pkg}: downloading {wheel_name}...");
-        let bytes = download::bytes(&wheel_url)
-            .await
-            .context("failed to download wheel")?;
-        let out = Arc::clone(&out_dir);
-
-        task::spawn_blocking(move || extract_from_wheel(&bytes, out.as_ref()))
-            .await?
-            .context("failed to extract dylibs")?;
-        debug!("{pkg}: download and extract complete");
-        Ok(())
-    } else {
-        debug!("{pkg}: {wheel_name} runtime libs are up-to-date");
-        Ok(())
-    }
-}
-
-fn extract_from_wheel(bytes: &[u8], out_dir: &Path) -> Result<()> {
-    // First, list target entries to extract
-    let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
-    let mut targets: Vec<(String, String)> = Vec::new(); // (full archive path, output basename)
-    for i in 0..archive.len() {
-        let file = archive.by_index(i)?;
-        if let Some(spec) = wanted_spec(file.name()) {
-            targets.push((file.name().to_owned(), spec.alias_filename.to_owned()));
-        }
-    }
-    drop(archive);
-
-    if targets.is_empty() {
-        anyhow::bail!("no runtime libraries found in wheel");
-    }
-
-    let results: Result<Vec<(String, u64)>> = targets
-        .par_iter()
-        .map(|(full_name, base_name)| -> Result<(String, u64)> {
-            let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes))?;
-            let mut file = zip.by_name(full_name)?;
-
-            let out_path = out_dir.join(base_name);
-            let out = fs::File::create(&out_path)?;
-            // Preallocate to uncompressed size if known to reduce fragmentation.
-            let _ = out.set_len(file.size());
-
-            // Buffered copy with large chunk size
-            let mut writer = BufWriter::with_capacity(8 * 1024 * 1024, out);
-            let mut buf = vec![0u8; 8 * 1024 * 1024];
-            let mut written: u64 = 0;
-            loop {
-                let n = file.read(&mut buf)?;
-                if n == 0 {
-                    break;
-                }
-                writer.write_all(&buf[..n])?;
-                written += n as u64;
-            }
-            writer.flush()?;
-            Ok((base_name.clone(), written))
-        })
-        .collect();
-
-    let results = results?;
-    let _total_bytes: u64 = results.iter().map(|(_, w)| *w).sum();
-    debug!(
-        "extract: copied {} libraries into {}",
-        results.len(),
-        out_dir.display()
-    );
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    #[ignore]
-    async fn test_skip_download_if_up_to_date() -> Result<()> {
-        let temp_dir = tempfile::tempdir()?;
-        let out_dir = temp_dir.path();
-
-        let t0 = std::time::Instant::now();
-        ensure_dylibs(out_dir).await?;
-
-        let elapsed = t0.elapsed();
-        println!("Elapsed time: {:?}", elapsed);
-
-        let t1 = std::time::Instant::now();
-        ensure_dylibs(out_dir).await?;
-
-        let elapsed = t1.elapsed();
-        println!("Elapsed time: {:?}", elapsed);
-
-        assert!(elapsed < t0.elapsed());
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[ignore]
-    async fn test_preload_dylibs() -> Result<()> {
-        let temp_dir = std::env::temp_dir();
-        let out_dir = temp_dir.join("cuda_rt_test_dylibs");
-
-        ensure_dylibs(&out_dir).await?;
-        preload_dylibs(&out_dir)?;
-
-        Ok(())
-    }
-}

koharu-runtime/src/lib.rs

@@ -1,4 +1,107 @@
-mod dylib;
-mod zip;
+mod archive;
+mod cuda;
+mod llama;
+mod loader;
 
-pub use dylib::{ensure_dylibs, preload_dylibs};
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+
+pub use cuda::{CudaDriverVersion, driver_version as cuda_driver_version};
+pub use loader::load_library_by_name;
+
+const DOWNLOADS_DIR: &str = ".downloads";
+const INSTALLED_MARKER: &str = ".installed";
+
+pub async fn initialize() -> Result<()> {
+    initialize_with_root(&runtime_root()).await
+}
+
+async fn initialize_with_root(root: &Path) -> Result<()> {
+    fs::create_dir_all(root)
+        .with_context(|| format!("failed to create runtime root `{}`", root.display()))?;
+    let downloads_dir = root.join(DOWNLOADS_DIR);
+    fs::create_dir_all(&downloads_dir)
+        .with_context(|| format!("failed to create `{}`", downloads_dir.display()))?;
+
+    if cuda::is_available() {
+        cuda::ensure_ready(root, &downloads_dir).await?;
+    }
+    llama::ensure_ready(root, &downloads_dir).await?;
+
+    Ok(())
+}
+
+#[doc(hidden)]
+pub fn llama_runtime_dir() -> Result<PathBuf> {
+    llama::runtime_dir(&runtime_root())
+}
+
+fn runtime_root() -> PathBuf {
+    if let Some(path) = env::var_os("KOHARU_RUNTIME_ROOT") {
+        return PathBuf::from(path);
+    }
+    dirs::data_local_dir()
+        .unwrap_or_else(env::temp_dir)
+        .join("Koharu")
+        .join("runtime")
+}
+
+fn is_up_to_date(install_dir: &Path, source_id: &str) -> bool {
+    matches!(
+        fs::read_to_string(install_dir.join(INSTALLED_MARKER)),
+        Ok(content) if content == source_id
+    )
+}
+
+fn reset_dir(dir: &Path) -> Result<()> {
+    if dir.exists() {
+        fs::remove_dir_all(dir)
+            .with_context(|| format!("failed to reset `{}`", dir.display()))?;
+    }
+    fs::create_dir_all(dir).with_context(|| format!("failed to create `{}`", dir.display()))?;
+    Ok(())
+}
+
+fn mark_installed(install_dir: &Path, source_id: &str) -> Result<()> {
+    fs::write(install_dir.join(INSTALLED_MARKER), source_id)
+        .with_context(|| format!("failed to write marker in `{}`", install_dir.display()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    #[ignore]
+    async fn initializes_llama_runtime_into_shared_root() -> Result<()> {
+        let tempdir = tempfile::tempdir()?;
+        initialize_with_root(tempdir.path()).await?;
+        let dir = llama::runtime_dir(tempdir.path())?;
+        assert!(dir.exists());
+        Ok(())
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn repeated_basename_loads_succeed_after_runtime_initialize() -> Result<()> {
+        let tempdir = tempfile::tempdir()?;
+        initialize_with_root(tempdir.path()).await?;
+        let dir = llama::runtime_dir(tempdir.path())?;
+
+        let lib_name = fs::read_dir(&dir)?
+            .filter_map(|e| e.ok())
+            .filter_map(|e| {
+                let name = e.file_name().to_string_lossy().into_owned();
+                name.contains("llama").then_some(name)
+            })
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("no llama library found"))?;
+
+        let _first = load_library_by_name(&lib_name)?;
+        let _second = load_library_by_name(&lib_name)?;
+        Ok(())
+    }
+}

koharu-runtime/src/llama.rs

@@ -0,0 +1,259 @@
+use std::path::{Path, PathBuf};
+
+use anyhow::{Result, bail};
+use libloading::Library;
+
+use crate::archive;
+use crate::loader::{add_runtime_search_path, preload_library};
+
+const LLAMA_CPP_TAG: &str = env!("LLAMA_CPP_TAG");
+const RELEASE_BASE_URL: &str = "https://github.com/ggml-org/llama.cpp/releases/download";
+
+#[derive(Debug, Clone, Copy)]
+#[allow(dead_code)]
+enum LlamaRuntime {
+    WindowsCuda13X64,
+    WindowsVulkanX64,
+    LinuxVulkanX64,
+    MacosArm64,
+}
+
+impl LlamaRuntime {
+    #[allow(clippy::needless_return)]
+    fn detect() -> Result<Self> {
+        #[cfg(all(target_os = "windows", target_arch = "x86_64"))]
+        if unsafe { Library::new("nvcuda.dll") }.is_ok() {
+            return Ok(Self::WindowsCuda13X64);
+        } else {
+            return Ok(Self::WindowsVulkanX64);
+        }
+
+        #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
+        return Ok(Self::LinuxVulkanX64);
+
+        #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+        return Ok(Self::MacosArm64);
+
+        #[cfg(not(any(
+            all(target_os = "windows", target_arch = "x86_64"),
+            all(target_os = "linux", target_arch = "x86_64"),
+            all(target_os = "macos", target_arch = "aarch64")
+        )))]
+        bail!(
+            "unsupported platform: os={}, arch={}",
+            std::env::consts::OS,
+            std::env::consts::ARCH
+        )
+    }
+
+    fn id(self) -> &'static str {
+        match self {
+            Self::WindowsCuda13X64 => "windows-cuda13-x64",
+            Self::WindowsVulkanX64 => "windows-vulkan-x64",
+            Self::LinuxVulkanX64 => "linux-vulkan-x64",
+            Self::MacosArm64 => "macos-arm64",
+        }
+    }
+
+    fn assets(self) -> &'static [&'static str] {
+        match self {
+            Self::WindowsCuda13X64 => &[
+                "llama-b8233-bin-win-cuda-13.1-x64.zip",
+                "cudart-llama-bin-win-cuda-13.1-x64.zip",
+            ],
+            Self::WindowsVulkanX64 => &["llama-b8233-bin-win-vulkan-x64.zip"],
+            Self::LinuxVulkanX64 => &["llama-b8233-bin-ubuntu-vulkan-x64.tar.gz"],
+            Self::MacosArm64 => &["llama-b8233-bin-macos-arm64.tar.gz"],
+        }
+    }
+
+    fn libraries(self) -> &'static [&'static str] {
+        match self {
+            Self::WindowsCuda13X64 => &[
+                "cudart64_13.dll",
+                "cublasLt64_13.dll",
+                "cublas64_13.dll",
+                "libomp140.x86_64.dll",
+                "ggml-base.dll",
+                "ggml.dll",
+                "ggml-cpu-alderlake.dll",
+                "ggml-cpu-cannonlake.dll",
+                "ggml-cpu-cascadelake.dll",
+                "ggml-cpu-cooperlake.dll",
+                "ggml-cpu-haswell.dll",
+                "ggml-cpu-icelake.dll",
+                "ggml-cpu-ivybridge.dll",
+                "ggml-cpu-piledriver.dll",
+                "ggml-cpu-sandybridge.dll",
+                "ggml-cpu-sapphirerapids.dll",
+                "ggml-cpu-skylakex.dll",
+                "ggml-cpu-sse42.dll",
+                "ggml-cpu-x64.dll",
+                "ggml-cpu-zen4.dll",
+                "ggml-cuda.dll",
+                "ggml-rpc.dll",
+                "llama.dll",
+                "mtmd.dll",
+            ],
+            Self::WindowsVulkanX64 => &[
+                "libomp140.x86_64.dll",
+                "ggml-base.dll",
+                "ggml.dll",
+                "ggml-cpu-alderlake.dll",
+                "ggml-cpu-cannonlake.dll",
+                "ggml-cpu-cascadelake.dll",
+                "ggml-cpu-cooperlake.dll",
+                "ggml-cpu-haswell.dll",
+                "ggml-cpu-icelake.dll",
+                "ggml-cpu-ivybridge.dll",
+                "ggml-cpu-piledriver.dll",
+                "ggml-cpu-sandybridge.dll",
+                "ggml-cpu-sapphirerapids.dll",
+                "ggml-cpu-skylakex.dll",
+                "ggml-cpu-sse42.dll",
+                "ggml-cpu-x64.dll",
+                "ggml-cpu-zen4.dll",
+                "ggml-rpc.dll",
+                "ggml-vulkan.dll",
+                "llama.dll",
+                "mtmd.dll",
+            ],
+            Self::LinuxVulkanX64 => &[
+                "libggml-base.so",
+                "libggml.so",
+                "libggml-cpu-alderlake.so",
+                "libggml-cpu-cannonlake.so",
+                "libggml-cpu-cascadelake.so",
+                "libggml-cpu-cooperlake.so",
+                "libggml-cpu-haswell.so",
+                "libggml-cpu-icelake.so",
+                "libggml-cpu-ivybridge.so",
+                "libggml-cpu-piledriver.so",
+                "libggml-cpu-sandybridge.so",
+                "libggml-cpu-sapphirerapids.so",
+                "libggml-cpu-skylakex.so",
+                "libggml-cpu-sse42.so",
+                "libggml-cpu-x64.so",
+                "libggml-cpu-zen4.so",
+                "libggml-rpc.so",
+                "libggml-vulkan.so",
+                "libllama.so",
+                "libmtmd.so",
+            ],
+            Self::MacosArm64 => &[
+                "libggml-base.dylib",
+                "libggml.dylib",
+                "libggml-blas.dylib",
+                "libggml-cpu.dylib",
+                "libggml-metal.dylib",
+                "libggml-rpc.dylib",
+                "libllama.dylib",
+                "libmtmd.dylib",
+            ],
+        }
+    }
+
+    fn install_dir(self, root: &Path) -> PathBuf {
+        root.join("llama.cpp").join(LLAMA_CPP_TAG).join(self.id())
+    }
+
+    fn source_id(self) -> String {
+        format!("llama-{LLAMA_CPP_TAG}-{}", self.id())
+    }
+
+    fn is_zip_asset(asset: &str) -> bool {
+        asset.ends_with(".zip")
+    }
+
+    async fn install(self, install_dir: &Path, downloads_dir: &Path) -> Result<()> {
+        for asset in self.assets() {
+            let url = format!("{RELEASE_BASE_URL}/{LLAMA_CPP_TAG}/{asset}");
+            let archive = archive::download_cached(&url, asset, downloads_dir).await?;
+            if Self::is_zip_asset(asset) {
+                archive::extract_zip(&archive, install_dir)?;
+            } else {
+                archive::extract_tar_gz(&archive, install_dir)?;
+            }
+        }
+
+        for lib in self.libraries() {
+            if !install_dir.join(lib).exists() {
+                bail!(
+                    "required library `{lib}` missing from `{}`",
+                    install_dir.display()
+                );
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub(crate) async fn ensure_ready(root: &Path, downloads_dir: &Path) -> Result<()> {
+    let runtime = LlamaRuntime::detect()?;
+    let install_dir = runtime.install_dir(root);
+    let source_id = runtime.source_id();
+
+    if !crate::is_up_to_date(&install_dir, &source_id) {
+        crate::reset_dir(&install_dir)?;
+        runtime.install(&install_dir, downloads_dir).await?;
+        crate::mark_installed(&install_dir, &source_id)?;
+    }
+
+    add_runtime_search_path(&install_dir)?;
+    for lib in runtime.libraries() {
+        preload_library(&install_dir.join(lib))?;
+    }
+
+    Ok(())
+}
+
+pub(crate) fn runtime_dir(root: &Path) -> Result<PathBuf> {
+    Ok(LlamaRuntime::detect()?.install_dir(root))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs;
+    use std::path::Path;
+
+    use super::*;
+
+    fn touch(path: &Path) {
+        fs::write(path, b"ok").unwrap();
+    }
+
+    #[test]
+    fn detect_returns_a_variant_for_current_platform() {
+        let runtime = LlamaRuntime::detect().unwrap();
+        assert!(!runtime.id().is_empty());
+        assert!(!runtime.assets().is_empty());
+        assert!(!runtime.libraries().is_empty());
+    }
+
+    #[test]
+    fn install_dir_includes_tag_and_id() {
+        let root = Path::new("/tmp/rt");
+        let dir = LlamaRuntime::WindowsVulkanX64.install_dir(root);
+        assert!(dir.ends_with(
+            Path::new("llama.cpp")
+                .join(LLAMA_CPP_TAG)
+                .join("windows-vulkan-x64")
+        ));
+    }
+
+    #[test]
+    fn preload_order_matches_libraries() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let root = tempdir.path();
+        let rt = LlamaRuntime::WindowsCuda13X64;
+
+        for lib in rt.libraries() {
+            touch(&root.join(lib));
+        }
+
+        let paths: Vec<PathBuf> = rt.libraries().iter().map(|l| root.join(l)).collect();
+        assert!(paths.iter().all(|p| p.exists()));
+        assert_eq!(paths.len(), rt.libraries().len());
+    }
+}

koharu-runtime/src/loader.rs

@@ -0,0 +1,79 @@
+use std::ffi::OsStr;
+use std::path::Path;
+
+use anyhow::{Context, Result, bail};
+use libloading::Library;
+
+pub(crate) fn add_runtime_search_path(path: &Path) -> Result<()> {
+    #[cfg(not(target_os = "windows"))]
+    {
+        let _ = path;
+        Ok(())
+    }
+
+    #[cfg(target_os = "windows")]
+    {
+        use std::os::windows::ffi::OsStrExt;
+        use windows_sys::Win32::System::LibraryLoader::{
+            AddDllDirectory, LOAD_LIBRARY_SEARCH_SYSTEM32, LOAD_LIBRARY_SEARCH_USER_DIRS,
+            SetDefaultDllDirectories,
+        };
+
+        let path = path
+            .canonicalize()
+            .with_context(|| format!("failed to canonicalize `{}`", path.display()))?;
+        let wide = path
+            .as_os_str()
+            .encode_wide()
+            .chain(std::iter::once(0))
+            .collect::<Vec<_>>();
+        unsafe {
+            if SetDefaultDllDirectories(
+                LOAD_LIBRARY_SEARCH_USER_DIRS | LOAD_LIBRARY_SEARCH_SYSTEM32,
+            ) == 0
+            {
+                bail!(
+                    "failed to set default DLL directories: {}",
+                    std::io::Error::last_os_error()
+                );
+            }
+            if AddDllDirectory(wide.as_ptr()).is_null() {
+                bail!(
+                    "failed to add DLL directory: {}",
+                    std::io::Error::last_os_error()
+                );
+            }
+        }
+        Ok(())
+    }
+}
+
+pub(crate) fn preload_library(path: &Path) -> Result<()> {
+    let path = path
+        .canonicalize()
+        .with_context(|| format!("failed to canonicalize `{}`", path.display()))?;
+    let library = load_library(path.as_os_str())
+        .with_context(|| format!("failed to preload `{}`", path.display()))?;
+    std::mem::forget(library);
+    Ok(())
+}
+
+pub fn load_library_by_name(name: &str) -> Result<Library> {
+    load_library(OsStr::new(name)).with_context(|| format!("failed to load `{name}`"))
+}
+
+fn load_library(target: &OsStr) -> Result<Library> {
+    #[cfg(target_os = "windows")]
+    {
+        let library = unsafe { Library::new(target) }?;
+        Ok(library)
+    }
+
+    #[cfg(not(target_os = "windows"))]
+    {
+        use libloading::os::unix::{Library as UnixLibrary, RTLD_GLOBAL, RTLD_NOW};
+
+        let library = unsafe { UnixLibrary::open(Some(target), RTLD_NOW | RTLD_GLOBAL) }?;
+        Ok(library.into())
+    }
+}

koharu-runtime/src/zip.rs

@@ -1,226 +0,0 @@
-use std::io;
-
-use anyhow::Result;
-use flate2::read::DeflateDecoder;
-#[cfg(test)]
-use koharu_http::http::http_client;
-use koharu_http::range;
-
-// ZIP signatures and constants
-const SIG_EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; // End Of Central Directory
-const SIG_CFH: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; // Central Directory File Header
-const SIG_LFH: [u8; 4] = [0x50, 0x4b, 0x03, 0x04]; // Local File Header
-const EOCD_MIN_LEN: usize = 22; // Minimum EOCD size
-const CFH_FIXED_LEN: usize = 46; // CFH fixed-length portion
-const LFH_FIXED_LEN: u64 = 30; // LFH fixed-length portion
-
-// EOCD field offsets (relative to EOCD start)
-const EOCD_OFF_CD_SIZE: usize = 12; // u32: size of central directory
-const EOCD_OFF_CD_OFFSET: usize = 16; // u32: offset of central directory
-
-// CFH offsets (relative to CFH start)
-const CFH_OFF_COMP_METHOD: usize = 10; // u16
-const CFH_OFF_COMP_SIZE: usize = 20; // u32
-const CFH_OFF_NAME_LEN: usize = 28; // u16
-const CFH_OFF_EXTRA_LEN: usize = 30; // u16
-const CFH_OFF_COMMENT_LEN: usize = 32; // u16
-const CFH_OFF_LFH_OFFSET: usize = 42; // u32
-
-// LFH offsets (relative to LFH start)
-const LFH_OFF_NAME_LEN: usize = 26; // u16
-const LFH_OFF_EXTRA_LEN: usize = 28; // u16
-
-#[inline]
-fn le_u16(bytes: &[u8], off: usize) -> u16 {
-    u16::from_le_bytes([bytes[off], bytes[off + 1]])
-}
-
-#[inline]
-fn le_u32(bytes: &[u8], off: usize) -> u32 {
-    u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]])
-}
-
-#[allow(unused)]
-#[derive(Debug, Clone)]
-pub struct RecordEntry {
-    pub path: String,
-    pub hash: Option<String>, // e.g., Some("sha256-<urlsafe_b64>") or None
-    pub size: Option<u64>,
-}
-
-pub async fn fetch_record(url: &str) -> Result<Vec<RecordEntry>> {
-    // 1) Locate EOCD and central directory
-    let (cd_offset, cd_size) = http_zip_eocd_and_cd(url).await?;
-    let cd_bytes = range::get_range(url, cd_offset, cd_offset + cd_size - 1).await?;
-    let (lh_off, comp_size, comp_method) = parse_central_directory_for_record(&cd_bytes)?;
-
-    // 2) Read local header to compute exact data offset
-    let lh_fixed = range::get_range(url, lh_off, lh_off + LFH_FIXED_LEN - 1).await?;
-    if lh_fixed.len() < LFH_FIXED_LEN as usize || lh_fixed[0..4] != SIG_LFH {
-        anyhow::bail!("bad local file header");
-    }
-    let name_len = le_u16(&lh_fixed, LFH_OFF_NAME_LEN) as u64;
-    let extra_len = le_u16(&lh_fixed, LFH_OFF_EXTRA_LEN) as u64;
-    let data_off = lh_off + LFH_FIXED_LEN + name_len + extra_len;
-    let data_end = data_off + comp_size as u64 - 1;
-
-    // 3) Fetch and decode RECORD (deflate only, method 8)
-    let comp = range::get_range(url, data_off, data_end).await?;
-    if comp_method != 8 {
-        anyhow::bail!("RECORD compression method unsupported: {comp_method}");
-    }
-    let mut dec = DeflateDecoder::new(std::io::Cursor::new(comp));
-    let mut rec_csv = Vec::new();
-    io::copy(&mut dec, &mut rec_csv)?;
-
-    // 4) Parse CSV for all entries
-    parse_record_csv(&rec_csv)
-}
-
-fn parse_central_directory_for_record(cd: &[u8]) -> Result<(u64, u32, u16)> {
-    let mut i = 0usize;
-    while i + CFH_FIXED_LEN <= cd.len() {
-        if cd[i..i + 4] != SIG_CFH {
-            i += 1;
-            continue;
-        }
-
-        let comp_method = le_u16(cd, i + CFH_OFF_COMP_METHOD);
-        let comp_size = le_u32(cd, i + CFH_OFF_COMP_SIZE);
-        let name_len = le_u16(cd, i + CFH_OFF_NAME_LEN);
-        let extra_len = le_u16(cd, i + CFH_OFF_EXTRA_LEN);
-        let comment_len = le_u16(cd, i + CFH_OFF_COMMENT_LEN);
-        let lh_off = le_u32(cd, i + CFH_OFF_LFH_OFFSET) as u64;
-
-        let name_start = i + CFH_FIXED_LEN;
-        let name_end = name_start + name_len as usize;
-        if name_end > cd.len() {
-            anyhow::bail!("bad central directory entry");
-        }
-        let name = &cd[name_start..name_end];
-        let name_str = std::str::from_utf8(name).unwrap_or("");
-        if name_str.ends_with("/RECORD") && name_str.contains(".dist-info/") {
-            if comp_size == u32::MAX {
-                anyhow::bail!("zip64 not supported for RECORD");
-            }
-            return Ok((lh_off, comp_size, comp_method));
-        }
-
-        i = name_end + extra_len as usize + comment_len as usize;
-    }
-    anyhow::bail!("RECORD not found")
-}
-
-async fn http_zip_eocd_and_cd(url: &str) -> Result<(u64, u64)> {
-    // Fetch last ~70KiB to find EOCD (max comment is 64KiB; add slack)
-    let tail = range::get_tail(url, 70 * 1024).await?;
-    let mut found = None;
-    for i in (0..=tail.len().saturating_sub(EOCD_MIN_LEN)).rev() {
-        if tail[i..i + 4] == SIG_EOCD {
-            found = Some(i);
-            break;
-        }
-    }
-    let pos = found.ok_or_else(|| anyhow::anyhow!("zip EOCD not found"))?;
-    let eocd = &tail[pos..];
-    if eocd.len() < EOCD_MIN_LEN {
-        anyhow::bail!("truncated EOCD");
-    }
-    let cd_size = le_u32(eocd, EOCD_OFF_CD_SIZE) as u64;
-    let cd_off = le_u32(eocd, EOCD_OFF_CD_OFFSET) as u64;
-    Ok((cd_off, cd_size))
-}
-
-fn parse_record_csv(csv_bytes: &[u8]) -> Result<Vec<RecordEntry>> {
-    let mut rdr = csv::ReaderBuilder::new()
-        .has_headers(false)
-        .from_reader(csv_bytes);
-    let mut out = Vec::new();
-    for rec in rdr.records() {
-        let r = rec?;
-        if r.is_empty() {
-            continue;
-        }
-        let path = r.get(0).unwrap_or("").to_string();
-        let hash = r.get(1).and_then(|s| {
-            if s.is_empty() {
-                None
-            } else {
-                Some(s.to_string())
-            }
-        });
-        let size = r.get(2).and_then(|s| s.parse::<u64>().ok());
-        out.push(RecordEntry { path, hash, size });
-    }
-    Ok(out)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_pip_wheels() -> Result<()> {
-        async fn pick_wheel_url(pkg: &str, version: &str, tag: &str) -> Result<String> {
-            let meta_url = format!("https://pypi.org/pypi/{pkg}/{version}/json");
-            let resp = http_client().get(&meta_url).send().await?;
-            let json: serde_json::Value = resp.json().await?;
-            let urls = json
-                .get("urls")
-                .and_then(|v| v.as_array())
-                .ok_or_else(|| anyhow::anyhow!("bad json: urls"))?;
-
-            urls.iter()
-                .filter_map(|f| {
-                    let filename = f.get("filename").and_then(|v| v.as_str())?;
-                    let file_url = f.get("url").and_then(|v| v.as_str())?;
-                    if !filename.ends_with(".whl") {
-                        return None;
-                    }
-                    let matches = if tag == "manylinux" {
-                        filename.contains("manylinux") && filename.contains("x86_64")
-                    } else {
-                        filename.contains(tag)
-                    };
-                    if matches {
-                        Some(file_url.to_string())
-                    } else {
-                        None
-                    }
-                })
-                .next()
-                .ok_or_else(|| anyhow::anyhow!("no suitable wheel for tag {tag}"))
-        }
-
-        // Pin versions that are known to publish both win_amd64 and manylinux wheels.
-        // The latest PyPI release may only have a source tarball (e.g. cublas 12.9.2.10).
-        for (pkg, version) in [
-            ("nvidia-cuda-runtime-cu12", "12.9.79"),
-            ("nvidia-cublas-cu12", "12.9.1.4"),
-        ] {
-            for tag in ["win_amd64", "manylinux"] {
-                let url = pick_wheel_url(pkg, version, tag).await?;
-                let entries = fetch_record(&url).await?;
-                assert!(
-                    !entries.is_empty(),
-                    "{} {}: record entries should not be empty",
-                    pkg,
-                    tag
-                );
-                assert!(
-                    entries.iter().any(|e| e.path.ends_with("/RECORD")),
-                    "{} {}: RECORD file should be present",
-                    pkg,
-                    tag
-                );
-                assert!(
-                    entries.iter().any(|e| e.size.is_some()),
-                    "{} {}: some RECORD entries should have sizes",
-                    pkg,
-                    tag
-                );
-            }
-        }
-        Ok(())
-    }
-}

koharu/Cargo.toml

@@ -40,7 +40,6 @@ tauri-plugin-updater = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-libloading = { workspace = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 koharu-llm = { workspace = true }
@@ -55,8 +54,7 @@ koharu-llm = { workspace = true }
 koharu-llm = { workspace = true }
 
 [features]
-cuda = ["koharu-runtime/cuda", "koharu-ml/cuda"]
-cudnn = ["koharu-runtime/cudnn", "koharu-ml/cudnn"]
+cuda = ["koharu-ml/cuda"]
 metal = ["koharu-ml/metal"]
 
 [build-dependencies]

koharu/src/app.rs

@@ -8,12 +8,11 @@ use tauri::{Manager, WebviewWindowBuilder};
 use tokio::{net::TcpListener, sync::RwLock};
 use tracing_subscriber::fmt::format::FmtSpan;
 
-use koharu_llm::{facade, runtime_dir as llm_runtime_dir};
+use koharu_llm::facade;
 use koharu_ml::{cuda_is_available, device};
 use koharu_pipeline::AppResources;
 use koharu_renderer::facade::Renderer;
 use koharu_rpc::{SharedResources, server};
-use koharu_runtime::{ensure_dylibs, preload_dylibs};
 use koharu_types::State;
 
 static APP_ROOT: Lazy<PathBuf> = Lazy::new(|| {
@@ -21,8 +20,6 @@ static APP_ROOT: Lazy<PathBuf> = Lazy::new(|| {
         .map(|path| path.join("Koharu"))
         .unwrap_or_default()
 });
-static LIB_ROOT: Lazy<PathBuf> = Lazy::new(|| APP_ROOT.join("libs"));
-static LLAMA_RUNTIME_ROOT: Lazy<PathBuf> = Lazy::new(llm_runtime_dir);
 static MODEL_ROOT: Lazy<PathBuf> = Lazy::new(|| APP_ROOT.join("models"));
 
 #[derive(Parser)]
@@ -108,24 +105,14 @@ fn initialize(headless: bool, _debug: bool) -> Result<()> {
 }
 
 async fn prefetch() -> Result<()> {
-    ensure_dylibs(LIB_ROOT.to_path_buf()).await?;
-    koharu_llm::safe::runtime::ensure_dylibs(LLAMA_RUNTIME_ROOT.as_path())
+    koharu_runtime::initialize()
         .await
-        .context("Failed to ensure llama.cpp runtime libraries")?;
+        .context("Failed to initialize runtime packages")?;
     koharu_ml::facade::prefetch().await?;
 
     Ok(())
 }
 
-async fn initialize_llama_runtime() -> Result<()> {
-    koharu_llm::safe::runtime::ensure_dylibs(LLAMA_RUNTIME_ROOT.as_path())
-        .await
-        .context("Failed to ensure llama.cpp runtime libraries")?;
-    koharu_llm::safe::runtime::initialize(LLAMA_RUNTIME_ROOT.as_path())
-        .context("Failed to initialize llama.cpp runtime libraries")?;
-    Ok(())
-}
-
 fn warning(headless: bool, title: &str, description: &str) {
     tracing::warn!("{description}");
 
@@ -144,7 +131,7 @@ async fn build_resources(cpu: bool, headless: bool) -> Result<AppResources> {
     let mut cpu = cpu;
 
     if !cpu && cuda_is_available() {
-        match crate::nvidia::driver_version() {
+        match koharu_runtime::cuda_driver_version() {
             Ok(version) if version.supports_cuda_13_1() => {
                 tracing::info!("NVIDIA driver reports CUDA {version} support");
             }
@@ -171,29 +158,21 @@ async fn build_resources(cpu: bool, headless: bool) -> Result<AppResources> {
         }
     }
 
-    if !cpu && cuda_is_available() {
-        ensure_dylibs(LIB_ROOT.to_path_buf())
-            .await
-            .context("Failed to ensure dynamic libraries")?;
-        preload_dylibs(LIB_ROOT.to_path_buf()).context("Failed to preload dynamic libraries")?;
+    koharu_runtime::initialize()
+        .await
+        .context("Failed to initialize runtime packages")?;
 
+    if !cpu && cuda_is_available() {
         #[cfg(target_os = "windows")]
         {
             if let Err(err) = crate::windows::register_khr() {
                 tracing::warn!(?err, "Failed to register .khr file association");
             }
-
-            crate::windows::add_dll_directory(&LIB_ROOT).context("Failed to add DLL directory")?;
         }
 
-        tracing::info!(
-            "CUDA is available, loaded dynamic libraries from {:?}",
-            *LIB_ROOT
-        );
+        tracing::info!("CUDA is available and runtime packages were initialized");
     }
 
-    initialize_llama_runtime().await?;
-
     let ml = Arc::new(
         koharu_ml::facade::Model::new(cpu)
             .await
@@ -313,3 +292,4 @@ pub async fn run() -> Result<()> {
 
     Ok(())
 }
+

koharu/src/lib.rs

@@ -1,6 +1,5 @@
 pub mod app;
 pub mod assets;
-pub mod nvidia;
 pub mod version;
 #[cfg(target_os = "windows")]
 pub mod windows;

koharu/src/nvidia.rs

@@ -1,103 +0,0 @@
-use std::fmt;
-
-use anyhow::{Context, Result};
-
-const CUDA_SUCCESS: i32 = 0;
-const CUDA_13_1_DRIVER_VERSION: i32 = 13010;
-
-type CuInit = unsafe extern "C" fn(flags: u32) -> i32;
-type CuDriverGetVersion = unsafe extern "C" fn(driver_version: *mut i32) -> i32;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct CudaDriverVersion {
-    raw: i32,
-}
-
-impl CudaDriverVersion {
-    pub const fn from_raw(raw: i32) -> Self {
-        Self { raw }
-    }
-
-    pub const fn raw(self) -> i32 {
-        self.raw
-    }
-
-    pub const fn major(self) -> i32 {
-        self.raw / 1000
-    }
-
-    pub const fn minor(self) -> i32 {
-        (self.raw % 1000) / 10
-    }
-
-    pub const fn supports_cuda_13_1(self) -> bool {
-        self.raw >= CUDA_13_1_DRIVER_VERSION
-    }
-}
-
-impl fmt::Display for CudaDriverVersion {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}.{}", self.major(), self.minor())
-    }
-}
-
-pub fn supports_cuda_13_1() -> Result<bool> {
-    Ok(driver_version()?.supports_cuda_13_1())
-}
-
-pub fn driver_version() -> Result<CudaDriverVersion> {
-    let lib_name = if cfg!(target_os = "windows") {
-        "nvcuda.dll"
-    } else {
-        "libcuda.so"
-    };
-
-    unsafe {
-        let library = libloading::Library::new(lib_name)
-            .with_context(|| format!("Failed to load NVIDIA driver library {lib_name}"))?;
-        let cu_init = *library
-            .get::<CuInit>(b"cuInit\0")
-            .context("Failed to load cuInit from NVIDIA driver")?;
-        let cu_driver_get_version = *library
-            .get::<CuDriverGetVersion>(b"cuDriverGetVersion\0")
-            .context("Failed to load cuDriverGetVersion from NVIDIA driver")?;
-
-        let init_status = cu_init(0);
-        ensure_cuda_success(init_status, "cuInit")?;
-
-        let mut raw = 0;
-        let version_status = cu_driver_get_version(&mut raw);
-        ensure_cuda_success(version_status, "cuDriverGetVersion")?;
-
-        Ok(CudaDriverVersion::from_raw(raw))
-    }
-}
-
-fn ensure_cuda_success(code: i32, op: &str) -> Result<()> {
-    if code == CUDA_SUCCESS {
-        return Ok(());
-    }
-
-    anyhow::bail!("{op} failed with CUDA driver error code {code}")
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn parses_major_minor_from_driver_version() {
-        let version = CudaDriverVersion::from_raw(13010);
-        assert_eq!(version.major(), 13);
-        assert_eq!(version.minor(), 1);
-        assert_eq!(version.to_string(), "13.1");
-    }
-
-    #[test]
-    fn checks_cuda_13_1_threshold() {
-        assert!(CudaDriverVersion::from_raw(13010).supports_cuda_13_1());
-        assert!(CudaDriverVersion::from_raw(13020).supports_cuda_13_1());
-        assert!(!CudaDriverVersion::from_raw(13000).supports_cuda_13_1());
-        assert!(!CudaDriverVersion::from_raw(12080).supports_cuda_13_1());
-    }
-}

koharu/tauri.linux.conf.json

@@ -1,10 +1,9 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
-  "identifier": "Koharu",
-  "build": {
-    "features": [
-      "cuda",
-      "cudnn"
-    ]
-  }
-}
+  "identifier": "Koharu",
+  "build": {
+    "features": [
+      "cuda"
+    ]
+  }
+}

koharu/tauri.windows.conf.json

@@ -1,10 +1,9 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
-  "identifier": "Koharu",
-  "build": {
-    "features": [
-      "cuda",
-      "cudnn"
-    ]
-  }
-}
+  "identifier": "Koharu",
+  "build": {
+    "features": [
+      "cuda"
+    ]
+  }
+}

scripts/dev.ts

@@ -62,37 +62,6 @@ async function setupCuda() {
   )
 }
 
-async function setupCudnn() {
-  const cudnnRoot = 'C:/Program Files/NVIDIA/CUDNN'
-  const versions = await readdir(cudnnRoot).catch(() => [])
-
-  sortVersionsDesc(versions)
-
-  for (const version of versions) {
-    if (version.startsWith('v')) {
-      const binPath = path.join(cudnnRoot, version, 'bin')
-
-      if (await pathExists(binPath)) {
-        const versions = await readdir(binPath)
-
-        sortVersionsDesc(versions)
-
-        for (const version of versions) {
-          const fullPath = path.join(binPath, version, 'x64')
-          process.env.PATH = `${fullPath}${path.delimiter}${process.env.PATH}`
-
-          console.log(`Added cuDNN to PATH: ${fullPath}`)
-          return
-        }
-      }
-    }
-
-    throw new Error(
-      'cuDNN not found. Please install cuDNN from https://developer.nvidia.com/rdp/cudnn-download',
-    )
-  }
-}
-
 async function setupCl() {
   const vsRoot = 'C:/Program Files/Microsoft Visual Studio'
   const vsVersions = await readdir(vsRoot).catch(() => [])
@@ -129,9 +98,6 @@ async function dev() {
         await checkNvcc()
       })
 
-    // Setup cuDNN path
-    await setupCudnn()
-
     // Setup cl.exe path
     await setupCl()
   }