build.toml

# One backend per `[kernel.*]` section, all implementing the same two entry points declared in
# torch-ext/torch_binding.h. Adding Metal means adding a section and a `gguf_metal/` directory —
# no change to the bindings' schema or to the Python API.
#
# `vendor/` is a pinned subset of llama.cpp (revision in vendor/UPSTREAM); refresh it with
# `python vendor.py --rev <sha>`.

[general]
name = "gguf-kernels"
version = 1
license = "MIT"
backends = ["cuda"]

[torch]
src = [
    "torch-ext/torch_binding.cpp",
    "torch-ext/torch_binding.h",
]

[kernel.gguf_cuda]
backend = "cuda"
depends = ["torch"]
cuda-capabilities = ["7.5", "8.0", "8.6", "8.9", "9.0", "10.0", "12.0"]
include = ["gguf_cuda", "torch-ext", "vendor/include", "vendor/src", "vendor/src/ggml-cuda"]
src = [
    "gguf_cuda/gguf_cuda.cu",
    "gguf_cuda/ggml_dispatch.cu",
    "gguf_cuda/ggml_stubs.cu",
    "vendor/src/ggml-cuda/convert.cu",
    "vendor/src/ggml-cuda/quantize.cu",
    # headers are listed so the builder stages them; only the files above are compiled
    "vendor/include/ggml-alloc.h",
    "vendor/include/ggml-backend.h",
    "vendor/include/ggml-cuda.h",
    "vendor/include/ggml.h",
    "vendor/include/gguf.h",
    "vendor/src/ggml-common.h",
    "vendor/src/ggml-cuda/common.cuh",
    "vendor/src/ggml-cuda/convert.cuh",
    "vendor/src/ggml-cuda/dequantize.cuh",
    "vendor/src/ggml-cuda/mma.cuh",
    "vendor/src/ggml-cuda/mmq-config-ampere.cuh",
    "vendor/src/ggml-cuda/mmq-config-blackwell.cuh",
    "vendor/src/ggml-cuda/mmq-config-cdna.cuh",
    "vendor/src/ggml-cuda/mmq-config-pascal.cuh",
    "vendor/src/ggml-cuda/mmq-config-rdna2.cuh",
    "vendor/src/ggml-cuda/mmq-config-rdna3-5.cuh",
    "vendor/src/ggml-cuda/mmq-config-rdna3.cuh",
    "vendor/src/ggml-cuda/mmq-config-rdna4.cuh",
    "vendor/src/ggml-cuda/mmq-load-tiles.cuh",
    "vendor/src/ggml-cuda/mmq-vec-dot.cuh",
    "vendor/src/ggml-cuda/mmq.cuh",
    "vendor/src/ggml-cuda/mmvf.cuh",
    "vendor/src/ggml-cuda/mmvq.cuh",
    "vendor/src/ggml-cuda/mmvq-impl.cuh",
    "vendor/src/ggml-cuda/quantize.cuh",
    "vendor/src/ggml-cuda/unary.cuh",
    "vendor/src/ggml-cuda/vecdotq.cuh",
    "vendor/src/ggml-cuda/vendors/cuda.h",
    "vendor/src/ggml-impl.h",
]
# ggml's half/bfloat16 arithmetic needs the operators torch's build disables
cuda-flags = [
    "-DGGML_USE_CUDA",
    "-DNDEBUG",
    "-U__CUDA_NO_HALF_OPERATORS__",
    "-U__CUDA_NO_HALF_CONVERSIONS__",
    "-U__CUDA_NO_HALF2_OPERATORS__",
    "-U__CUDA_NO_BFLOAT16_CONVERSIONS__",
    "-U__CUDA_NO_BFLOAT16_OPERATORS__",
    "-U__CUDA_NO_BFLOAT162_OPERATORS__",
    "--expt-relaxed-constexpr",
    "--expt-extended-lambda",
]
cxx-flags = ["-DGGML_USE_CUDA", "-DNDEBUG"]