Runtime 0.9.2.1: the MLX backend bounds the memory it keeps between requests

MANIFEST.sha256

@@ -2,7 +2,7 @@
 50cbab8a892c5f2993b8c7351a99182507472def3b1374558308605d99b86b32  ./LICENSE-base
 2014f13a6fe1d9178c84cde47bb087a7a74adbe38650f1823a944077081e86d0  ./LICENSING.md
 e4dcac398e39a098e07f625ed239949296a2000d95dcc275cb0d8083ebb51dc9  ./NOTICE
-9862c836bb1e4ce3a083862e649fd5bfc2cefadf5ee85edfe98a3113cfb1d1b8  ./README.md
+8e38c861ae8950a50cb8fc12d334cc5a10136bf7b2204c5661bc7a29e035ed94  ./README.md
 c44489690e63adb3abd8c65ede675fadc5714126e3f866cbd292df824880cb77  ./assets/decisionnode-banner.png
 544b3d40cc9e355b16f8c131426c21245c1b5847a46c40c1b4be2c33a447b4fa  ./box_calibration.json
 5febf5c3cb5843034f2251d959c475a4a629dec65bf3ea494b4fed0a3b8aa3aa  ./calibration.json
@@ -15,13 +15,13 @@ d3d84ca9d2701443b84960ba25b4ee21f5125cdf03007b1dc3633c315447ba6a  ./model.safete
 27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516  ./preprocessor_config.json
 04603015c9b8bc70dd5e7a5c8f0ff14a3ac52aaa359b16b23f662f8f51edd5cd  ./readout.safetensors
 0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0  ./runtime/LICENSE
-4343f7ad5a0283d7ecd8c1630ee1c82fde7e8f5c18696f5444629e9399902415  ./runtime/README.md
+06b52d22d46c230f3d6215f370520038c6ccabac5f2c9c224c766a3e5aa8e43a  ./runtime/README.md
 e0a87595ad939efa5585b61844d3a2a70d29ba7752f68b8139f2195753a6bfde  ./runtime/examples/request.json
-af77c7510284ebcae3fe0fd6ebd951c2b6bcf88d0b0d73b85c8e45b03b1b8776  ./runtime/pyproject.toml
-1551cc741199ad4ec30b8d16b2a3b82432471e4ef36a75f8677bad7c50771338  ./runtime/src/decisionnode_edge/__init__.py
+fdab0f52237084c3793475ffa211fafdf5c3fb5973674131587dae0788613780  ./runtime/pyproject.toml
+0c067d73ac9a01af3049c980c08f35acd257e4044372fd2066a74db466499a39  ./runtime/src/decisionnode_edge/__init__.py
 245003271a41ecab8fe982194acf84f9c4fc0b686762583931a9444ad2aa6354  ./runtime/src/decisionnode_edge/__main__.py
 736741c822983f1d7d8ed26ea01227f2f2e902c98dc7d0c21d8fa5abcf219150  ./runtime/src/decisionnode_edge/answers.py
-c34f84c5da93d012d329762b6e099bc5aa7b7d998f790b88471f75399cd3eee8  ./runtime/src/decisionnode_edge/backends.py
+1488cd43642f6bd49b810683b7e72688e98dc62d2e955c202331e027a62b1889  ./runtime/src/decisionnode_edge/backends.py
 4ace977690bb7ae16010a298c478f1a5056caaf724c0d696b5ddac6699225f6b  ./runtime/src/decisionnode_edge/build.py
 3e2d2198d9fd4fac076773e655a0c73c810a932be07ce99460edee64fb7004bc  ./runtime/src/decisionnode_edge/model.py
 3be9ac7eb2807241d0b9b49cb5797049395f7fbca6ddee9a645274edd1c3e4a5  ./runtime/src/decisionnode_edge/places.py

README.md

@@ -117,7 +117,7 @@ Pass images as base64 `data` with a `media_type` (not a local `path`), so the sa
 | | |
 |---|---|
 | Developer | Bynn Intelligence Inc. |
-| Version | 0.9.2 (October 2026): the 0.9.1 weights with runtime 0.9.2, which adds point and box questions |
+| Version | 0.9.2 (October 2026): the 0.9.1 weights with runtime 0.9.2, which adds point and box questions. Runtime 0.9.2.1 bounds the memory MLX keeps between requests on Apple silicon |
 | Parameters | 2.21 billion (1.88 billion in the language model, 0.33 billion in the vision encoder) |
 | Base model | [Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base) |
 | Input | Text, and up to 16 images per request (JPEG, PNG or WebP) |

runtime/README.md

@@ -204,6 +204,7 @@ Options can be flat (`{"threads": 16}`) or under a backend's name (`{"openvino":
 | `llamacpp` | `server`, `gguf`: the program and the model file | `llama-server` on `PATH`, the build's file |
 | `llamacpp` | `gpu_layers`, `threads` | 999 (all layers on the GPU when there is one), llama.cpp's choice |
 | `llamacpp` | `context`: the longest prompt llama.cpp takes | 16,384 |
+| `mlx` | `cache_limit_gb`: how much freed GPU memory MLX keeps for reuse | 2 |
 
 vLLM runs its engine inside your process and uses the first visible GPU (choose it with `CUDA_VISIBLE_DEVICES`). Its first load takes about half a minute while it compiles. MLX uses the Apple GPU.
 

runtime/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "decisionnode-edge"
-version = "0.9.2"
+version = "0.9.2.1"
 description = "Run DecisionNode Edge locally: typed, calibrated decisions in the shape of the DecisionNode API."
 readme = "README.md"
 requires-python = ">=3.10"

runtime/src/decisionnode_edge/__init__.py

@@ -10,4 +10,4 @@ from .model import EdgeModel
 from .request import RequestError
 
 __all__ = ["EdgeModel", "RequestError"]
-__version__ = "0.9.2"
+__version__ = "0.9.2.1"

runtime/src/decisionnode_edge/backends.py

@@ -165,7 +165,7 @@ def load(name: str, path: str, config: dict, artifacts: str | None, options: dic
     if name == "vllm":
         return VLLM(path, max_tokens, options, device)
     if name == "mlx":
-        return MLX(path)
+        return MLX(path, options)
     build = read_build(artifacts)
     if build is not None and build.get("model_fingerprint") != fingerprint(path):
         raise ValueError(f"{artifacts} was built from another model than {path}")
@@ -304,14 +304,18 @@ class MLX:
     touches readout.safetensors); the text model's output is the hidden state after the final norm."""
 
     name = "mlx"
+    cache_limit_gb = 2.0
 
-    def __init__(self, path: str):
+    def __init__(self, path: str, options: dict | None = None):
         from pathlib import Path
 
         import mlx.core as mx
         from mlx_lm.utils import load_model
 
         self.mx = mx
+        # MLX keeps freed buffers for reuse, by default up to its memory limit, so a server reading long prompts grew its
+        # cache to most of the machine's memory. A bound keeps the reuse that matters for speed.
+        mx.set_cache_limit(int(float((options or {}).get("cache_limit_gb", self.cache_limit_gb)) * 2**30))
         model, _ = load_model(Path(path))
         self.text = model.language_model.model