feat: switch local GLiNER2 extraction to ONNX runtime backend

Replace the PyTorch (fastino/gliner2-base-v1) backend with the monolithic
lion-ai/gliner2-base-v1-onnx export running on onnxruntime. Identical output
and accuracy on single-invoice docs, ~1.8-2.3x faster, no torch on the hot
path. New GLiNER2ONNXEngine (gliner_onnx.py) reproduces gliner2 schema
semantics (entities, str/list, choice/classification fields) with one ONNX
forward pass per schema group. PyTorch backend preserved as commented
DEPRECATED code for rollback/fallback. Config, env, Dockerfile and
requirements updated; tokenizers pinned for direct tokenizer loading.

.env.example

@@ -48,6 +48,8 @@ DATA_DIR=./data
 
 # --- Local on-device extraction (/json/feature-extract) ---
 # All keys are optional; unset values fall back to the defaults shown.
+# GLINER_MODEL=lion-ai/gliner2-base-v1-onnx
+# DEPRECATED: PyTorch model (torch backend) -- restore to roll back:
 # GLINER_MODEL=fastino/gliner2-base-v1
 # GLINER_ENABLED=true
 # GLINER_DEVICE=cpu

Dockerfile

@@ -95,10 +95,13 @@ RUN chmod +x /app/whatsapp-service/server
 
 RUN mkdir -p /app/models && python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='ibm-granite/granite-embedding-small-english-r2', local_dir='/app/models/bge-384')" && chown -R appuser:appuser /app/models
 
+# DEPRECATED: PyTorch (torch) backend -- restore to roll back:
+# RUN mkdir -p /app/models && python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='fastino/gliner2-base-v1', local_dir='/app/models/gliner2-base-v1')" && chown -R appuser:appuser /app/models
+
 # Pre-download the local on-device extraction model (used by /json/feature-extract)
 # so first boot does not hit Hugging Face. The GLINER_MODEL env below points the
 # service at this local copy.
-RUN mkdir -p /app/models && python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='fastino/gliner2-base-v1', local_dir='/app/models/gliner2-base-v1')" && chown -R appuser:appuser /app/models
+RUN mkdir -p /app/models && python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='lion-ai/gliner2-base-v1-onnx', local_dir='/app/models/gliner2-base-v1-onnx')" && chown -R appuser:appuser /app/models
 
 RUN mkdir -p /app/data /app/logs && \
     chown -R appuser:appuser /app/data /app/logs
@@ -111,7 +114,9 @@ ENV PYTHONPATH=/app
 ENV PYTHONUNBUFFERED=1
 
 # Local copy of the on-device extraction model baked in at build time.
-ENV GLINER_MODEL=/app/models/gliner2-base-v1
+ENV GLINER_MODEL=/app/models/gliner2-base-v1-onnx
+# DEPRECATED: PyTorch (torch) backend -- restore to roll back:
+# ENV GLINER_MODEL=/app/models/gliner2-base-v1
 
 # Path to the embedded WhatsApp service binary (started by start.sh as a
 # sibling process). Set to an empty value to disable the WhatsApp gateway.

app/config.py

@@ -223,7 +223,9 @@ class Settings(BaseSettings):
     # on-device (no external AI/LLM API). The model is loaded once and cached at
     # startup; disable to skip loading and return 503 from the route. Every key
     # is optional -- if not set, the default value below is used.
-    gliner_model: str = Field(default="fastino/gliner2-base-v1", alias="GLINER_MODEL")
+    # DEPRECATED: PyTorch model (torch backend) -- restore to roll back:
+    # gliner_model: str = Field(default="fastino/gliner2-base-v1", alias="GLINER_MODEL")
+    gliner_model: str = Field(default="lion-ai/gliner2-base-v1-onnx", alias="GLINER_MODEL")
     gliner_enabled: bool = Field(default=True, alias="GLINER_ENABLED")
     gliner_device: str = Field(default="cpu", alias="GLINER_DEVICE")
     gliner_max_concurrent: int = Field(default=2, alias="GLINER_MAX_CONCURRENT")

app/services/gliner_onnx.py

@@ -0,0 +1,481 @@
+"""ONNX Runtime-backed GLiNER2 extraction engine (no PyTorch at inference).
+
+Runs the monolithic ``lion-ai/gliner2-base-v1-onnx`` export -- the GLiNER2
+encoder + span head fused into a single ``model.onnx`` -- on ONNX Runtime.
+The companion ``tokenizer.json`` is loaded through the ``tokenizers`` library,
+so inference does not depend on torch/transformers.
+
+This engine reproduces the GLiNER2 schema semantics on top of the ONNX span
+head for the ``/json/feature-extract`` route:
+
+* zero-shot entity extraction (mode='entities')
+* structured-JSON field extraction with ``str``/``list`` dtypes and choice
+  ("classification") fields (mode='json')
+
+Each schema group (the entity list, or each structure parent) runs as its own
+ONNX forward pass, because the graph expects a single ``[P]`` schema per call.
+
+Known behavioural differences vs. the PyTorch model (``fastino/gliner2-base-v1``):
+the export fixes the count dimension to 1, so a structure with multiple
+repeating objects yields only the single best object. For documents that
+contain exactly one object (the common case for this route) the output matches
+the PyTorch model.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import onnxruntime as ort
+from huggingface_hub import snapshot_download
+from tokenizers import Tokenizer
+
+from gliner2.inference.schema import Schema
+
+from app.core.logger import get_logger
+
+logger = get_logger(__name__)
+
+MAX_WIDTH = 8
+SEP_TEXT = "[SEP_TEXT]"
+DESC_TOKEN = "[DESCRIPTION]"
+P_TOKEN = "[P]"
+C_TOKEN = "[C]"
+E_TOKEN = "[E]"
+R_TOKEN = "[R]"
+L_TOKEN = "[L]"
+SPECIAL_TOKENS = frozenset((P_TOKEN, C_TOKEN, E_TOKEN, R_TOKEN, L_TOKEN))
+
+# Identical to gliner2.processor.WhitespaceTokenSplitter.
+_WORD_RE = re.compile(
+    r"""(?:https?://[^\s]+|www\.[^\s]+)
+    |[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}
+    |@[a-z0-9_]+
+    |\w+(?:[-_]\w+)*
+    |\S""",
+    re.VERBOSE | re.IGNORECASE,
+)
+
+
+def _parse_field_spec(spec: Any) -> Tuple[str, str, Optional[List[str]], Optional[str]]:
+    """Parse a structure field spec into ``(name, dtype, choices, description)``.
+
+    Mirrors ``gliner2``'s ``GLiNER2._parse_field_spec``. Field specs look like
+    ``"name::dtype::choices::description"`` where all parts after ``name`` are
+    optional; dict specs with ``name``/``dtype``/``choices``/``description``
+    keys are accepted too.
+    """
+    if isinstance(spec, dict):
+        return (
+            spec.get("name", ""),
+            spec.get("dtype", "list"),
+            spec.get("choices"),
+            spec.get("description"),
+        )
+
+    parts = spec.split("::")
+    name = parts[0]
+    dtype: str = "list"
+    choices: Optional[List[str]] = None
+    desc: Optional[str] = None
+    dtype_explicitly_set = False
+
+    for part in parts[1:]:
+        if part in ("str", "list"):
+            dtype = part
+            dtype_explicitly_set = True
+        elif part.startswith("[") and part.endswith("]"):
+            choices = [c.strip() for c in part[1:-1].split("|")]
+            if not dtype_explicitly_set:
+                dtype = "str"
+        else:
+            desc = part
+
+    return name, dtype, choices, desc
+
+
+class GLiNER2ONNXEngine:
+    """Cached wrapper around the monolithic GLiNER2 ONNX model.
+
+    The model and tokenizer are loaded once and reused for the life of the
+    process. ``onnxruntime``'s ``InferenceSession.run`` is thread-safe, so the
+    engine can serve concurrent requests (the caller bounds concurrency).
+    """
+
+    def __init__(self, model_id: str, device: str = "cpu") -> None:
+        self._model_id = model_id
+        self._device = device
+        self._session: Optional[ort.InferenceSession] = None
+        self._tokenizer: Optional[Tokenizer] = None
+        self._load_error: Optional[str] = None
+
+    # ------------------------------------------------------------------ #
+    # Lifecycle
+    # ------------------------------------------------------------------ #
+
+    def load(self) -> None:
+        """Load and cache the ONNX session + tokenizer. Idempotent."""
+        if self._session is not None:
+            return
+        try:
+            model_dir = self._resolve_model_dir()
+            tokenizer_path = os.path.join(model_dir, "tokenizer.json")
+            model_path = os.path.join(model_dir, "model.onnx")
+            if not os.path.exists(tokenizer_path):
+                raise FileNotFoundError(f"tokenizer.json not found in {model_dir}")
+            if not os.path.exists(model_path):
+                raise FileNotFoundError(f"model.onnx not found in {model_dir}")
+
+            providers = ["CPUExecutionProvider"]
+            if self._device == "cuda":
+                providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
+            self._tokenizer = Tokenizer.from_file(tokenizer_path)
+            self._session = ort.InferenceSession(model_path, providers=providers)
+            self._load_error = None
+        except Exception as exc:  # noqa: BLE001
+            self._load_error = str(exc)
+            logger.exception("GLiNER2 ONNX model load failed")
+            raise
+
+    def unload(self) -> None:
+        """Release the cached session + tokenizer."""
+        self._session = None
+        self._tokenizer = None
+        self._load_error = None
+
+    def is_loaded(self) -> bool:
+        return self._session is not None
+
+    def load_error(self) -> Optional[str]:
+        return self._load_error
+
+    @property
+    def model_id(self) -> str:
+        return self._model_id
+
+    @property
+    def device(self) -> str:
+        return self._device
+
+    def _resolve_model_dir(self) -> str:
+        """Return a local directory containing ``model.onnx`` + ``tokenizer.json``."""
+        if os.path.isdir(self._model_id):
+            return self._model_id
+        return snapshot_download(self._model_id)
+
+    def _ensure_loaded(self) -> None:
+        if self._session is None:
+            self.load()
+
+    # ------------------------------------------------------------------ #
+    # Inference
+    # ------------------------------------------------------------------ #
+
+    def extract_entities(self, text: str, labels: List[str], threshold: float = 0.5) -> Dict[str, Any]:
+        """Zero-shot entity extraction. Returns ``{"entities": {label: [texts]}}``."""
+        self._ensure_loaded()
+        result: Dict[str, List[str]] = {label: [] for label in labels}
+
+        text = self._normalize_text(text)
+        words = self._split_words(text)
+        if not words:
+            return {"entities": result}
+
+        word_strings = [w for w, _, _ in words]
+        start_map = [w[1] for w in words]
+        end_map = [w[2] for w in words]
+
+        schema_tokens = self._build_schema_tokens("entities", list(labels), E_TOKEN, {})
+        feeds = self._build_feeds(schema_tokens, word_strings)
+        grid = self._session.run(None, feeds)[0][0]
+
+        text_len = len(word_strings)
+        for i, label in enumerate(labels):
+            spans = self._find_spans(grid[i], threshold, text_len, text, start_map, end_map)
+            result[label] = self._format_spans(spans)
+
+        return {"entities": result}
+
+    def extract_json(self, text: str, structure: Dict[str, Any], threshold: float = 0.5) -> Dict[str, Any]:
+        """Structured JSON extraction. Returns ``{parent: [object]}``."""
+        self._ensure_loaded()
+        if not structure:
+            return {}
+
+        schema_obj = self._build_structure_schema(structure)
+        schema = schema_obj.build()
+        prefix = self._build_classification_prefix(schema)
+
+        text = self._normalize_text(text)
+        words = self._split_words(text)
+        if not words:
+            return {parent: [] for parent in schema.get("json_structures", []) for parent in parent}
+
+        word_strings = [w for w, _, _ in words]
+        start_map = [w[1] for w in words]
+        end_map = [w[2] for w in words]
+        text_tokens = prefix + word_strings
+        num_prefix = len(prefix)
+        text_len = len(word_strings)
+
+        json_descs = schema.get("json_descriptions", {})
+        result: Dict[str, Any] = {}
+
+        for struct in schema.get("json_structures", []):
+            for parent, fields in struct.items():
+                field_names = list(fields.keys())
+                schema_tokens = self._build_schema_tokens(
+                    parent, field_names, C_TOKEN, json_descs.get(parent, {})
+                )
+                feeds = self._build_feeds(schema_tokens, text_tokens)
+                grid = self._session.run(None, feeds)[0][0]
+
+                obj: Dict[str, Any] = {}
+                for fname in schema_obj._field_orders.get(parent, field_names):
+                    fidx = field_names.index(fname)
+                    full = grid[fidx]
+                    meta = schema_obj._field_metadata.get(f"{parent}.{fname}", {})
+                    dtype = meta.get("dtype", "list")
+                    field_threshold = (
+                        meta.get("threshold") if meta.get("threshold") is not None else threshold
+                    )
+
+                    fval = fields[fname]
+                    if isinstance(fval, dict) and "choices" in fval:
+                        obj[fname] = self._decode_choice_field(
+                            full, prefix, fval["choices"], num_prefix, field_threshold, dtype
+                        )
+                    else:
+                        scores = full[num_prefix:num_prefix + text_len]
+                        spans = self._find_spans(scores, field_threshold, text_len, text, start_map, end_map)
+                        if dtype == "list":
+                            obj[fname] = self._dedup_texts(self._format_spans(spans))
+                        else:
+                            obj[fname] = spans[0][0] if spans else None
+
+                instances = [obj] if any(v is not None and v != [] for v in obj.values()) else []
+                result[parent] = instances
+
+        return result
+
+    # ------------------------------------------------------------------ #
+    # Schema construction (mirrors gliner2.processor / gliner2 engine)
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _normalize_text(text: str) -> str:
+        """Append a period so the model sees the same input gliner2 would."""
+        if text and not text.endswith((".", "!", "?")):
+            return text + "."
+        return text if text else "."
+
+    @staticmethod
+    def _split_words(text: str) -> List[Tuple[str, int, int]]:
+        """Split text into lowercased ``(word, char_start, char_end)`` tuples."""
+        lowered = text.lower()
+        return [(m.group(), m.start(), m.end()) for m in _WORD_RE.finditer(lowered)]
+
+    def _build_structure_schema(self, structure: Dict[str, Any]) -> Schema:
+        """Build a gliner2 ``Schema`` object from a route ``structure`` dict."""
+        schema = Schema()
+        for parent, specs in structure.items():
+            builder = schema.structure(parent)
+            for spec in specs:
+                name, dtype, choices, desc = _parse_field_spec(spec)
+                builder.field(name, dtype=dtype, choices=choices, description=desc)
+            builder._auto_finish()
+        return schema
+
+    @staticmethod
+    def _build_classification_prefix(schema: Dict[str, Any]) -> List[str]:
+        """Build the classification (choice) prefix tokens prepended to the text."""
+        prefix_tokens: List[str] = []
+        for struct in schema.get("json_structures", []):
+            for parent, fields in struct.items():
+                cls_fields = [
+                    (fname, fval) for fname, fval in fields.items()
+                    if isinstance(fval, dict) and "value" in fval and "choices" in fval
+                ]
+                inner: List[str] = []
+                for fname, fval in cls_fields:
+                    choices = fval["choices"]
+                    choice_tokens: List[str] = []
+                    for i, c in enumerate(choices):
+                        if i > 0:
+                            choice_tokens.append("|")
+                        choice_tokens.append(c)
+                    inner.extend([fname, "("] + choice_tokens + [")", ","])
+                if inner:
+                    inner = inner[:-1]
+                    prefix_tokens.extend(["(", f"{parent}:", *inner, ")"])
+        return prefix_tokens
+
+    @staticmethod
+    def _build_schema_tokens(
+        parent: str,
+        fields: List[str],
+        child_token: str,
+        descriptions: Dict[str, str],
+    ) -> List[str]:
+        """Build a schema token sequence like gliner2's ``_transform_schema``."""
+        prompt_str = parent
+        descs = [(lbl, d) for lbl, d in descriptions.items() if lbl in fields]
+        for lbl, d in descs:
+            prompt_str += f" {DESC_TOKEN} {lbl}: {d}"
+        tokens = ["(", P_TOKEN, prompt_str, "("]
+        for field in fields:
+            tokens.extend([child_token, field])
+        tokens.extend([")", ")"])
+        return tokens
+
+    # ------------------------------------------------------------------ #
+    # ONNX inputs
+    # ------------------------------------------------------------------ #
+
+    def _build_feeds(self, schema_tokens: List[str], text_tokens: List[str]) -> Dict[str, np.ndarray]:
+        """Build the five ONNX input tensors for one schema + text pair."""
+        combined = schema_tokens + [SEP_TEXT] + text_tokens
+        encoding = self._tokenizer.encode(combined, is_pretokenized=True, add_special_tokens=False)
+        token_ids = encoding.ids
+        word_ids = encoding.word_ids
+
+        input_ids = np.array([token_ids], dtype=np.int64)
+        attention_mask = np.ones((1, len(token_ids)), dtype=np.int64)
+
+        # Number of words before the text segment: schema words + [SEP_TEXT].
+        num_schema_words = len(schema_tokens) + 1
+
+        word_first: Dict[int, int] = {}
+        for tok_pos, wid in enumerate(word_ids):
+            if wid is not None and wid not in word_first:
+                word_first[wid] = tok_pos
+
+        text_positions = np.array(
+            [word_first[num_schema_words + i] for i in range(len(text_tokens))], dtype=np.int64
+        )
+        schema_positions = np.array(
+            [word_first[i] for i, tok in enumerate(schema_tokens) if tok in SPECIAL_TOKENS],
+            dtype=np.int64,
+        )
+
+        num_words = len(text_positions)
+        spans = []
+        for start in range(num_words):
+            for width in range(1, MAX_WIDTH + 1):
+                end = start + width
+                if end <= num_words:
+                    spans.append((start, end - 1))
+                else:
+                    spans.append((0, 0))
+        span_idx = np.array(spans, dtype=np.int64).reshape(1, -1, 2)
+
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "text_positions": text_positions,
+            "schema_positions": schema_positions,
+            "span_idx": span_idx,
+        }
+
+    # ------------------------------------------------------------------ #
+    # Decoding (mirrors gliner2's span/choice decoding on the ONNX scores)
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _find_spans(
+        scores: np.ndarray,
+        threshold: float,
+        text_len: int,
+        text: str,
+        start_map: List[int],
+        end_map: List[int],
+    ) -> List[Tuple[str, float, int, int]]:
+        """Find spans above threshold. Returns ``(text, confidence, start, end)``."""
+        valid = np.argwhere(scores >= threshold)
+        spans: List[Tuple[str, float, int, int]] = []
+        for start, width in valid:
+            end = start + width + 1
+            if 0 <= start < text_len and end <= text_len:
+                try:
+                    char_start = start_map[start]
+                    char_end = end_map[end - 1]
+                    text_span = text[char_start:char_end].strip()
+                except (IndexError, KeyError):
+                    continue
+                if text_span:
+                    spans.append((text_span, float(scores[start, width]), char_start, char_end))
+        return spans
+
+    @staticmethod
+    def _format_spans(spans: List[Tuple[str, float, int, int]]) -> List[str]:
+        """Score-sort + overlap removal, matching gliner2's ``_format_spans``."""
+        if not spans:
+            return []
+        sorted_spans = sorted(spans, key=lambda x: x[1], reverse=True)
+        selected: List[Tuple[str, float, int, int]] = []
+        for text, conf, start, end in sorted_spans:
+            overlap = any(not (end <= s[2] or start >= s[3]) for s in selected)
+            if not overlap:
+                selected.append((text, conf, start, end))
+        return [s[0] for s in selected]
+
+    @staticmethod
+    def _dedup_texts(values: List[str]) -> List[str]:
+        """Dedup a list of extracted texts (case-insensitive), matching gliner2."""
+        unique: List[str] = []
+        seen = set()
+        for v in values:
+            if v and v.lower() not in seen:
+                seen.add(v.lower())
+                unique.append(v)
+        return unique
+
+    @staticmethod
+    def _find_choice_idx(choice: str, tokens: List[str]) -> int:
+        """Index of a choice token inside the classification prefix tokens."""
+        choice_lower = choice.lower()
+        for i, tok in enumerate(tokens):
+            if tok.lower() == choice_lower or choice_lower in tok.lower():
+                return i
+        return -1
+
+    def _decode_choice_field(
+        self,
+        full: np.ndarray,
+        prefix: List[str],
+        choices: List[str],
+        num_prefix: int,
+        threshold: float,
+        dtype: str,
+    ) -> Any:
+        """Score the classification-prefix choice tokens (gliner2 semantics)."""
+        prefix_scores = full[:num_prefix, 0]
+
+        if dtype == "list":
+            selected: List[str] = []
+            seen = set()
+            for choice in choices:
+                if choice in seen:
+                    continue
+                idx = self._find_choice_idx(choice, prefix)
+                if 0 <= idx < len(prefix_scores):
+                    score = float(prefix_scores[idx])
+                    if score >= threshold:
+                        selected.append(choice)
+                        seen.add(choice)
+            return selected
+
+        best: Optional[str] = None
+        best_score = -1.0
+        for choice in choices:
+            idx = self._find_choice_idx(choice, prefix)
+            if 0 <= idx < len(prefix_scores):
+                score = float(prefix_scores[idx])
+                if score > best_score:
+                    best_score = score
+                    best = choice
+        return best if best and best_score >= threshold else None

app/services/gliner_service.py

@@ -9,29 +9,32 @@ loaded on first use if startup warm-up failed). Inference is CPU-bound and is
 dispatched on the shared thread pool by the route layer; a bounded semaphore
 serializes concurrent forwards on the shared model instance so a burst of
 requests cannot oversubscribe the CPU or the model's memory buffers.
+
+Inference backend: the ONNX export ``lion-ai/gliner2-base-v1-onnx`` running on
+ONNX Runtime via ``GLiNER2ONNXEngine`` (see ``gliner_onnx.py``). This removes
+the PyTorch dependency from the extraction hot path.
 """
 
 from __future__ import annotations
 
-import contextlib
-import io
 import threading
 import time
 from typing import Any, Dict, List, Optional
 
 from app.config import get_settings
 from app.core.logger import get_logger
+from app.services.gliner_onnx import GLiNER2ONNXEngine
 
 logger = get_logger(__name__)
 
-DEFAULT_MODEL_ID = "fastino/gliner2-base-v1"
+DEFAULT_MODEL_ID = "lion-ai/gliner2-base-v1-onnx"
 DEFAULT_DEVICE = "cpu"
 DEFAULT_MAX_CONCURRENT = 2
 DEFAULT_MAX_CONTENT_LENGTH = 100_000
 
 
 class GLiNERService:
-    """Cached, concurrency-bounded wrapper around a GLiNER2 model instance."""
+    """Cached, concurrency-bounded wrapper around the GLiNER2 ONNX engine."""
 
     def __init__(
         self,
@@ -44,7 +47,7 @@ class GLiNERService:
         self._device = device
         self._max_content_length = max_content_length
         self._max_concurrent = max(1, max_concurrent)
-        self._model: Any = None
+        self._engine: Optional[GLiNER2ONNXEngine] = None
         self._lock = threading.Lock()
         self._semaphore = threading.BoundedSemaphore(self._max_concurrent)
         self._load_error: Optional[str] = None
@@ -54,7 +57,7 @@ class GLiNERService:
     # ------------------------------------------------------------------ #
 
     def load_model(self) -> None:
-        """Load and cache the GLiNER2 model. Idempotent and thread-safe."""
+        """Load and cache the GLiNER2 ONNX model. Idempotent and thread-safe."""
         if self.is_loaded():
             return
         with self._lock:
@@ -62,33 +65,29 @@ class GLiNERService:
                 return
             t0 = time.perf_counter()
             try:
-                from gliner2 import GLiNER2
-
-                # GLiNER2 prints an emoji config banner to stdout on load, which
-                # crashes consoles with a non-UTF-8 encoding (e.g. Windows cp1252).
-                # Swallow it so loading works everywhere.
-                with contextlib.redirect_stdout(io.StringIO()):
-                    self._model = GLiNER2.from_pretrained(self._model_id, map_location=self._device)
+                engine = GLiNER2ONNXEngine(model_id=self._model_id, device=self._device)
+                engine.load()
+                self._engine = engine
                 self._load_error = None
                 logger.info(
-                    "GLiNER2 model loaded and cached (%s, device=%s) in %.2fs",
+                    "GLiNER2 ONNX model loaded and cached (%s, device=%s) in %.2fs",
                     self._model_id,
                     self._device,
                     time.perf_counter() - t0,
                 )
             except Exception as exc:  # noqa: BLE001
                 self._load_error = str(exc)
-                logger.exception("GLiNER2 model load failed")
+                logger.exception("GLiNER2 ONNX model load failed")
                 raise
 
     def unload(self) -> None:
-        """Release the cached model (frees ~1.3 GB). Used at shutdown."""
+        """Release the cached model. Used at shutdown."""
         with self._lock:
-            self._model = None
+            self._engine = None
             self._load_error = None
 
     def is_loaded(self) -> bool:
-        return self._model is not None
+        return self._engine is not None
 
     def load_error(self) -> Optional[str]:
         return self._load_error
@@ -110,16 +109,16 @@ class GLiNERService:
     # ------------------------------------------------------------------ #
 
     def extract_json(self, text: str, structure: Dict[str, Any], threshold: float = 0.5) -> Dict[str, Any]:
-        """Structured JSON extraction with the cached local model."""
+        """Structured JSON extraction with the cached local ONNX model."""
         self._ensure_loaded()
         with self._semaphore:
-            return self._model.extract_json(text, structure, threshold=threshold)
+            return self._engine.extract_json(text, structure, threshold=threshold)
 
     def extract_entities(self, text: str, labels: List[str], threshold: float = 0.5) -> Dict[str, Any]:
-        """Zero-shot entity extraction with the cached local model."""
+        """Zero-shot entity extraction with the cached local ONNX model."""
         self._ensure_loaded()
         with self._semaphore:
-            return self._model.extract_entities(text, labels, threshold=threshold)
+            return self._engine.extract_entities(text, labels, threshold=threshold)
 
     def _ensure_loaded(self) -> None:
         if not self.is_loaded():
@@ -134,3 +133,144 @@ gliner_service = GLiNERService(
     max_concurrent=_settings.gliner_max_concurrent,
     max_content_length=_settings.gliner_max_content_length,
 )
+
+
+# =============================================================================
+# DEPRECATED -- PyTorch (``gliner2`` / torch) backend.
+# -----------------------------------------------------------------------------
+# Kept for reference and easy rollback / later fallback implementation. The old
+# implementation loaded ``fastino/gliner2-base-v1`` via the ``gliner2`` PyTorch
+# library, which pulled torch into the extraction hot path and required
+# redirecting stdout to swallow the emoji config banner.
+#
+# To restore the PyTorch backend:
+#   1. Set GLINER_MODEL back to a PyTorch repo (e.g. fastino/gliner2-base-v1)
+#      in config/.env/.env.example and the Dockerfile.
+#   2. Un-comment the whole block below and delete/rename the ONNX
+#      ``GLiNERService`` above (or point the module-level ``gliner_service``
+#      instantiation at the deprecated class).
+# =============================================================================
+# from __future__ import annotations
+#
+# import contextlib
+# import io
+# import threading
+# import time
+# from typing import Any, Dict, List, Optional
+#
+# from app.config import get_settings
+# from app.core.logger import get_logger
+#
+# logger = get_logger(__name__)
+#
+# DEFAULT_MODEL_ID = "fastino/gliner2-base-v1"
+# DEFAULT_DEVICE = "cpu"
+# DEFAULT_MAX_CONCURRENT = 2
+# DEFAULT_MAX_CONTENT_LENGTH = 100_000
+#
+#
+# class GLiNERService:
+#     """Cached, concurrency-bounded wrapper around a GLiNER2 model instance."""
+#
+#     def __init__(
+#         self,
+#         model_id: str = DEFAULT_MODEL_ID,
+#         device: str = DEFAULT_DEVICE,
+#         max_concurrent: int = DEFAULT_MAX_CONCURRENT,
+#         max_content_length: int = DEFAULT_MAX_CONTENT_LENGTH,
+#     ) -> None:
+#         self._model_id = model_id
+#         self._device = device
+#         self._max_content_length = max_content_length
+#         self._max_concurrent = max(1, max_concurrent)
+#         self._model: Any = None
+#         self._lock = threading.Lock()
+#         self._semaphore = threading.BoundedSemaphore(self._max_concurrent)
+#         self._load_error: Optional[str] = None
+#
+#     # ------------------------------------------------------------------ #
+#     # Lifecycle
+#     # ------------------------------------------------------------------ #
+#
+#     def load_model(self) -> None:
+#         """Load and cache the GLiNER2 model. Idempotent and thread-safe."""
+#         if self.is_loaded():
+#             return
+#         with self._lock:
+#             if self.is_loaded():
+#                 return
+#             t0 = time.perf_counter()
+#             try:
+#                 from gliner2 import GLiNER2
+#
+#                 # GLiNER2 prints an emoji config banner to stdout on load, which
+#                 # crashes consoles with a non-UTF-8 encoding (e.g. Windows cp1252).
+#                 # Swallow it so loading works everywhere.
+#                 with contextlib.redirect_stdout(io.StringIO()):
+#                     self._model = GLiNER2.from_pretrained(self._model_id, map_location=self._device)
+#                 self._load_error = None
+#                 logger.info(
+#                     "GLiNER2 model loaded and cached (%s, device=%s) in %.2fs",
+#                     self._model_id,
+#                     self._device,
+#                     time.perf_counter() - t0,
+#                 )
+#             except Exception as exc:  # noqa: BLE001
+#                 self._load_error = str(exc)
+#                 logger.exception("GLiNER2 model load failed")
+#                 raise
+#
+#     def unload(self) -> None:
+#         """Release the cached model (frees ~1.3 GB). Used at shutdown."""
+#         with self._lock:
+#             self._model = None
+#             self._load_error = None
+#
+#     def is_loaded(self) -> bool:
+#         return self._model is not None
+#
+#     def load_error(self) -> Optional[str]:
+#         return self._load_error
+#
+#     @property
+#     def model_id(self) -> str:
+#         return self._model_id
+#
+#     @property
+#     def device(self) -> str:
+#         return self._device
+#
+#     @property
+#     def max_content_length(self) -> int:
+#         return self._max_content_length
+#
+#     # ------------------------------------------------------------------ #
+#     # Inference (called on the shared thread pool)
+#     # ------------------------------------------------------------------ #
+#
+#     def extract_json(self, text: str, structure: Dict[str, Any], threshold: float = 0.5) -> Dict[str, Any]:
+#         """Structured JSON extraction with the cached local model."""
+#         self._ensure_loaded()
+#         with self._semaphore:
+#             return self._model.extract_json(text, structure, threshold=threshold)
+#
+#     def extract_entities(self, text: str, labels: List[str], threshold: float = 0.5) -> Dict[str, Any]:
+#         """Zero-shot entity extraction with the cached local model."""
+#         self._ensure_loaded()
+#         with self._semaphore:
+#             return self._model.extract_entities(text, labels, threshold=threshold)
+#
+#     def _ensure_loaded(self) -> None:
+#         if not self.is_loaded():
+#             self.load_model()
+#
+#
+# _settings = get_settings()
+#
+# gliner_service = GLiNERService(
+#     model_id=_settings.gliner_model,
+#     device=_settings.gliner_device,
+#     max_concurrent=_settings.gliner_max_concurrent,
+#     max_content_length=_settings.gliner_max_content_length,
+# )
+# =============================================================================

requirements.txt

@@ -45,7 +45,12 @@ zvec==0.4.0
 # Semantic routing (aurelio-labs) used by the /semantic-router/route endpoint
 semantic-router==0.1.16
 # Local on-device model powering the /json/feature-extract route (no external AI).
+# gliner2==1.3.2 stays for its torch-free Schema builder; the extraction hot
+# path now runs the ONNX export (lion-ai/gliner2-base-v1-onnx) via onnxruntime.
+# DEPRECATED: the PyTorch backend used gliner2's GLiNER2.from_pretrained(torch).
 gliner2==1.3.2
+# Direct tokenizer loading for the ONNX GLiNER2 backend (gliner_onnx.py).
+tokenizers==0.21.1
 sentence-transformers==3.4.1
 transformers==4.50.2
 torch==2.5.1