code/infer_fast.py

import argparse
import hashlib
import json
import sqlite3
import sys
import time
from collections import Counter
from functools import lru_cache
from pathlib import Path

from normalization import NORMALIZATION_VERSION, normalize_input
from romaji_kana import (
    GENERIC_FALLBACK_VERSION,
    generic_romaji_fallback,
    load_general_lexicon,
    prepare_generic_lexicon,
)
from general_phrase import (
    GENERAL_PHRASE_VERSION,
    build_general_phrase_index,
    canonicalize_romaji_variants,
    general_phrase_rescue,
)

try:
    from rapidfuzz.distance import Levenshtein as _RapidLevenshtein
except Exception:
    _RapidLevenshtein = None

BOS_IN = "\uEE00"
BOS_OUT = "\uEE01"
DEFAULT_LEXICON_CANDIDATES = (
    "artifacts/lexicon/romaji2ja_typo_95.json",
    "artifacts/lexicon/romaji2ja_feedback_95.json",
    "artifacts/lexicon/romaji2ja.json",
)
DEFAULT_CANDIDATE_FEEDBACK = "artifacts/lexicon/candidate_feedback.jsonl"
DEFAULT_GENERAL_LEXICON = "artifacts/lexicon/general_reading_lexicon.json"
DEFAULT_AUX_LEXICON = "artifacts/lexicon/adversarial_piece_aliases.json"
FUZZY_CANDIDATE_LIMIT = 160
WEIGHTED_FUZZY_CANDIDATE_LIMIT = 128
WEIGHTED_FUZZY_FAST_CANDIDATE_LIMIT = 16
WEIGHTED_FUZZY_FAST_ACCEPT_SCORE = 0.26
FUZZY_SEGMENT_MAX_SPLITS = 4
FUZZY_TRIPLE_MAX_SPLITS = 48
MULTI_SEGMENT_FUZZY_PIECE_PROBE_LIMIT = 12
ANCHORED_FUZZY_MIN_ANCHOR_LEN = 12
DEEP_MULTI_SEGMENT_MIN_LEN = 56
SANDWICH_FUZZY_MIN_MIDDLE_LEN = 5
FAST_PATH_VERSION = "fastpath-v46-general-phrase-prefuzzy-exact-20260614"
LONG_FUZZY_SEGMENT_MAX_SCORE = 0.42
MULTI_SEGMENT_FUZZY_MAX_SCORE = 0.26
RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE = 0.28
EXTENDED_MULTI_SEGMENT_MIN_LEN = 100
LONG_SEGMENT_MIN_LEN = 100
LONG_SEGMENT_MAX_SEGMENTS = 20
WIDE_MULTI_SEGMENT_MIN_LEN = 120
WIDE_MULTI_SEGMENT_MAX_FUZZY_SEGMENTS = 8
WIDE_MULTI_SEGMENT_MAX_TRANSITIONS = 8
WIDE_MULTI_SEGMENT_BEAM_WIDTH = 16
WIDE_MULTI_SEGMENT_PIECE_PROBE_LIMIT = 8
WIDE_MULTI_SEGMENT_MIN_OUTPUT_RATIO = 0.25
WIDE_MULTI_SEGMENT_WEIGHTED_CANDIDATE_LIMIT = 32
WIDE_MULTI_SEGMENT_WIDE_PLAIN_ACCEPT_SCORE = 0.28
WIDE_MULTI_SEGMENT_BOUNDARY_LENGTH_DELTA = 1
WIDE_MULTI_SEGMENT_RESCUE_MIN_LEN = 260
WIDE_MULTI_SEGMENT_RESCUE_MAX_FUZZY_SEGMENTS = 12
WIDE_MULTI_SEGMENT_RESCUE_BEAM_WIDTH = 24
WIDE_MULTI_SEGMENT_RESCUE_PIECE_PROBE_LIMIT = 12
WIDE_MULTI_SEGMENT_RESCUE_WEIGHTED_CANDIDATE_LIMIT = 16
WIDE_MULTI_SEGMENT_RESCUE_PLAIN_ACCEPT_SCORE = 0.28
WIDE_MULTI_SEGMENT_PLAIN_ACCEPT_SCORE = 0.22
LONG_SINGLE_FUZZY_WEIGHTED_CANDIDATE_LIMIT = 16
LONG_SINGLE_FUZZY_PLAIN_ACCEPT_SCORE = 0.28
LONG_FUZZY_BOUNDARY_LENGTH_DELTA = 3
DENSE_OVERFLOW_MIN_LEN = 240
DENSE_OVERFLOW_DIRECT_MIN_LEN = 400
DENSE_OVERFLOW_MIN_SEGMENTS = LONG_SEGMENT_MAX_SEGMENTS + 1
DENSE_OVERFLOW_MAX_SEGMENTS = 64
DENSE_OVERFLOW_MAX_FUZZY_SEGMENTS = 32
DENSE_OVERFLOW_BEAM_WIDTH = 4
DENSE_OVERFLOW_MAX_SCORE = 0.24
DENSE_OVERFLOW_MAX_DISTANCE_RATIO = 0.30
DENSE_OVERFLOW_FUZZY_COST = 0.03
DENSE_COMPACT_FUZZY_KEEP_PER_OUTPUT_LENGTH = 2
SHORT_VITERBI_COMPACT_FUZZY_KEEP_PER_OUTPUT_LENGTH = 4
DENSE_OVERFLOW_RESCUE_MAX_SCORE = 0.27
DENSE_OVERFLOW_RESCUE_BEAM_WIDTH = 2
DENSE_OVERFLOW_WEIGHTED_RESCUE_BEAM_WIDTH = 1
DENSE_OVERFLOW_FAST_WEIGHTED_RESCUE_BEAM_WIDTH = 2
DENSE_OVERFLOW_MAX_FULL_RESCUE_SEGMENTS = 1
DENSE_OVERFLOW_FULL_RESCUE_COST = 0.02
DENSE_OVERFLOW_WEIGHTED_CANDIDATE_LIMIT = 32
DENSE_OVERFLOW_FAST_WEIGHTED_CANDIDATE_LIMIT = 4
DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_SCORE = 0.24
DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_MARGIN = 0.02
DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_CANDIDATE_LIMIT = 32
DENSE_OVERFLOW_RELAXED_WEIGHTED_RESCUE_MAX_SCORE = 0.29
DENSE_OVERFLOW_RELAXED_WEIGHTED_CANDIDATE_LIMIT = 8
DENSE_OVERFLOW_SHORT_VITERBI_MIN_LEN = 800
DENSE_OVERFLOW_SHORT_VITERBI_MIN_SEGMENTS = 50
DENSE_OVERFLOW_SHORT_VITERBI_MAX_SEGMENTS = 96
DENSE_OVERFLOW_SHORT_VITERBI_MIN_PIECE_LEN = 5
DENSE_OVERFLOW_SHORT_VITERBI_MAX_PIECE_LEN = 28
DENSE_OVERFLOW_SHORT_VITERBI_MAX_SCORE = 0.32
DENSE_OVERFLOW_SHORT_VITERBI_MAX_DISTANCE_RATIO = 0.32
DENSE_OVERFLOW_SHORT_VITERBI_WEIGHTED_CANDIDATE_LIMIT = 3
DENSE_OVERFLOW_SHORT_VITERBI_POSITION_BEAM = 3
DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_WEIGHTED_CANDIDATE_LIMIT = 32
DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_MAX_SCORE = 0.42
DENSE_OVERFLOW_SHORT_VITERBI_MAX_FUZZY_SEGMENTS = 64
DENSE_OVERFLOW_SHORT_VITERBI_MAX_COST_PER_SEGMENT = 0.20
DENSE_OVERFLOW_SHORT_VITERBI_MAX_FUZZY_RATIO = 0.75
DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_SCORE = 0.31
DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MARGIN = 0.02
DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_CANDIDATE_LIMIT = 32
DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MIN_DISTANCE = 5
DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MIN_LEN_DELTA = 2
QWERTY_ROWS = (
    ("1234567890", 0.0),
    ("qwertyuiop", 0.25),
    ("asdfghjkl", 0.75),
    ("zxcvbnm", 1.25),
)
KEY_POS = {c: (x + i, float(y)) for y, (row, x) in enumerate(QWERTY_ROWS) for i, c in enumerate(row)}
VOWEL_NEIGHBORS = {
    "a": "ieo",
    "i": "aeu",
    "u": "ioe",
    "e": "iau",
    "o": "aue",
}


def resolve_lexicon_path(path: str | None) -> str | None:
    if not path:
        return None
    if path == "auto":
        for candidate in DEFAULT_LEXICON_CANDIDATES:
            if Path(candidate).exists():
                return candidate
        return None
    return path


def load_lexicon(path: str | None, *, include_aux: bool = True) -> dict:
    resolved = resolve_lexicon_path(path)
    if not resolved:
        return {}
    p = Path(resolved)
    if not p.exists():
        return {}
    lexicon = json.loads(p.read_text(encoding="utf-8"))
    aux_path = Path(DEFAULT_AUX_LEXICON)
    if include_aux and aux_path.exists() and aux_path.resolve() != p.resolve():
        aux = json.loads(aux_path.read_text(encoding="utf-8"))
        for key, value in aux.items():
            lexicon.setdefault(normalize_input(key), value)
    return lexicon


def load_choice_feedback(path: str | None) -> dict[str, str]:
    if not path:
        return {}
    p = Path(path)
    if not p.exists():
        return {}
    choices = {}
    with p.open(encoding="utf-8") as f:
        for line in f:
            if not line.strip():
                continue
            row = json.loads(line)
            inp = normalize_input(row.get("input", ""))
            out = row.get("output")
            if inp and out:
                choices[inp] = out
    return choices


def file_fingerprint(label: str, path: str | None) -> str:
    if not path:
        return f"{label}:none"
    p = Path(path)
    if not p.exists():
        return f"{label}:{path}:missing"
    st = p.stat()
    return f"{label}:{p.resolve()}:{st.st_size}:{int(st.st_mtime)}"


def build_lexicon_index(lexicon: dict) -> dict[int, list[tuple[str, str]]]:
    by_len = {}
    for key, value in lexicon.items():
        by_len.setdefault(len(key), []).append((key, value))
    return by_len


def build_lexicon_lengths(lexicon: dict) -> list[int]:
    return sorted({len(key) for key in lexicon}, reverse=True)


def boundary_fuzzy_lengths(
    fuzzy_lengths: list[int],
    exact_lengths: list[int],
    max_len: int,
    *,
    min_len: int = 5,
    delta: int = LONG_FUZZY_BOUNDARY_LENGTH_DELTA,
) -> list[int]:
    lengths = {length for length in fuzzy_lengths if min_len <= length <= max_len}
    for base_len in exact_lengths:
        for offset in range(-delta, delta + 1):
            length = base_len + offset
            if min_len <= length <= max_len:
                lengths.add(length)
    return sorted(lengths, reverse=True)


def key_quality(key: str, median_len: float) -> tuple[float, int, str]:
    noisy = sum(1 for ch in key if ch.isdigit() or not ch.isalpha())
    rare = sum(1 for ch in key if ch in "qxz")
    return (abs(len(key) - median_len) + noisy * 8 + rare * 0.25, len(key), key)


def build_compact_fuzzy_lexicon(lexicon: dict, keep_per_output: int = 8) -> dict:
    by_output = {}
    for key, value in lexicon.items():
        by_output.setdefault(value, []).append(key)

    compact = {}
    for value, keys in by_output.items():
        lengths = sorted(len(key) for key in keys)
        mid = len(lengths) // 2
        median_len = lengths[mid] if len(lengths) % 2 else (lengths[mid - 1] + lengths[mid]) / 2
        for key in sorted(keys, key=lambda item: key_quality(item, median_len))[:keep_per_output]:
            compact[key] = value
    return compact


def build_dense_compact_fuzzy_lexicon(
    lexicon: dict,
    keep_per_output: int = 8,
    keep_per_output_length: int = DENSE_COMPACT_FUZZY_KEEP_PER_OUTPUT_LENGTH,
) -> dict:
    compact = build_compact_fuzzy_lexicon(lexicon, keep_per_output=keep_per_output)
    by_output_length = {}
    for key, value in lexicon.items():
        by_output_length.setdefault((value, len(key)), []).append(key)
    for (value, length), keys in by_output_length.items():
        for key in sorted(keys, key=lambda item: key_quality(item, length))[:keep_per_output_length]:
            compact[key] = value
    return compact


def char_grams(text: str) -> set[str]:
    if len(text) <= 3:
        return {text}
    width = 2 if len(text) <= 10 else 3
    return {text[i:i + width] for i in range(0, len(text) - width + 1)}


def build_lexicon_gram_index(lexicon: dict) -> dict[str, list[tuple[str, str]]]:
    index = {}
    for key, value in lexicon.items():
        for gram in char_grams(key):
            index.setdefault(gram, []).append((key, value))
    return index


def edit_distance(a: str, b: str, max_dist: int | None = None) -> int:
    if a == b:
        return 0
    if max_dist is not None and abs(len(a) - len(b)) > max_dist:
        return max_dist + 1
    if _RapidLevenshtein is not None:
        if max_dist is None:
            return int(_RapidLevenshtein.distance(a, b))
        return int(_RapidLevenshtein.distance(a, b, score_cutoff=max_dist))
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i] + [0] * len(b)
        row_min = cur[0]
        for j, cb in enumerate(b, 1):
            cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))
            row_min = min(row_min, cur[j])
        if max_dist is not None and row_min > max_dist:
            return max_dist + 1
        prev = cur
    return prev[-1]


def keyboard_substitution_cost(a: str, b: str) -> float:
    if a == b:
        return 0.0
    if a in VOWEL_NEIGHBORS and b in VOWEL_NEIGHBORS[a]:
        return 0.45
    if a in KEY_POS and b in KEY_POS:
        ax, ay = KEY_POS[a]
        bx, by = KEY_POS[b]
        dist = ((ax - bx) ** 2 + (ay - by) ** 2) ** 0.5
        if dist <= 1.15:
            return 0.35
        if dist <= 1.7:
            return 0.65
    return 1.0


KEYBOARD_SUBSTITUTION_COSTS = {
    ca: {cb: keyboard_substitution_cost(ca, cb) for cb in "abcdefghijklmnopqrstuvwxyz0123456789"}
    for ca in "abcdefghijklmnopqrstuvwxyz0123456789"
}


def weighted_edit_distance(a: str, b: str, max_dist: float | None = None) -> float:
    if a == b:
        return 0.0
    if max_dist is not None and abs(len(a) - len(b)) * 0.9 > max_dist:
        return max_dist + 1.0
    prev = [i * 0.9 for i in range(len(b) + 1)]
    for i, ca in enumerate(a, 1):
        cur = [i * 0.9] + [0.0] * len(b)
        row_min = cur[0]
        substitution_costs = KEYBOARD_SUBSTITUTION_COSTS.get(ca)
        for j, cb in enumerate(b, 1):
            substitution_cost = substitution_costs.get(cb, 1.0) if substitution_costs is not None else keyboard_substitution_cost(ca, cb)
            delete_cost = prev[j] + 0.9
            insert_cost = cur[j - 1] + 0.9
            replace_cost = prev[j - 1] + substitution_cost
            best_cost = delete_cost if delete_cost < insert_cost else insert_cost
            if replace_cost < best_cost:
                best_cost = replace_cost
            cur[j] = best_cost
            if best_cost < row_min:
                row_min = best_cost
        if max_dist is not None and row_min > max_dist:
            return max_dist + 1.0
        prev = cur
    return prev[-1]


def fuzzy_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_by_len: dict | None = None,
    lexicon_gram_index: dict | None = None,
):
    match = fuzzy_lexicon_match(inp, lexicon, lexicon_by_len, lexicon_gram_index)
    return match["value"] if match else None


def fuzzy_lexicon_match(
    inp: str,
    lexicon: dict,
    lexicon_by_len: dict | None = None,
    lexicon_gram_index: dict | None = None,
):
    if not lexicon:
        return None
    max_dist = 2 if len(inp) <= 8 else 3 if len(inp) <= 12 else 5 if len(inp) <= 18 else 6 if len(inp) <= 32 else 7
    best = None
    tied_values = set()
    allowed_lengths = set(range(len(inp) - max_dist, len(inp) + max_dist + 1))
    if lexicon_gram_index is not None:
        counts = Counter()
        values = {}
        for gram in char_grams(inp):
            for key, value in lexicon_gram_index.get(gram, ()):
                if len(key) in allowed_lengths:
                    counts[key] += 1
                    values[key] = value
        if counts:
            candidates = [
                (key, values[key])
                for key, _ in counts.most_common(FUZZY_CANDIDATE_LIMIT)
            ]
        elif lexicon_by_len is not None:
            candidates = []
            for length in allowed_lengths:
                candidates.extend(lexicon_by_len.get(length, ()))
        else:
            candidates = lexicon.items()
    elif lexicon_by_len is not None:
        candidates = []
        for length in allowed_lengths:
            candidates.extend(lexicon_by_len.get(length, ()))
    else:
        candidates = lexicon.items()
    for key, value in candidates:
        dist = edit_distance(inp, key, max_dist=max_dist)
        if dist > max_dist:
            continue
        score = dist / max(1, max(len(inp), len(key)))
        cand = (dist, score, key, value)
        if best is None or cand[:2] < best[:2]:
            best = cand
            tied_values = {value}
        elif cand[:2] == best[:2]:
            tied_values.add(value)
    if best is None:
        return None
    if best[1] > 0.36:
        return None
    if len(tied_values) > 1:
        return None
    return {
        "key": best[2],
        "value": best[3],
        "distance": best[0],
        "score": best[1],
    }


def weighted_fuzzy_lexicon_match(
    inp: str,
    lexicon: dict,
    lexicon_by_len: dict | None = None,
    lexicon_gram_index: dict | None = None,
    *,
    candidate_limit: int | None = None,
):
    if not lexicon:
        return None
    threshold = 0.46
    max_dist = threshold * max(1, len(inp))
    allowed_lengths = set(range(len(inp) - 7, len(inp) + 8))
    if lexicon_gram_index is not None:
        counts = Counter()
        values = {}
        for gram in char_grams(inp):
            for key, value in lexicon_gram_index.get(gram, ()):
                if len(key) in allowed_lengths:
                    counts[key] += 1
                    values[key] = value
        limit = candidate_limit or WEIGHTED_FUZZY_CANDIDATE_LIMIT
        candidates = [(key, values[key]) for key, _ in counts.most_common(limit)]
    elif lexicon_by_len is not None:
        candidates = []
        for length in allowed_lengths:
            candidates.extend(lexicon_by_len.get(length, ()))
    else:
        candidates = list(lexicon.items())

    def scan(candidate_slice, best=None, tied_values=None):
        if tied_values is None:
            tied_values = set()
        for key, value in candidate_slice:
            dist = weighted_edit_distance(inp, key, max_dist=max_dist)
            if dist > max_dist:
                continue
            score = dist / max(1, max(len(inp), len(key)))
            cand = (dist, score, key, value)
            if best is None or cand[:2] < best[:2]:
                best = cand
                tied_values = {value}
            elif cand[:2] == best[:2]:
                tied_values.add(value)
        return best, tied_values

    fast_limit = min(WEIGHTED_FUZZY_FAST_CANDIDATE_LIMIT, len(candidates))
    best, tied_values = scan(candidates[:fast_limit])
    if (
        best is None
        or best[1] > WEIGHTED_FUZZY_FAST_ACCEPT_SCORE
        or len(tied_values) > 1
    ) and fast_limit < len(candidates):
        best, tied_values = scan(candidates[fast_limit:], best, tied_values)

    if best is None or best[1] > threshold or len(tied_values) > 1:
        return None
    return {
        "key": best[2],
        "value": best[3],
        "distance": best[0],
        "score": best[1],
    }


def segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None = None,
    *,
    min_segments: int = 2,
    max_segments: int = 8,
):
    if not lexicon or len(inp) < 16:
        return None
    lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    if not lengths:
        return None

    n = len(inp)
    dp = [None] * (n + 1)
    dp[n] = (0, {""})
    for i in range(n - 1, -1, -1):
        best_score = None
        best_outputs = set()
        for length in lengths:
            j = i + length
            if j > n:
                continue
            piece = inp[i:j]
            value = lexicon.get(piece)
            if value is None or dp[j] is None:
                continue
            tail_segments, tail_outputs = dp[j]
            segments = tail_segments + 1
            if segments > max_segments:
                continue
            score = (segments, -length)
            outputs = {value + tail for tail in tail_outputs}
            if best_score is None or score < best_score:
                best_score = score
                best_outputs = outputs
            elif score == best_score:
                best_outputs.update(outputs)
                if len(best_outputs) > 1:
                    best_outputs = set(list(best_outputs)[:2])
        if best_score is not None:
            dp[i] = (best_score[0], best_outputs)

    if dp[0] is None:
        return None
    segments, outputs = dp[0]
    if segments < min_segments or len(outputs) != 1:
        return None
    return next(iter(outputs))


def has_exact_subpiece(inp: str, lexicon: dict, lexicon_lengths: list[int] | None = None, min_len: int = 10) -> bool:
    if not lexicon or len(inp) < min_len:
        return False
    lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    n = len(inp)
    for i in range(n - min_len + 1):
        for length in lengths:
            if length < min_len:
                continue
            j = i + length
            if j > n:
                continue
            if inp[i:j] in lexicon:
                return True
    return False


def fuzzy_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_by_len: dict | None = None,
    lexicon_gram_index: dict | None = None,
    lexicon_lengths: list[int] | None = None,
    *,
    max_delta: int = 7,
    max_splits: int = FUZZY_SEGMENT_MAX_SPLITS,
):
    if not lexicon or len(inp) < 16:
        return None
    lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    if not lengths:
        return None

    n = len(inp)
    split_candidates = set()
    min_len = min(lengths)
    max_len = max(lengths)
    for length in lengths:
        for delta in range(-max_delta, max_delta + 1):
            split = length + delta
            if min_len <= split <= n - min_len:
                right_len = n - split
                if min_len - max_delta <= right_len <= max_len + max_delta:
                    split_candidates.add(split)

    def overlap_score(piece: str) -> float:
        if piece in lexicon:
            return 10.0
        if lexicon_gram_index is None:
            return 0.0
        grams = char_grams(piece)
        if not grams:
            return 0.0
        counts = Counter()
        allowed = set(range(len(piece) - max_delta, len(piece) + max_delta + 1))
        for gram in grams:
            for key, _ in lexicon_gram_index.get(gram, ()):
                if len(key) in allowed:
                    counts[key] += 1
        if not counts:
            return 0.0
        return counts.most_common(1)[0][1] / max(1, len(grams))

    if len(split_candidates) > max_splits:
        split_candidates = {
            split
            for split, _ in sorted(
                (
                    (split, overlap_score(inp[:split]) + overlap_score(inp[split:]))
                    for split in split_candidates
                ),
                key=lambda item: (-item[1], abs(item[0] - n / 2), item[0]),
            )[:max_splits]
        }

    best_score = None
    best_outputs = set()
    match_cache = {}

    def piece_match(piece: str):
        if piece in lexicon:
            return {"key": piece, "value": lexicon[piece], "distance": 0, "score": 0.0, "exact": True}
        if piece not in match_cache:
            match = fuzzy_lexicon_match(piece, lexicon, lexicon_by_len, lexicon_gram_index)
            if match is None:
                match = weighted_fuzzy_lexicon_match(piece, lexicon, lexicon_by_len, lexicon_gram_index)
            match_cache[piece] = match
        match = match_cache[piece]
        if match is None:
            return None
        return {**match, "exact": False}

    for split in sorted(split_candidates):
        left = piece_match(inp[:split])
        if left is None:
            continue
        right = piece_match(inp[split:])
        if right is None:
            continue

        fuzzy_count = int(not left["exact"]) + int(not right["exact"])
        if fuzzy_count == 0:
            continue
        dist = left["distance"] + right["distance"]
        score = left["score"] + right["score"]
        # Prefer exact+fuzzy repairs, then lower edit cost, then cleaner scores.
        cand_score = (fuzzy_count, dist, round(score, 6), abs(split - n / 2))
        output = left["value"] + right["value"]
        if best_score is None or cand_score < best_score:
            best_score = cand_score
            best_outputs = {output}
        elif cand_score == best_score:
            best_outputs.add(output)
            if len(best_outputs) > 1:
                best_outputs = set(list(best_outputs)[:2])

    if best_score is None or len(best_outputs) != 1:
        return None
    # Avoid very loose two-sided repairs; whole-model fallback is better there.
    if best_score[0] == 2 and best_score[1] > 10:
        return None
    # Long two-piece repairs can accidentally swallow multiple intended phrases
    # into one noisy lexicon alias. Prefer multi-segment repair or model fallback.
    if len(inp) >= 40 and best_score[0] == 1 and best_score[2] > LONG_FUZZY_SEGMENT_MAX_SCORE:
        return None
    return next(iter(best_outputs))


def anchored_fuzzy_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    *,
    min_anchor_len: int = ANCHORED_FUZZY_MIN_ANCHOR_LEN,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < 32:
        return None
    lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    if not lengths:
        return None

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > 0.32 or match["distance"] > max(5, len(piece) * 0.28):
            return None
        return match

    def fuzzy_piece_match(piece: str):
        if len(piece) < 5:
            return None
        match = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(weighted_fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        return match

    candidates = []
    for length in lengths:
        if length < min_anchor_len or length >= len(inp) - 5:
            continue
        prefix = inp[:length]
        prefix_value = lexicon.get(prefix)
        if prefix_value is not None:
            rem = inp[length:]
            match = fuzzy_piece_match(rem)
            if match is not None:
                candidates.append((match["distance"], match["score"], -length, prefix_value + match["value"]))

        suffix = inp[-length:]
        suffix_value = lexicon.get(suffix)
        if suffix_value is not None:
            rem = inp[:-length]
            match = fuzzy_piece_match(rem)
            if match is not None:
                candidates.append((match["distance"], match["score"], -length, match["value"] + suffix_value))

    if not candidates:
        return None
    ranked = sorted(candidates)
    if len(ranked) > 1 and ranked[0][:3] == ranked[1][:3] and ranked[0][3] != ranked[1][3]:
        return None
    return ranked[0][3]


def sandwich_fuzzy_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_anchor_len: int = ANCHORED_FUZZY_MIN_ANCHOR_LEN,
    min_middle_len: int = SANDWICH_FUZZY_MIN_MIDDLE_LEN,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < 40:
        return None
    lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    if not lengths:
        return None

    prefix_hits = []
    suffix_hits = []
    n = len(inp)
    for length in lengths:
        if length < min_anchor_len or length > n - min_middle_len:
            continue
        prefix_value = lexicon.get(inp[:length])
        if prefix_value is not None:
            prefix_hits.append((length, prefix_value))
        suffix_value = lexicon.get(inp[n - length :])
        if suffix_value is not None:
            suffix_hits.append((n - length, suffix_value, length))
    if not prefix_hits or not suffix_hits:
        return None

    match_cache = {}

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > 0.36 or match["distance"] > max(7, len(piece) * 0.34):
            return None
        return match

    def middle_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]
        match = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(weighted_fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(weighted_fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index), piece)
        match_cache[piece] = match
        return match

    candidates = []
    for prefix_end, prefix_value in prefix_hits:
        for suffix_start, suffix_value, suffix_len in suffix_hits:
            if suffix_start - prefix_end < min_middle_len:
                continue
            middle = inp[prefix_end:suffix_start]
            if middle in lexicon:
                continue
            match = middle_match(middle)
            if match is None:
                continue
            anchor_len = prefix_end + suffix_len
            candidates.append((
                match["distance"],
                round(match["score"], 6),
                -anchor_len,
                abs((suffix_start - prefix_end) - n / 3),
                prefix_value + match["value"] + suffix_value,
            ))

    if not candidates:
        return None
    ranked = sorted(candidates)
    if len(ranked) > 1 and ranked[0][:4] == ranked[1][:4] and ranked[0][4] != ranked[1][4]:
        return None
    return ranked[0][4]


def fuzzy_multi_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    fuzzy_lexicon_lengths: list[int] | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_segments: int = 3,
    max_segments: int = 8,
    max_fuzzy_segments: int = 2,
    max_fuzzy_transitions: int = 6,
    beam_width: int = 12,
    max_fuzzy_score: float = MULTI_SEGMENT_FUZZY_MAX_SCORE,
    piece_probe_limit: int = MULTI_SEGMENT_FUZZY_PIECE_PROBE_LIMIT,
    prefer_short_pieces: bool = False,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < 24:
        return None
    exact_lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    fuzzy_lengths = fuzzy_lexicon_lengths or build_lexicon_lengths(fuzzy_lexicon)
    if not exact_lengths or not fuzzy_lengths:
        return None

    n = len(inp)
    match_cache = {}
    overlap_cache = {}

    def fuzzy_piece_overlap(piece: str) -> float:
        if piece in overlap_cache:
            return overlap_cache[piece]
        if fuzzy_lexicon_gram_index is None:
            overlap_cache[piece] = 0.0
            return 0.0
        grams = char_grams(piece)
        if not grams:
            overlap_cache[piece] = 0.0
            return 0.0
        counts = Counter()
        allowed = set(range(len(piece) - 7, len(piece) + 8))
        for gram in grams:
            for key, _ in fuzzy_lexicon_gram_index.get(gram, ()):
                if len(key) in allowed:
                    counts[key] += 1
        score = counts.most_common(1)[0][1] / max(1, len(grams)) if counts else 0.0
        overlap_cache[piece] = score
        return score

    def fuzzy_piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]

        def shared_prefix_len(a: str, b: str) -> int:
            count = 0
            for ca, cb in zip(a, b):
                if ca != cb:
                    break
                count += 1
            return count

        def acceptable(match):
            if match is None:
                return None
            # Multi-segment repair is powerful; keep it conservative so a loose
            # fuzzy chunk does not swallow multiple intended phrases.
            if match["score"] > max_fuzzy_score or match["distance"] > max(5, len(piece) * 0.28):
                return None
            return match

        compact_plain = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index))
        full_plain = acceptable(fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index))
        if (
            full_plain is not None
            and full_plain["distance"] <= 2
            and full_plain["score"] <= 0.16
            and shared_prefix_len(piece, full_plain["key"]) >= min(2, len(piece), len(full_plain["key"]))
        ):
            preferred = dict(full_plain)
            preferred["score"] = min(preferred["score"], preferred["distance"] * 0.02)
            match_cache[piece] = preferred
            return preferred

        candidates = [match for match in (compact_plain, full_plain) if match is not None]
        compact_weighted = acceptable(weighted_fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index))
        full_weighted = acceptable(weighted_fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index))
        candidates.extend(match for match in (compact_weighted, full_weighted) if match is not None)
        match = None
        if candidates:
            candidates.sort(key=lambda item: (round(item["score"], 6), item["distance"], len(item["key"])))
            match = candidates[0]
        match_cache[piece] = match
        return match

    min_fuzzy_len = max(5, min(fuzzy_lengths))

    def prune(items):
        best_by_output = {}
        for item in items:
            key = item[3]
            score = item[:3]
            if key not in best_by_output or score < best_by_output[key][:3]:
                best_by_output[key] = item
        return sorted(best_by_output.values(), key=lambda item: (item[0], item[1], item[2], len(item[3])))[:beam_width]

    @lru_cache(maxsize=None)
    def solve_exact(pos: int):
        if pos == n:
            return ((0, 0.0, 0, ""),)

        exact_results = []
        for length in exact_lengths:
            j = pos + length
            if j > n:
                continue
            value = lexicon.get(inp[pos:j])
            if value is None:
                continue
            for rest_fuzzy, rest_cost, rest_segments, rest_output in solve_exact(j):
                segments = rest_segments + 1
                if segments <= max_segments:
                    exact_results.append((rest_fuzzy, rest_cost, segments, value + rest_output))
        return tuple(prune(exact_results))

    @lru_cache(maxsize=None)
    def solve(pos: int, fuzzy_left: int):
        if pos == n:
            return ((0, 0.0, 0, ""),)

        exact_results = []
        for length in exact_lengths:
            j = pos + length
            if j > n:
                continue
            value = lexicon.get(inp[pos:j])
            if value is None:
                continue
            for rest_fuzzy, rest_cost, rest_segments, rest_output in solve(j, fuzzy_left):
                segments = rest_segments + 1
                if segments <= max_segments:
                    exact_results.append((rest_fuzzy, rest_cost, segments, value + rest_output))

        # If exact segmentation can continue to the end, keep it. This makes the
        # expensive fuzzy branch run only at the first position where exact
        # segmentation gets stuck, which is the common long-input typo shape.
        if exact_results:
            return tuple(prune(exact_results))

        if fuzzy_left <= 0:
            return ()

        candidate_pieces = []
        for length in fuzzy_lengths:
            if length < min_fuzzy_len:
                continue
            j = pos + length
            if j > n:
                continue
            piece = inp[pos:j]
            if piece in lexicon:
                continue
            overlap = fuzzy_piece_overlap(piece)
            if overlap <= 0.0:
                continue
            if prefer_short_pieces:
                tail_exact_penalty = 0 if solve_exact(j) else 1
                candidate_pieces.append((
                    tail_exact_penalty,
                    -overlap,
                    length,
                    abs((n - j) - (n / 2)),
                    j,
                    piece,
                ))
            else:
                candidate_pieces.append((-overlap, abs((n - j) - (n / 2)), length, j, piece))
        candidate_pieces = sorted(candidate_pieces)[:piece_probe_limit]

        fuzzy_transitions = []
        for candidate in candidate_pieces:
            j, piece = candidate[-2], candidate[-1]
            tail_states = solve_exact(j)
            if not tail_states and fuzzy_left > 1:
                tail_states = solve(j, fuzzy_left - 1)
            if not tail_states:
                continue
            match = fuzzy_piece_match(piece)
            if match is None:
                continue
            cost = match["score"] + match["distance"] * 0.01
            fuzzy_transitions.append((cost, j, match, tail_states))

        fuzzy_results = []
        for cost, j, match, tail_states in sorted(
            fuzzy_transitions,
            key=lambda item: (item[0], abs((n - item[1]) - (n / 2))),
        )[:max_fuzzy_transitions]:
            for rest_fuzzy, rest_cost, rest_segments, rest_output in tail_states:
                segments = rest_segments + 1
                if segments <= max_segments:
                    fuzzy_results.append((
                        rest_fuzzy + 1,
                        rest_cost + cost,
                        segments,
                        match["value"] + rest_output,
                    ))
        return tuple(prune(fuzzy_results))

    finals = [state for state in solve(0, max_fuzzy_segments) if state[2] >= min_segments and state[0] > 0]
    if not finals:
        return None
    best_by_output = {}
    for item in finals:
        key = item[3]
        score = item[:3]
        if key not in best_by_output or score < best_by_output[key][:3]:
            best_by_output[key] = item
    ranked = sorted(best_by_output.values(), key=lambda item: (item[0], item[1], item[2], len(item[3])))
    if not ranked:
        return None
    if len(ranked) > 1 and ranked[0][:3] == ranked[1][:3] and ranked[0][3] != ranked[1][3]:
        return None
    return ranked[0][3]


def wide_beam_multi_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    fuzzy_lexicon_lengths: list[int] | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_segments: int = 3,
    max_segments: int = LONG_SEGMENT_MAX_SEGMENTS,
    max_fuzzy_segments: int = WIDE_MULTI_SEGMENT_MAX_FUZZY_SEGMENTS,
    beam_width: int = 32,
    piece_probe_limit: int = 18,
    max_fuzzy_score: float = RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
    plain_accept_score: float | None = WIDE_MULTI_SEGMENT_PLAIN_ACCEPT_SCORE,
    use_compact_weighted: bool = True,
    weighted_candidate_limit: int | None = None,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < WIDE_MULTI_SEGMENT_MIN_LEN:
        return None
    exact_lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    fuzzy_lengths = fuzzy_lexicon_lengths or build_lexicon_lengths(fuzzy_lexicon)
    if not exact_lengths or not fuzzy_lengths:
        return None

    n = len(inp)
    min_fuzzy_len = max(5, min(fuzzy_lengths))
    match_cache = {}
    overlap_cache = {}

    def shared_prefix_len(a: str, b: str) -> int:
        count = 0
        for ca, cb in zip(a, b):
            if ca != cb:
                break
            count += 1
        return count

    def fuzzy_piece_overlap(piece: str) -> float:
        if piece in overlap_cache:
            return overlap_cache[piece]
        if fuzzy_lexicon_gram_index is None:
            overlap_cache[piece] = 0.0
            return 0.0
        grams = char_grams(piece)
        if not grams:
            overlap_cache[piece] = 0.0
            return 0.0
        counts = Counter()
        allowed = set(range(len(piece) - 7, len(piece) + 8))
        for gram in grams:
            for key, _ in fuzzy_lexicon_gram_index.get(gram, ()):
                if len(key) in allowed:
                    counts[key] += 1
        score = counts.most_common(1)[0][1] / max(1, len(grams)) if counts else 0.0
        overlap_cache[piece] = score
        return score

    def fuzzy_piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]

        def acceptable(match):
            if match is None:
                return None
            if match["score"] > max_fuzzy_score or match["distance"] > max(5, len(piece) * 0.28):
                return None
            if len(match["value"]) < max(5, int(len(piece) * WIDE_MULTI_SEGMENT_MIN_OUTPUT_RATIO)):
                return None
            return match

        compact_plain = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index))
        full_plain = acceptable(fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index))
        if (
            full_plain is not None
            and full_plain["distance"] <= 2
            and full_plain["score"] <= 0.16
            and shared_prefix_len(piece, full_plain["key"]) >= min(2, len(piece), len(full_plain["key"]))
        ):
            preferred = dict(full_plain)
            preferred["score"] = min(preferred["score"], preferred["distance"] * 0.02)
            match_cache[piece] = preferred
            return preferred

        candidates = [match for match in (compact_plain, full_plain) if match is not None]
        if candidates:
            candidates.sort(key=lambda item: (round(item["score"], 6), item["distance"], len(item["key"])))
            if plain_accept_score is not None and candidates[0]["score"] <= plain_accept_score:
                match_cache[piece] = candidates[0]
                return candidates[0]
        compact_weighted = None
        if use_compact_weighted:
            compact_weighted = acceptable(weighted_fuzzy_lexicon_match(
                piece,
                fuzzy_lexicon,
                fuzzy_lexicon_by_len,
                fuzzy_lexicon_gram_index,
                candidate_limit=weighted_candidate_limit,
            ))
        full_weighted = acceptable(weighted_fuzzy_lexicon_match(
            piece,
            lexicon,
            full_lexicon_by_len,
            full_lexicon_gram_index,
            candidate_limit=weighted_candidate_limit,
        ))
        candidates.extend(match for match in (compact_weighted, full_weighted) if match is not None)
        match = None
        if candidates:
            candidates.sort(key=lambda item: (round(item["score"], 6), item["distance"], len(item["key"])))
            match = candidates[0]
        match_cache[piece] = match
        return match

    def prune_states(items):
        best_by_position_output = {}
        for item in items:
            fuzzy_count, cost, segments, pos, output = item
            key = (pos, output)
            score = (fuzzy_count, -segments, round(cost, 6))
            if key not in best_by_position_output or score < best_by_position_output[key][0]:
                best_by_position_output[key] = (score, (fuzzy_count, cost, segments, pos, output))
        return sorted(
            (item for _, item in best_by_position_output.values()),
            key=lambda item: (item[0], -item[2], round(item[1], 6), -item[3], len(item[4])),
        )[:beam_width]

    states = [(0, 0.0, 0, 0, "")]
    finals = []
    for _ in range(max_segments):
        next_states = []
        for fuzzy_count, cost, segments, pos, output in states:
            if pos == n:
                if segments >= min_segments and fuzzy_count > 0:
                    finals.append((fuzzy_count, cost, segments, output))
                continue

            exact_transitions = []
            for length in exact_lengths:
                j = pos + length
                if j > n:
                    continue
                value = lexicon.get(inp[pos:j])
                if value is None:
                    continue
                exact_transitions.append((fuzzy_count, cost, segments + 1, j, output + value))
            next_states.extend(exact_transitions)

            if fuzzy_count >= max_fuzzy_segments:
                continue
            piece_candidates = []
            for length in boundary_fuzzy_lengths(
                fuzzy_lengths,
                exact_lengths,
                n - pos,
                min_len=min_fuzzy_len,
                delta=WIDE_MULTI_SEGMENT_BOUNDARY_LENGTH_DELTA,
            ):
                if length < min_fuzzy_len:
                    continue
                j = pos + length
                if j > n:
                    continue
                piece = inp[pos:j]
                if piece in lexicon:
                    continue
                overlap = fuzzy_piece_overlap(piece)
                if overlap <= 0.0:
                    continue
                piece_candidates.append((-overlap, length, abs((n - j) - (n / 2)), j, piece))
            for _, _, _, j, piece in sorted(piece_candidates)[:piece_probe_limit]:
                match = fuzzy_piece_match(piece)
                if match is None:
                    continue
                match_cost = match["score"] + match["distance"] * 0.01
                next_states.append((
                    fuzzy_count + 1,
                    cost + match_cost,
                    segments + 1,
                    j,
                    output + match["value"],
                ))

        states = prune_states(next_states)
        if not states:
            break

    for fuzzy_count, cost, segments, pos, output in states:
        if pos == n and segments >= min_segments and fuzzy_count > 0:
            finals.append((fuzzy_count, cost, segments, output))

    if not finals:
        return None
    best_by_output = {}
    for item in finals:
        key = item[3]
        score = item[:3]
        if key not in best_by_output or score < best_by_output[key][:3]:
            best_by_output[key] = item
    ranked = sorted(best_by_output.values(), key=lambda item: (item[0], -item[2], round(item[1], 6), len(item[3])))
    if len(ranked) > 1 and ranked[0][:3] == ranked[1][:3] and ranked[0][3] != ranked[1][3]:
        return None
    return ranked[0][3]


def single_fuzzy_long_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    fuzzy_lexicon_lengths: list[int] | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_segments: int = 3,
    max_segments: int = LONG_SEGMENT_MAX_SEGMENTS,
    max_fuzzy_score: float = RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
    plain_accept_score: float | None = None,
    use_compact_weighted: bool = True,
    weighted_candidate_limit: int | None = None,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < WIDE_MULTI_SEGMENT_MIN_LEN:
        return None
    exact_lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    fuzzy_lengths = fuzzy_lexicon_lengths or build_lexicon_lengths(fuzzy_lexicon)
    if not exact_lengths or not fuzzy_lengths:
        return None

    n = len(inp)
    min_fuzzy_len = max(5, min(fuzzy_lengths))
    match_cache = {}

    def shared_prefix_len(a: str, b: str) -> int:
        count = 0
        for ca, cb in zip(a, b):
            if ca != cb:
                break
            count += 1
        return count

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > max_fuzzy_score or match["distance"] > max(5, len(piece) * 0.28):
            return None
        if len(match["value"]) < max(5, int(len(piece) * WIDE_MULTI_SEGMENT_MIN_OUTPUT_RATIO)):
            return None
        return match

    def fuzzy_piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]
        compact_plain = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        full_plain = acceptable(fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index), piece)
        if (
            full_plain is not None
            and full_plain["distance"] <= 2
            and full_plain["score"] <= 0.16
            and shared_prefix_len(piece, full_plain["key"]) >= min(2, len(piece), len(full_plain["key"]))
        ):
            preferred = dict(full_plain)
            preferred["score"] = min(preferred["score"], preferred["distance"] * 0.02)
            match_cache[piece] = preferred
            return preferred
        candidates = [match for match in (compact_plain, full_plain) if match is not None]
        if plain_accept_score is not None and candidates:
            candidates.sort(key=lambda item: (round(item["score"], 6), item["distance"], len(item["key"])))
            if candidates[0]["score"] <= plain_accept_score:
                match_cache[piece] = candidates[0]
                return candidates[0]
        compact_weighted = None
        if use_compact_weighted:
            compact_weighted = acceptable(weighted_fuzzy_lexicon_match(
                piece,
                fuzzy_lexicon,
                fuzzy_lexicon_by_len,
                fuzzy_lexicon_gram_index,
                candidate_limit=weighted_candidate_limit,
            ), piece)
        full_weighted = acceptable(weighted_fuzzy_lexicon_match(
            piece,
            lexicon,
            full_lexicon_by_len,
            full_lexicon_gram_index,
            candidate_limit=weighted_candidate_limit,
        ), piece)
        candidates.extend(match for match in (compact_weighted, full_weighted) if match is not None)
        match = None
        if candidates:
            candidates.sort(key=lambda item: (round(item["score"], 6), item["distance"], len(item["key"])))
            match = candidates[0]
        match_cache[piece] = match
        return match

    @lru_cache(maxsize=None)
    def exact_suffix(pos: int):
        if pos == n:
            return ((0, ""),)
        results = []
        for length in exact_lengths:
            j = pos + length
            if j > n:
                continue
            value = lexicon.get(inp[pos:j])
            if value is None:
                continue
            for rest_segments, rest_output in exact_suffix(j):
                segments = rest_segments + 1
                if segments <= max_segments:
                    results.append((segments, value + rest_output))
        if not results:
            return ()
        best_by_output = {}
        for item in results:
            output = item[1]
            if output not in best_by_output or item[0] > best_by_output[output][0]:
                best_by_output[output] = item
        return tuple(sorted(best_by_output.values(), key=lambda item: (-item[0], len(item[1])))[:8])

    prefix_states = [(0, 0, "")]
    seen_prefix = {(0, 0, "")}
    candidates = []
    for _ in range(max_segments):
        next_prefix = []
        for pos, prefix_segments, prefix_output in prefix_states:
            if pos >= n:
                continue
            for length in boundary_fuzzy_lengths(fuzzy_lengths, exact_lengths, n - pos, min_len=min_fuzzy_len):
                if length < min_fuzzy_len:
                    continue
                j = pos + length
                if j > n:
                    continue
                piece = inp[pos:j]
                if piece in lexicon:
                    continue
                suffixes = exact_suffix(j)
                if not suffixes:
                    continue
                match = fuzzy_piece_match(piece)
                if match is None:
                    continue
                for suffix_segments, suffix_output in suffixes:
                    segments = prefix_segments + 1 + suffix_segments
                    if segments < min_segments or segments > max_segments:
                        continue
                    cost = match["score"] + match["distance"] * 0.01
                    candidates.append((cost, -segments, prefix_output + match["value"] + suffix_output))

            for length in exact_lengths:
                j = pos + length
                if j > n:
                    continue
                value = lexicon.get(inp[pos:j])
                if value is None:
                    continue
                state = (j, prefix_segments + 1, prefix_output + value)
                if state[1] <= max_segments and state not in seen_prefix:
                    seen_prefix.add(state)
                    next_prefix.append(state)
        if candidates:
            break
        prefix_states = sorted(next_prefix, key=lambda item: (-item[0], -item[1], len(item[2])))[:32]
        if not prefix_states:
            break

    if not candidates:
        return None
    best_by_output = {}
    for item in candidates:
        output = item[2]
        if output not in best_by_output or item[:2] < best_by_output[output][:2]:
            best_by_output[output] = item
    ranked = sorted(best_by_output.values())
    if len(ranked) > 1 and ranked[0][:2] == ranked[1][:2] and ranked[0][2] != ranked[1][2]:
        return None
    return ranked[0][2]


def dense_overflow_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_segments: int = DENSE_OVERFLOW_MIN_SEGMENTS,
    max_segments: int = DENSE_OVERFLOW_MAX_SEGMENTS,
    max_fuzzy_segments: int = DENSE_OVERFLOW_MAX_FUZZY_SEGMENTS,
    beam_width: int = DENSE_OVERFLOW_BEAM_WIDTH,
    max_fuzzy_score: float = DENSE_OVERFLOW_MAX_SCORE,
    allow_full_rescue: bool = False,
    max_full_rescue_segments: int = DENSE_OVERFLOW_MAX_FULL_RESCUE_SEGMENTS,
    use_weighted_piece: bool = False,
    weighted_candidate_limit: int = DENSE_OVERFLOW_WEIGHTED_CANDIDATE_LIMIT,
    validate_weighted_with_full: bool = False,
    weighted_validation_score: float = DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_SCORE,
    weighted_validation_margin: float = DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_MARGIN,
    weighted_validation_candidate_limit: int = DENSE_OVERFLOW_FAST_WEIGHTED_VALIDATE_CANDIDATE_LIMIT,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < DENSE_OVERFLOW_MIN_LEN:
        return None
    exact_lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    if not exact_lengths:
        return None

    n = len(inp)
    match_cache = {}
    full_match_cache = {}

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > max_fuzzy_score:
            return None
        if match["distance"] > max(5, len(piece) * DENSE_OVERFLOW_MAX_DISTANCE_RATIO):
            return None
        if len(match["value"]) < max(2, int(len(piece) * WIDE_MULTI_SEGMENT_MIN_OUTPUT_RATIO)):
            return None
        return match

    def fuzzy_piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]
        match = acceptable(
            fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index),
            piece,
        )
        if match is None and use_weighted_piece:
            match = acceptable(
                weighted_fuzzy_lexicon_match(
                    piece,
                    fuzzy_lexicon,
                    fuzzy_lexicon_by_len,
                    fuzzy_lexicon_gram_index,
                    candidate_limit=weighted_candidate_limit,
                ),
                piece,
            )
            if (
                validate_weighted_with_full
                and match is not None
                and match["score"] >= weighted_validation_score
                and full_lexicon_by_len is not None
                and full_lexicon_gram_index is not None
            ):
                full_match = acceptable(
                    weighted_fuzzy_lexicon_match(
                        piece,
                        lexicon,
                        full_lexicon_by_len,
                        full_lexicon_gram_index,
                        candidate_limit=weighted_validation_candidate_limit,
                    ),
                    piece,
                )
                if full_match is not None and full_match["score"] + weighted_validation_margin < match["score"]:
                    match = full_match
        match_cache[piece] = match
        return match

    def full_piece_match(piece: str):
        if piece in full_match_cache:
            return full_match_cache[piece]
        match = acceptable(
            fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index),
            piece,
        )
        if match is None and use_weighted_piece:
            match = acceptable(
                weighted_fuzzy_lexicon_match(
                    piece,
                    lexicon,
                    full_lexicon_by_len,
                    full_lexicon_gram_index,
                    candidate_limit=weighted_candidate_limit,
                ),
                piece,
            )
        full_match_cache[piece] = match
        return match

    states = [(0, 0.0, 0, 0, 0, "")]
    finals = []
    for _ in range(max_segments):
        next_states = []
        for pos, cost, fuzzy_count, full_count, segments, output in states:
            if pos == n:
                finals.append((cost, fuzzy_count, full_count, segments, output))
                continue

            remaining = n - pos
            local_states = []
            full_rescue_pieces = []
            for length in exact_lengths:
                if length > remaining:
                    continue
                j = pos + length
                piece = inp[pos:j]
                value = lexicon.get(piece)
                if value is not None:
                    local_states.append((j, cost, fuzzy_count, full_count, segments + 1, output + value))
                    continue

                if fuzzy_count >= max_fuzzy_segments:
                    continue
                match = fuzzy_piece_match(piece)
                if match is not None:
                    piece_cost = match["score"] + match["distance"] * 0.01 + DENSE_OVERFLOW_FUZZY_COST
                    local_states.append((
                        j,
                        cost + piece_cost,
                        fuzzy_count + 1,
                        full_count,
                        segments + 1,
                        output + match["value"],
                    ))
                elif allow_full_rescue and full_count < max_full_rescue_segments:
                    full_rescue_pieces.append((j, piece))

            if local_states:
                next_states.extend(local_states)
                continue

            if not allow_full_rescue or full_count >= max_full_rescue_segments:
                continue

            for j, piece in full_rescue_pieces:
                match = full_piece_match(piece)
                if match is None:
                    continue
                piece_cost = match["score"] + match["distance"] * 0.01 + DENSE_OVERFLOW_FUZZY_COST
                next_states.append((
                    j,
                    cost + piece_cost + DENSE_OVERFLOW_FULL_RESCUE_COST,
                    fuzzy_count + 1,
                    full_count + 1,
                    segments + 1,
                    output + match["value"],
                ))

        if not next_states:
            break

        best_by_position_output = {}
        for item in next_states:
            key = (item[0], item[5])
            score = (round(item[1], 6), item[2], item[3], item[4])
            if key not in best_by_position_output or score < best_by_position_output[key][0]:
                best_by_position_output[key] = (score, item)
        states = sorted(
            (item for _, item in best_by_position_output.values()),
            key=lambda item: (round(item[1], 6), item[3], item[2], -item[4], -item[0], len(item[5])),
        )[:beam_width]

    for item in states:
        if item[0] == n:
            finals.append((item[1], item[2], item[3], item[4], item[5]))

    finals = [item for item in finals if item[3] >= min_segments]
    if not finals:
        return None
    best_by_output = {}
    for item in finals:
        key = item[4]
        score = item[:4]
        if key not in best_by_output or score < best_by_output[key][:4]:
            best_by_output[key] = item
    ranked = sorted(
        best_by_output.values(),
        key=lambda item: (round(item[0], 6), item[2], item[1], -item[3], len(item[4])),
    )
    if len(ranked) > 1:
        first = (round(ranked[0][0], 6), ranked[0][1], ranked[0][2], ranked[0][3])
        second = (round(ranked[1][0], 6), ranked[1][1], ranked[1][2], ranked[1][3])
        if first == second and ranked[0][4] != ranked[1][4]:
            return None
    return ranked[0][4]


def dense_overflow_short_viterbi_rescue_lookup(
    inp: str,
    lexicon: dict,
    lexicon_lengths: list[int] | None,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    min_segments: int = DENSE_OVERFLOW_MIN_SEGMENTS,
    max_segments: int = DENSE_OVERFLOW_SHORT_VITERBI_MAX_SEGMENTS,
    max_score: float = DENSE_OVERFLOW_SHORT_VITERBI_MAX_SCORE,
    max_distance_ratio: float = DENSE_OVERFLOW_SHORT_VITERBI_MAX_DISTANCE_RATIO,
    weighted_candidate_limit: int = DENSE_OVERFLOW_SHORT_VITERBI_WEIGHTED_CANDIDATE_LIMIT,
    position_beam: int = DENSE_OVERFLOW_SHORT_VITERBI_POSITION_BEAM,
    max_fuzzy_segments: int = DENSE_OVERFLOW_SHORT_VITERBI_MAX_FUZZY_SEGMENTS,
    max_cost_per_segment: float = DENSE_OVERFLOW_SHORT_VITERBI_MAX_COST_PER_SEGMENT,
    max_fuzzy_ratio: float = DENSE_OVERFLOW_SHORT_VITERBI_MAX_FUZZY_RATIO,
    validate_with_full: bool = False,
    full_rerank_score: float = DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_SCORE,
    full_rerank_margin: float = DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MARGIN,
    full_rerank_candidate_limit: int = DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_CANDIDATE_LIMIT,
    full_rerank_min_distance: int = DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MIN_DISTANCE,
    full_rerank_min_len_delta: int = DENSE_OVERFLOW_SHORT_VITERBI_FULL_RERANK_MIN_LEN_DELTA,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < DENSE_OVERFLOW_SHORT_VITERBI_MIN_LEN:
        return None
    exact_lengths = lexicon_lengths or build_lexicon_lengths(lexicon)
    lengths = [
        length for length in exact_lengths
        if DENSE_OVERFLOW_SHORT_VITERBI_MIN_PIECE_LEN <= length <= DENSE_OVERFLOW_SHORT_VITERBI_MAX_PIECE_LEN
    ]
    if not lengths:
        return None

    n = len(inp)
    match_cache = {}

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > max_score:
            return None
        if match["distance"] > max(5, len(piece) * max_distance_ratio):
            return None
        if len(match["value"]) < max(2, int(len(piece) * WIDE_MULTI_SEGMENT_MIN_OUTPUT_RATIO)):
            return None
        return match

    def fuzzy_piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]
        match = acceptable(
            fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index),
            piece,
        )
        if match is None:
            match = acceptable(
                weighted_fuzzy_lexicon_match(
                    piece,
                    fuzzy_lexicon,
                    fuzzy_lexicon_by_len,
                    fuzzy_lexicon_gram_index,
                    candidate_limit=weighted_candidate_limit,
                ),
                piece,
            )
        if (
            validate_with_full
            and match is not None
            and match["score"] >= full_rerank_score
            and (
                match["distance"] >= full_rerank_min_distance
                or abs(len(piece) - len(match["key"])) >= full_rerank_min_len_delta
            )
            and full_lexicon_by_len is not None
            and full_lexicon_gram_index is not None
        ):
            full_match = acceptable(
                weighted_fuzzy_lexicon_match(
                    piece,
                    lexicon,
                    full_lexicon_by_len,
                    full_lexicon_gram_index,
                    candidate_limit=full_rerank_candidate_limit,
                ),
                piece,
            )
            if full_match is not None and full_match["score"] + full_rerank_margin < match["score"]:
                match = full_match
        match_cache[piece] = match
        return match

    def state_score(state):
        return (round(state[0], 6), state[1], -state[2], len(state[3]))

    def add_state(position: int, candidate):
        bucket = best_by_position.setdefault(position, [])
        candidate_key = candidate[3]
        candidate_score = state_score(candidate)
        kept = []
        for existing in bucket:
            if existing[3] == candidate_key:
                if state_score(existing) <= candidate_score:
                    return
                continue
            kept.append(existing)
        kept.append(candidate)
        kept.sort(key=state_score)
        best_by_position[position] = kept[:position_beam]

    # Keep a tiny per-position beam. A single state is fast, but very long
    # typo-heavy concatenations can temporarily prefer a locally cheaper
    # ambiguous phrase and prune the globally correct path.
    best_by_position = {0: [(0.0, 0, 0, "")]}
    for pos in range(n + 1):
        states = best_by_position.get(pos)
        if states is None:
            continue
        for state in states:
            cost, fuzzy_count, segments, output = state
            if segments >= max_segments:
                continue
            remaining = n - pos
            for length in lengths:
                if length > remaining:
                    continue
                j = pos + length
                piece = inp[pos:j]
                value = lexicon.get(piece)
                next_cost = cost
                next_fuzzy_count = fuzzy_count
                if value is None:
                    if fuzzy_count >= max_fuzzy_segments:
                        continue
                    match = fuzzy_piece_match(piece)
                    if match is None:
                        continue
                    value = match["value"]
                    next_fuzzy_count += 1
                    next_cost += match["score"] + match["distance"] * 0.01 + DENSE_OVERFLOW_FUZZY_COST
                next_segments = segments + 1
                add_state(j, (next_cost, next_fuzzy_count, next_segments, output + value))

    finals = best_by_position.get(n)
    if not finals:
        return None
    valid_finals = []
    for final in finals:
        cost, fuzzy_count, segments, output = final
        if segments < max(min_segments, DENSE_OVERFLOW_SHORT_VITERBI_MIN_SEGMENTS):
            continue
        if cost > segments * max_cost_per_segment:
            continue
        if fuzzy_count > max(1, int(segments * max_fuzzy_ratio)):
            continue
        valid_finals.append(final)
    if not valid_finals:
        return None
    return sorted(valid_finals, key=state_score)[0][3]


def fuzzy_triple_segment_lexicon_lookup(
    inp: str,
    lexicon: dict,
    fuzzy_lexicon: dict,
    fuzzy_lexicon_by_len: dict | None = None,
    fuzzy_lexicon_gram_index: dict | None = None,
    fuzzy_lexicon_lengths: list[int] | None = None,
    full_lexicon_by_len: dict | None = None,
    full_lexicon_gram_index: dict | None = None,
    *,
    max_delta: int = 7,
    max_split_pairs: int = FUZZY_TRIPLE_MAX_SPLITS,
):
    if not lexicon or not fuzzy_lexicon or len(inp) < 36:
        return None

    fuzzy_lengths = fuzzy_lexicon_lengths or build_lexicon_lengths(fuzzy_lexicon)
    if not fuzzy_lengths:
        return None

    n = len(inp)
    min_len = max(5, min(fuzzy_lengths))
    max_len = max(fuzzy_lengths)
    split_positions = set()
    for length in fuzzy_lengths:
        for delta in range(-max_delta, max_delta + 1):
            split = length + delta
            if min_len <= split <= n - min_len:
                split_positions.add(split)

    overlap_cache = {}

    def overlap_score(piece: str) -> float:
        if piece in lexicon:
            return 10.0
        if piece in overlap_cache:
            return overlap_cache[piece]
        if fuzzy_lexicon_gram_index is None:
            overlap_cache[piece] = 0.0
            return 0.0
        grams = char_grams(piece)
        if not grams:
            overlap_cache[piece] = 0.0
            return 0.0
        counts = Counter()
        allowed = set(range(len(piece) - max_delta, len(piece) + max_delta + 1))
        for gram in grams:
            for key, _ in fuzzy_lexicon_gram_index.get(gram, ()):
                if len(key) in allowed:
                    counts[key] += 1
        score = counts.most_common(1)[0][1] / max(1, len(grams)) if counts else 0.0
        overlap_cache[piece] = score
        return score

    split_pairs = []
    for first in split_positions:
        for second in split_positions:
            if second <= first:
                continue
            lengths = (first, second - first, n - second)
            if any(length < min_len - max_delta or length > max_len + max_delta for length in lengths):
                continue
            pieces = (inp[:first], inp[first:second], inp[second:])
            score = sum(overlap_score(piece) for piece in pieces)
            if score <= 0:
                continue
            balance = sum(abs(length - n / 3) for length in lengths)
            split_pairs.append((-score, balance, first, second))

    if not split_pairs:
        return None

    match_cache = {}

    def acceptable(match, piece: str):
        if match is None:
            return None
        if match["score"] > MULTI_SEGMENT_FUZZY_MAX_SCORE or match["distance"] > max(5, len(piece) * 0.28):
            return None
        return match

    def piece_match(piece: str):
        if piece in match_cache:
            return match_cache[piece]
        value = lexicon.get(piece)
        if value is not None:
            match_cache[piece] = {"key": piece, "value": value, "distance": 0, "score": 0.0, "exact": True}
            return match_cache[piece]
        match = acceptable(fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(weighted_fuzzy_lexicon_match(piece, fuzzy_lexicon, fuzzy_lexicon_by_len, fuzzy_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index), piece)
        if match is None:
            match = acceptable(weighted_fuzzy_lexicon_match(piece, lexicon, full_lexicon_by_len, full_lexicon_gram_index), piece)
        if match is not None:
            match = {**match, "exact": False}
        match_cache[piece] = match
        return match

    best_score = None
    best_outputs = set()
    for _, balance, first, second in sorted(split_pairs)[:max_split_pairs]:
        pieces = (inp[:first], inp[first:second], inp[second:])
        matches = [piece_match(piece) for piece in pieces]
        if any(match is None for match in matches):
            continue
        fuzzy_count = sum(1 for match in matches if not match["exact"])
        if fuzzy_count == 0:
            continue
        distance = sum(match["distance"] for match in matches)
        score = sum(match["score"] for match in matches)
        cand_score = (fuzzy_count, distance, round(score, 6), balance)
        output = "".join(match["value"] for match in matches)
        if best_score is None or cand_score < best_score:
            best_score = cand_score
            best_outputs = {output}
        elif cand_score == best_score:
            best_outputs.add(output)
            if len(best_outputs) > 1:
                best_outputs = set(list(best_outputs)[:2])

    if best_score is None or len(best_outputs) != 1:
        return None
    return next(iter(best_outputs))


def model_key(model: str) -> str:
    p = Path(model)
    if not p.exists():
        return hashlib.sha256(model.encode("utf-8")).hexdigest()[:16]
    parts = [str(p.resolve())]
    for name in ("config.json", "model.safetensors", "pytorch_model.bin"):
        f = p / name
        if f.exists():
            st = f.stat()
            parts.append(f"{name}:{st.st_size}:{int(st.st_mtime)}")
    return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()[:16]


def runtime_key(
    model: str,
    lexicon_path: str | None,
    choice_feedback_path: str | None = None,
    general_lexicon_path: str | None = None,
    extra: str = "",
) -> str:
    parts = [model_key(model), FAST_PATH_VERSION, NORMALIZATION_VERSION, GENERIC_FALLBACK_VERSION]
    parts.append(GENERAL_PHRASE_VERSION)
    parts.append(file_fingerprint("lexicon", lexicon_path))
    parts.append(file_fingerprint("aux_lexicon", DEFAULT_AUX_LEXICON))
    parts.append(file_fingerprint("choice_feedback", choice_feedback_path))
    parts.append(file_fingerprint("general_lexicon", general_lexicon_path))
    if extra:
        parts.append(extra)
    return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()[:20]


def cache_connect(path: str | None):
    if not path:
        return None
    p = Path(path)
    p.parent.mkdir(parents=True, exist_ok=True)
    con = sqlite3.connect(str(p), timeout=30.0)
    con.execute("PRAGMA journal_mode=WAL")
    con.execute("PRAGMA synchronous=NORMAL")
    con.execute("PRAGMA cache_size=-64000")
    con.execute("PRAGMA temp_store=MEMORY")
    con.execute("PRAGMA mmap_size=268435456")
    con.execute("PRAGMA busy_timeout=5000")
    con.execute(
        "CREATE TABLE IF NOT EXISTS infer_cache ("
        "model_key TEXT NOT NULL, input TEXT NOT NULL, output TEXT NOT NULL, "
        "created_at INTEGER NOT NULL, PRIMARY KEY(model_key, input))"
    )
    return con


def cache_get(con, key: str, inp: str):
    if con is None:
        return None
    row = con.execute(
        "SELECT output FROM infer_cache WHERE model_key=? AND input=?",
        (key, inp),
    ).fetchone()
    return row[0] if row else None


def cache_put(con, key: str, inp: str, out: str):
    if con is None:
        return
    con.execute(
        "INSERT OR REPLACE INTO infer_cache(model_key,input,output,created_at) "
        "VALUES(?,?,?,strftime('%s','now'))",
        (key, inp, out),
    )
    con.commit()


def resolve_device(name: str):
    import torch

    if name == "auto":
        if torch.cuda.is_available():
            return "cuda"
        try:
            import torch_directml
            return torch_directml.device()
        except Exception:
            return "cpu"
    if name == "dml":
        import torch_directml
        return torch_directml.device()
    return name


def load_model(model_path: str, device):
    import torch
    from transformers import AutoModelForCausalLM, AutoTokenizer

    use_cuda = str(device) == "cuda"
    tok = AutoTokenizer.from_pretrained(model_path)
    model = AutoModelForCausalLM.from_pretrained(
        model_path,
        dtype=torch.bfloat16 if use_cuda else torch.float32,
        attn_implementation="eager",
    ).to(device).eval()
    return model, tok


def generate(model, tok, inp: str, device):
    import torch

    prompt = BOS_IN + inp + BOS_OUT
    enc = tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
    max_new = min(max(16, int(len(inp) * 0.8) + 8), 96)
    with torch.no_grad():
        out = model.generate(
            enc.input_ids,
            attention_mask=enc.attention_mask,
            max_new_tokens=max_new,
            do_sample=False,
            use_cache=True,
            eos_token_id=tok.eos_token_id,
            pad_token_id=tok.pad_token_id,
        )
    return tok.decode(out[0][enc.input_ids.shape[1]:], skip_special_tokens=True)


class FastConverter:
    def __init__(
        self,
        model_path,
        device_name="auto",
        lexicon_path=None,
        cache_path=None,
        fuzzy=True,
        segment=True,
        choice_feedback_path: str | None = DEFAULT_CANDIDATE_FEEDBACK,
        general_lexicon_path: str | None = DEFAULT_GENERAL_LEXICON,
        general_phrase: bool = True,
        general_phrase_aggressive: bool = False,
    ):
        self.model_path = model_path
        self.device_name = device_name
        self.lexicon_path = resolve_lexicon_path(lexicon_path)
        self.choice_feedback_path = choice_feedback_path
        self.general_lexicon_path = general_lexicon_path
        self.choice_feedback = load_choice_feedback(choice_feedback_path)
        self.general_lexicon = load_general_lexicon(general_lexicon_path)
        self.generic_phrase_prepared = prepare_generic_lexicon({})
        self.generic_prepared = prepare_generic_lexicon(self.general_lexicon)
        self.lexicon = load_lexicon(self.lexicon_path)
        self.lexicon_by_len = build_lexicon_index(self.lexicon)
        self.lexicon_lengths = build_lexicon_lengths(self.lexicon)
        self.lexicon_gram_index = build_lexicon_gram_index(self.lexicon)
        self.fuzzy_lexicon = build_compact_fuzzy_lexicon(self.lexicon)
        self.fuzzy_lexicon_by_len = build_lexicon_index(self.fuzzy_lexicon)
        self.fuzzy_lexicon_lengths = build_lexicon_lengths(self.fuzzy_lexicon)
        self.fuzzy_lexicon_gram_index = build_lexicon_gram_index(self.fuzzy_lexicon)
        self.dense_fuzzy_lexicon = build_dense_compact_fuzzy_lexicon(self.lexicon)
        self.dense_fuzzy_lexicon_by_len = build_lexicon_index(self.dense_fuzzy_lexicon)
        self.dense_fuzzy_lexicon_gram_index = build_lexicon_gram_index(self.dense_fuzzy_lexicon)
        self.short_viterbi_fuzzy_lexicon = build_dense_compact_fuzzy_lexicon(
            self.lexicon,
            keep_per_output_length=SHORT_VITERBI_COMPACT_FUZZY_KEEP_PER_OUTPUT_LENGTH,
        )
        self.short_viterbi_fuzzy_lexicon_by_len = build_lexicon_index(self.short_viterbi_fuzzy_lexicon)
        self.short_viterbi_fuzzy_lexicon_gram_index = build_lexicon_gram_index(self.short_viterbi_fuzzy_lexicon)
        self.general_phrase = general_phrase
        self.general_phrase_aggressive = general_phrase_aggressive
        self._general_phrase_index = None
        self.cache = cache_connect(cache_path)
        self.key = runtime_key(
            model_path,
            self.lexicon_path,
            choice_feedback_path,
            general_lexicon_path,
            extra=f"gp:{int(general_phrase)}:{int(general_phrase_aggressive)}",
        )
        self.fuzzy = fuzzy
        self.segment = segment
        self.device = None
        self.model = None
        self.tok = None

    def _ensure_general_phrase_index(self):
        if self._general_phrase_index is None:
            self._general_phrase_index = build_general_phrase_index(self.general_lexicon)
        return self._general_phrase_index

    def _try_general_phrase_rescue(self, inp, *, exact_only: bool = False):
        """Generic noisy-romaji rescue: canonical re-lookup (Alt A) then
        general-lexicon fuzzy Viterbi anchor-and-fill (recommended). Returns
        (output, source) or None. Runs only after earlier routes decline."""
        if not self.general_phrase:
            return None
        # Alt A: canonical-variant re-lookup through high-confidence routes.
        for variant in canonicalize_romaji_variants(inp):
            if variant == inp:
                continue
            exact = self.lexicon.get(variant)
            if exact is not None:
                return exact, "canonical_exact"
            if self.segment:
                seg = segment_lexicon_lookup(variant, self.lexicon, self.lexicon_lengths)
                if seg is not None:
                    return seg, "canonical_segment"
        # Recommended: general-lexicon fuzzy Viterbi anchor-and-fill.
        index = self._ensure_general_phrase_index()
        hit = general_phrase_rescue(
            inp,
            index,
            aggressive=self.general_phrase_aggressive,
            exact_only=exact_only,
        )
        if hit is not None:
            return hit[0], "general_phrase_viterbi"
        return None

    def ensure_model(self):
        if self.model is None:
            self.device = resolve_device(self.device_name)
            self.model, self.tok = load_model(self.model_path, self.device)

    def convert(self, text):
        inp = normalize_input(text)
        t0 = time.perf_counter()
        choice_hit = self.choice_feedback.get(inp)
        if choice_hit is not None:
            cache_put(self.cache, self.key, inp, choice_hit)
            return choice_hit, "choice_feedback", (time.perf_counter() - t0) * 1000

        if inp in self.lexicon:
            return self.lexicon[inp], "lexicon", (time.perf_counter() - t0) * 1000

        cached = cache_get(self.cache, self.key, inp)
        if cached is not None:
            return cached, "cache", (time.perf_counter() - t0) * 1000

        if self.segment:
            segment_hit = segment_lexicon_lookup(inp, self.lexicon, self.lexicon_lengths)
            if segment_hit is not None:
                cache_put(self.cache, self.key, inp, segment_hit)
                return segment_hit, "lexicon_segment", (time.perf_counter() - t0) * 1000
            if len(inp) >= LONG_SEGMENT_MIN_LEN:
                segment_hit = segment_lexicon_lookup(
                    inp,
                    self.lexicon,
                    self.lexicon_lengths,
                    max_segments=LONG_SEGMENT_MAX_SEGMENTS,
                )
                if segment_hit is not None:
                    cache_put(self.cache, self.key, inp, segment_hit)
                    return segment_hit, "lexicon_segment_long", (time.perf_counter() - t0) * 1000

        generic_hit = generic_romaji_fallback(inp, prepared=self.generic_phrase_prepared, min_coverage=0.45)
        if generic_hit is not None:
            cache_put(self.cache, self.key, inp, generic_hit)
            return generic_hit, "romaji_kana_fallback_early", (time.perf_counter() - t0) * 1000
        has_known_piece = self.segment and has_exact_subpiece(inp, self.lexicon, self.lexicon_lengths)
        if len(inp) >= 32 and not any(ch.isdigit() for ch in inp) and not has_known_piece:
            generic_hit = generic_romaji_fallback(inp, prepared=self.generic_prepared, min_coverage=0.75)
            if generic_hit is not None:
                cache_put(self.cache, self.key, inp, generic_hit)
                return generic_hit, "romaji_kana_general_fallback_early", (time.perf_counter() - t0) * 1000

        # Cheap default prefilter: if deterministic romaji canonicalization
        # actually changed the input, try only the exact/no-fill general phrase
        # lattice before the heavy fuzzy beams. Full fuzzy general_phrase remains
        # post-fallback (or opt-in aggressive), so ambiguous cases still abstain.
        if (
            self.general_phrase
            and not self.general_phrase_aggressive
            and len(inp) >= 24
            and not any(ch.isdigit() for ch in inp)
            and any(v != inp for v in canonicalize_romaji_variants(inp))
        ):
            rescue = self._try_general_phrase_rescue(inp, exact_only=True)
            if rescue is not None:
                rescue_out, rescue_src = rescue
                cache_put(self.cache, self.key, inp, rescue_out)
                return rescue_out, f"{rescue_src}_prefuzzy_exact", (time.perf_counter() - t0) * 1000

        # Alt B (explicit): let the full general-lexicon phrase route
        # participate earlier, before the heavy fuzzy beams, so colloquial
        # phrases can short-circuit. This changes ordering, hence opt-in.
        if self.general_phrase_aggressive:
            rescue = self._try_general_phrase_rescue(inp)
            if rescue is not None:
                rescue_out, rescue_src = rescue
                cache_put(self.cache, self.key, inp, rescue_out)
                return rescue_out, f"{rescue_src}_aggressive", (time.perf_counter() - t0) * 1000

        if self.fuzzy:
            fuzzy_hit = fuzzy_lexicon_lookup(inp, self.lexicon, self.lexicon_by_len, self.lexicon_gram_index)
            if fuzzy_hit is not None:
                cache_put(self.cache, self.key, inp, fuzzy_hit)
                return fuzzy_hit, "lexicon_fuzzy", (time.perf_counter() - t0) * 1000

            if self.segment:
                def try_fuzzy_multi_segment(**kwargs):
                    return fuzzy_multi_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                        self.fuzzy_lexicon_lengths,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        **kwargs,
                    )

                def try_fuzzy_triple_segment():
                    return fuzzy_triple_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                        self.fuzzy_lexicon_lengths,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                    )

                def try_single_fuzzy_long_segment():
                    return single_fuzzy_long_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                        self.fuzzy_lexicon_lengths,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        max_segments=LONG_SEGMENT_MAX_SEGMENTS,
                        max_fuzzy_score=RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
                        plain_accept_score=LONG_SINGLE_FUZZY_PLAIN_ACCEPT_SCORE,
                        use_compact_weighted=False,
                        weighted_candidate_limit=LONG_SINGLE_FUZZY_WEIGHTED_CANDIDATE_LIMIT,
                    )

                tried_fuzzy_multi = False
                tried_fuzzy_triple = False
                fuzzy_multi_hit = None
                exact_subpiece = has_known_piece
                prefer_triple = len(inp) >= 40 and not exact_subpiece
                if prefer_triple:
                    fuzzy_triple_hit = try_fuzzy_triple_segment()
                    tried_fuzzy_triple = True
                    if fuzzy_triple_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_triple_hit)
                        return fuzzy_triple_hit, "lexicon_triple_segment_fuzzy", (time.perf_counter() - t0) * 1000

                if len(inp) >= 40 and exact_subpiece:
                    anchored_hit = anchored_fuzzy_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                    )
                    if anchored_hit is not None:
                        cache_put(self.cache, self.key, inp, anchored_hit)
                        return anchored_hit, "lexicon_anchor_fuzzy", (time.perf_counter() - t0) * 1000

                    sandwich_hit = sandwich_fuzzy_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                    )
                    if sandwich_hit is not None:
                        cache_put(self.cache, self.key, inp, sandwich_hit)
                        return sandwich_hit, "lexicon_sandwich_fuzzy", (time.perf_counter() - t0) * 1000

                if len(inp) >= DENSE_OVERFLOW_DIRECT_MIN_LEN:
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        allow_full_rescue=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_FAST_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        use_weighted_piece=True,
                        weighted_candidate_limit=DENSE_OVERFLOW_FAST_WEIGHTED_CANDIDATE_LIMIT,
                        validate_weighted_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_fast_weighted_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_FAST_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RELAXED_WEIGHTED_RESCUE_MAX_SCORE,
                        use_weighted_piece=True,
                        weighted_candidate_limit=DENSE_OVERFLOW_RELAXED_WEIGHTED_CANDIDATE_LIMIT,
                        validate_weighted_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_relaxed_weighted_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_short_viterbi_rescue_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.short_viterbi_fuzzy_lexicon,
                        self.short_viterbi_fuzzy_lexicon_by_len,
                        self.short_viterbi_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        validate_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_short_viterbi_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_short_viterbi_rescue_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.short_viterbi_fuzzy_lexicon,
                        self.short_viterbi_fuzzy_lexicon_by_len,
                        self.short_viterbi_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        max_score=DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_MAX_SCORE,
                        weighted_candidate_limit=DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_WEIGHTED_CANDIDATE_LIMIT,
                        validate_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_short_viterbi_ultra_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        allow_full_rescue=True,
                        use_weighted_piece=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_weighted_rescue", (time.perf_counter() - t0) * 1000

                if len(inp) >= WIDE_MULTI_SEGMENT_MIN_LEN:
                    single_fuzzy_hit = try_single_fuzzy_long_segment()
                    if single_fuzzy_hit is not None:
                        cache_put(self.cache, self.key, inp, single_fuzzy_hit)
                        return single_fuzzy_hit, "lexicon_single_fuzzy_long_segment", (time.perf_counter() - t0) * 1000

                if DENSE_OVERFLOW_MIN_LEN <= len(inp) < DENSE_OVERFLOW_DIRECT_MIN_LEN:
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        allow_full_rescue=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_FAST_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        use_weighted_piece=True,
                        weighted_candidate_limit=DENSE_OVERFLOW_FAST_WEIGHTED_CANDIDATE_LIMIT,
                        validate_weighted_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_fast_weighted_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_FAST_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RELAXED_WEIGHTED_RESCUE_MAX_SCORE,
                        use_weighted_piece=True,
                        weighted_candidate_limit=DENSE_OVERFLOW_RELAXED_WEIGHTED_CANDIDATE_LIMIT,
                        validate_weighted_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_relaxed_weighted_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.dense_fuzzy_lexicon,
                        self.dense_fuzzy_lexicon_by_len,
                        self.dense_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        beam_width=DENSE_OVERFLOW_WEIGHTED_RESCUE_BEAM_WIDTH,
                        max_fuzzy_score=DENSE_OVERFLOW_RESCUE_MAX_SCORE,
                        allow_full_rescue=True,
                        use_weighted_piece=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_weighted_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_short_viterbi_rescue_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.short_viterbi_fuzzy_lexicon,
                        self.short_viterbi_fuzzy_lexicon_by_len,
                        self.short_viterbi_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        validate_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_short_viterbi_rescue", (time.perf_counter() - t0) * 1000
                    dense_overflow_hit = dense_overflow_short_viterbi_rescue_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.short_viterbi_fuzzy_lexicon,
                        self.short_viterbi_fuzzy_lexicon_by_len,
                        self.short_viterbi_fuzzy_lexicon_gram_index,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        max_score=DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_MAX_SCORE,
                        weighted_candidate_limit=DENSE_OVERFLOW_SHORT_VITERBI_ULTRA_WEIGHTED_CANDIDATE_LIMIT,
                        validate_with_full=True,
                    )
                    if dense_overflow_hit is not None:
                        cache_put(self.cache, self.key, inp, dense_overflow_hit)
                        return dense_overflow_hit, "lexicon_dense_overflow_short_viterbi_ultra_rescue", (time.perf_counter() - t0) * 1000

                if len(inp) >= 40:
                    fuzzy_multi_hit = try_fuzzy_multi_segment()
                    tried_fuzzy_multi = True
                if fuzzy_multi_hit is not None:
                    cache_put(self.cache, self.key, inp, fuzzy_multi_hit)
                    return fuzzy_multi_hit, "lexicon_multi_segment_fuzzy", (time.perf_counter() - t0) * 1000

                if len(inp) >= DEEP_MULTI_SEGMENT_MIN_LEN:
                    fuzzy_multi_hit = try_fuzzy_multi_segment(
                        max_fuzzy_segments=3,
                        max_fuzzy_transitions=4,
                        beam_width=8,
                    )
                    if fuzzy_multi_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_multi_hit)
                        return fuzzy_multi_hit, "lexicon_multi_segment_fuzzy_deep", (time.perf_counter() - t0) * 1000

                if len(inp) >= DEEP_MULTI_SEGMENT_MIN_LEN:
                    fuzzy_multi_hit = try_fuzzy_multi_segment(
                        max_fuzzy_segments=3,
                        max_fuzzy_transitions=6,
                        beam_width=12,
                        max_fuzzy_score=RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
                    )
                    if fuzzy_multi_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_multi_hit)
                        return fuzzy_multi_hit, "lexicon_multi_segment_fuzzy_relaxed", (time.perf_counter() - t0) * 1000

                if len(inp) >= WIDE_MULTI_SEGMENT_MIN_LEN:
                    wide_beam_hit = wide_beam_multi_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_lengths,
                        self.fuzzy_lexicon,
                        self.fuzzy_lexicon_by_len,
                        self.fuzzy_lexicon_gram_index,
                        self.fuzzy_lexicon_lengths,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        max_segments=LONG_SEGMENT_MAX_SEGMENTS,
                        max_fuzzy_segments=WIDE_MULTI_SEGMENT_MAX_FUZZY_SEGMENTS,
                        beam_width=WIDE_MULTI_SEGMENT_BEAM_WIDTH,
                        piece_probe_limit=WIDE_MULTI_SEGMENT_PIECE_PROBE_LIMIT,
                        max_fuzzy_score=RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
                        plain_accept_score=WIDE_MULTI_SEGMENT_WIDE_PLAIN_ACCEPT_SCORE,
                        use_compact_weighted=False,
                        weighted_candidate_limit=WIDE_MULTI_SEGMENT_WEIGHTED_CANDIDATE_LIMIT,
                    )
                    if wide_beam_hit is not None:
                        cache_put(self.cache, self.key, inp, wide_beam_hit)
                        return wide_beam_hit, "lexicon_multi_segment_fuzzy_wide_beam", (time.perf_counter() - t0) * 1000
                    if len(inp) >= WIDE_MULTI_SEGMENT_RESCUE_MIN_LEN:
                        wide_beam_hit = wide_beam_multi_segment_lexicon_lookup(
                            inp,
                            self.lexicon,
                            self.lexicon_lengths,
                            self.fuzzy_lexicon,
                            self.fuzzy_lexicon_by_len,
                            self.fuzzy_lexicon_gram_index,
                            self.fuzzy_lexicon_lengths,
                            self.lexicon_by_len,
                            self.lexicon_gram_index,
                            max_segments=LONG_SEGMENT_MAX_SEGMENTS,
                            max_fuzzy_segments=WIDE_MULTI_SEGMENT_RESCUE_MAX_FUZZY_SEGMENTS,
                            beam_width=WIDE_MULTI_SEGMENT_RESCUE_BEAM_WIDTH,
                            piece_probe_limit=WIDE_MULTI_SEGMENT_RESCUE_PIECE_PROBE_LIMIT,
                            max_fuzzy_score=RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
                            plain_accept_score=WIDE_MULTI_SEGMENT_RESCUE_PLAIN_ACCEPT_SCORE,
                            use_compact_weighted=False,
                            weighted_candidate_limit=WIDE_MULTI_SEGMENT_RESCUE_WEIGHTED_CANDIDATE_LIMIT,
                        )
                        if wide_beam_hit is not None:
                            cache_put(self.cache, self.key, inp, wide_beam_hit)
                            return wide_beam_hit, "lexicon_multi_segment_fuzzy_wide_rescue", (time.perf_counter() - t0) * 1000

                if len(inp) >= EXTENDED_MULTI_SEGMENT_MIN_LEN:
                    fuzzy_multi_hit = try_fuzzy_multi_segment(
                        max_segments=LONG_SEGMENT_MAX_SEGMENTS,
                        max_fuzzy_segments=4,
                        max_fuzzy_transitions=6,
                        beam_width=12,
                        max_fuzzy_score=RELAXED_MULTI_SEGMENT_FUZZY_MAX_SCORE,
                    )
                    if fuzzy_multi_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_multi_hit)
                        return fuzzy_multi_hit, "lexicon_multi_segment_fuzzy_extended", (time.perf_counter() - t0) * 1000

                if len(inp) >= 40 and not tried_fuzzy_triple and not exact_subpiece:
                    fuzzy_triple_hit = try_fuzzy_triple_segment()
                    tried_fuzzy_triple = True
                    if fuzzy_triple_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_triple_hit)
                        return fuzzy_triple_hit, "lexicon_triple_segment_fuzzy", (time.perf_counter() - t0) * 1000

                fuzzy_segment_hit = fuzzy_segment_lexicon_lookup(
                    inp,
                    self.fuzzy_lexicon,
                    self.fuzzy_lexicon_by_len,
                    self.fuzzy_lexicon_gram_index,
                    self.fuzzy_lexicon_lengths,
                )
                if fuzzy_segment_hit is None:
                    fuzzy_segment_hit = fuzzy_segment_lexicon_lookup(
                        inp,
                        self.lexicon,
                        self.lexicon_by_len,
                        self.lexicon_gram_index,
                        self.lexicon_lengths,
                    )

                if fuzzy_segment_hit is not None:
                    cache_put(self.cache, self.key, inp, fuzzy_segment_hit)
                    return fuzzy_segment_hit, "lexicon_segment_fuzzy", (time.perf_counter() - t0) * 1000

                if len(inp) >= 40 and not tried_fuzzy_triple:
                    fuzzy_triple_hit = try_fuzzy_triple_segment()
                    tried_fuzzy_triple = True
                    if fuzzy_triple_hit is not None:
                        cache_put(self.cache, self.key, inp, fuzzy_triple_hit)
                        return fuzzy_triple_hit, "lexicon_triple_segment_fuzzy", (time.perf_counter() - t0) * 1000

                if not tried_fuzzy_multi:
                    fuzzy_multi_hit = try_fuzzy_multi_segment()
                if fuzzy_multi_hit is not None:
                    cache_put(self.cache, self.key, inp, fuzzy_multi_hit)
                    return fuzzy_multi_hit, "lexicon_multi_segment_fuzzy", (time.perf_counter() - t0) * 1000

        generic_hit = generic_romaji_fallback(inp, prepared=self.generic_prepared)
        if generic_hit is not None:
            cache_put(self.cache, self.key, inp, generic_hit)
            return generic_hit, "romaji_kana_fallback", (time.perf_counter() - t0) * 1000

        # Generic noisy-romaji rescue: only fires here, after every earlier
        # route declined and before the neural model. On the acceptance gates
        # (model count 0) this stage is never reached, so it cannot change a
        # passing gate row; it only converts neural-fallback cases.
        rescue = self._try_general_phrase_rescue(inp)
        if rescue is not None:
            rescue_out, rescue_src = rescue
            cache_put(self.cache, self.key, inp, rescue_out)
            return rescue_out, rescue_src, (time.perf_counter() - t0) * 1000

        self.ensure_model()
        out = generate(self.model, self.tok, inp, self.device)
        cache_put(self.cache, self.key, inp, out)
        return out, f"model:{self.device}", (time.perf_counter() - t0) * 1000


def convert(
    text,
    model_path,
    device_name="auto",
    lexicon_path=None,
    cache_path=None,
    fuzzy=True,
    segment=True,
    choice_feedback_path: str | None = DEFAULT_CANDIDATE_FEEDBACK,
    general_lexicon_path: str | None = DEFAULT_GENERAL_LEXICON,
    general_phrase: bool = True,
    general_phrase_aggressive: bool = False,
):
    return FastConverter(
        model_path,
        device_name,
        lexicon_path,
        cache_path,
        fuzzy,
        segment,
        choice_feedback_path,
        general_lexicon_path,
        general_phrase,
        general_phrase_aggressive,
    ).convert(text)


def main():
    if hasattr(sys.stdout, "reconfigure"):
        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", required=True)
    ap.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda", "dml"])
    ap.add_argument(
        "--lexicon",
        default="auto",
        help=(
            "Path to lexicon JSON. 'auto' prefers romaji2ja_typo_95.json, "
            "then romaji2ja_feedback_95.json, then romaji2ja.json."
        ),
    )
    ap.add_argument("--cache", default="artifacts/cache/infer_cache.sqlite")
    ap.add_argument("--choice-feedback", default=DEFAULT_CANDIDATE_FEEDBACK)
    ap.add_argument("--general-lexicon", default=DEFAULT_GENERAL_LEXICON)
    ap.add_argument("--no-choice-feedback", action="store_true")
    ap.add_argument("--no-segment", action="store_true")
    ap.add_argument("--no-fuzzy", action="store_true")
    ap.add_argument("--no-general-phrase", action="store_true")
    ap.add_argument("--general-phrase-aggressive", action="store_true")
    ap.add_argument("--json", action="store_true")
    ap.add_argument("text")
    args = ap.parse_args()

    out, source, ms = convert(
        args.text,
        args.model,
        args.device,
        args.lexicon,
        args.cache,
        not args.no_fuzzy,
        not args.no_segment,
        None if args.no_choice_feedback else args.choice_feedback,
        args.general_lexicon,
        not args.no_general_phrase,
        args.general_phrase_aggressive,
    )
    if args.json:
        print(json.dumps({"output": out, "source": source, "latency_ms": round(ms, 2)}, ensure_ascii=False))
    else:
        print(out)
        print(f"({source}, {ms:.0f} ms)")


if __name__ == "__main__":
    main()