clairic-flash-4bit / clairic_runtime /ordinary_list_completion.py
lucataco's picture
Clairic Flash, MLX 4-bit, with runtime, server and conversion script
ae97c5d verified
Raw History Blame Contribute Delete
5.66 kB
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from . import batch_completion as shared
from . import schema
if TYPE_CHECKING:
from collections.abc import Callable
from transformers import PreTrainedTokenizerBase
def eligible(
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
kwargs: dict[str, Any],
) -> bool:
media = kwargs.get("image_data")
object_lists = any(
(
spec.get("type") == "list"
and isinstance(spec.get("items"), dict)
and (spec["items"].get("type") == "object")
for spec in record.get("fields", {}).values()
)
)
guarded = (
not record.get("_null_recovery")
and (not object_lists)
and (
record.get("_meta", {}).get("construction") == schema.EVIDENCE_CONSTRUCTION
)
and (not record.get("_media_payload"))
and (not record.get("_original_images"))
and all((payload is None for (_, _, payload) in tasks))
and isinstance(media, list)
and (len(media) == len(tasks))
and all((payload is None for payload in media))
and (
not set(kwargs)
- shared._BATCHED_KEYS
- shared._SHARED_KEYS
- {"parse_result", "normalize"}
)
)
return guarded and bool(
shared._ordinary_lists(record, tasks, schema.list_item_name)
)
async def generate(
engine: Any,
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
eos: int,
tokenizer: PreTrainedTokenizerBase,
*,
initial_frontier: int = 0,
next_frontier: int = 0,
parse_result: Callable[..., dict[str, Any]] | None = None,
normalize: Callable[
[dict[str, Any], dict[str, dict[str, Any]]], dict[str, dict[str, Any]]
]
| None = None,
**kwargs: Any,
) -> list[dict[str, Any]]:
enabled = initial_frontier != 0
if enabled and (
type(initial_frontier) is not int
or type(next_frontier) is not int
or initial_frontier < 1
or (next_frontier <= initial_frontier)
):
raise ValueError(
"Ordinary list scheduling requires positive increasing integer endpoints"
)
groups = (
shared._ordinary_lists(record, tasks, schema.list_item_name)
if enabled and eligible(record, tasks, kwargs)
else {}
)
if not groups:
return await shared.generate(
engine,
record,
tasks,
eos,
tokenizer,
parse_result=parse_result,
normalize=normalize,
**kwargs,
)
size = len(tasks)
rids = kwargs.get("rid")
if (
not isinstance(rids, list)
or len(rids) != size
or len(set(rids)) != size
or (not all((isinstance(rid, str) and rid for rid in rids)))
or (kwargs.get("input_ids") != [ids for (_, ids, _) in tasks])
):
raise ValueError(
"Ordinary list scheduling requires original distinct task bindings"
)
shared._select_kwargs(kwargs, list(range(size)), size)
results: list[Any] = [None] * size
unexecuted: dict[str, int] = {}
async def submit(indices: list[int]) -> None:
if not indices:
return
selected = shared._select_kwargs(kwargs, indices, size)
received = await shared._generate(
engine,
record,
[tasks[index] for index in indices],
eos,
tokenizer,
**selected,
)
if len(received) != len(indices):
raise RuntimeError("Ordinary list scheduling received wrong batch size")
for index, result, rid in zip(indices, received, selected["rid"], strict=True):
if result.get("meta_info", {}).get("id") != rid:
raise RuntimeError(
"Ordinary list scheduling received wrong task identity"
)
results[index] = result
tails = {index for group in groups.values() for index in group[initial_frontier:]}
await submit([index for index in range(size) if index not in tails])
active = dict(groups)
(start, end, following) = (0, initial_frontier, next_frontier)
while active:
continued: list[int] = []
for parent, group in list(active.items()):
stopped = False
for index in group[start:end]:
try:
if shared._completed_null(results[index], eos, tokenizer):
stopped = True
break
except (KeyError, TypeError, ValueError, OverflowError):
continue
if stopped:
for index in group[end:]:
(name, ids, _) = tasks[index]
unexecuted[name] = len(ids)
results[index] = {
"_clair_unused_row_proposal": True,
"_ordinary_list_unexecuted": True,
"meta_info": {"completion_tokens": 0},
}
del active[parent]
elif end >= len(group):
del active[parent]
else:
continued.extend(group[end:following])
if continued:
await submit(sorted(continued))
(start, end, following) = (end, following, following * 2)
if any((result is None for result in results)):
raise RuntimeError("Ordinary list scheduling left an unaccounted result")
record["_unexecuted_list_items"] = unexecuted
return results