Spaces:
Running on Zero
Running on Zero
File size: 2,729 Bytes
2f31638 cad6462 2f31638 cad6462 e2d9ab4 cad6462 0d0289f a71f0c7 2f31638 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Small generation helpers that do not require loading the model."""
from __future__ import annotations
import math
from collections.abc import Iterable
def gpu_duration_seconds(
prompt_characters: int,
output_tokens: int,
max_context_tokens: int,
max_duration_seconds: int = 120,
) -> int:
"""Estimate a ZeroGPU reservation for prompt prefill plus generation.
Large coding prompts spend substantial GPU time on their 32k-token prefill
even when the requested answer is short. Character count is available to
the ZeroGPU duration callback before tokenization and is a conservative
proxy for that cost.
"""
characters = max(0, int(prompt_characters))
output = max(1, int(output_tokens))
context_limit = max(1, int(max_context_tokens))
estimated_input_tokens = min(context_limit, math.ceil(characters / 3))
estimate = (
25
+ math.ceil(estimated_input_tokens * 0.005)
+ math.ceil(output * 0.08)
)
return min(max_duration_seconds, max(30, estimate))
def head_tail_token_counts(
total_tokens: int,
token_budget: int,
preserved_prefix_tokens: int,
) -> tuple[int, int]:
"""Split an oversized prompt budget between its prefix and recent tail.
Keeping only the tail can erase system instructions, tool definitions, and
a task stated before a large code block. Keeping a bounded prefix plus the
largest possible tail retains both the operating contract and the newest
conversation state.
"""
total = max(0, int(total_tokens))
budget = max(1, int(token_budget))
if total <= budget:
return total, 0
prefix = max(0, int(preserved_prefix_tokens))
head = min(prefix, budget - 1)
return head, budget - head
def ensure_bos_token(prompt: str, bos_token: str | None) -> str:
"""Prefix the model's BOS token when the chat template omits it."""
if not bos_token or prompt.startswith(bos_token):
return prompt
return bos_token + prompt
def merge_eos_token_ids(
model_ids: int | Iterable[int] | None,
tokenizer_id: int | None,
) -> int | list[int] | None:
"""Keep every model stop token while preserving the tokenizer fallback."""
if isinstance(model_ids, int):
candidates = [model_ids]
elif model_ids is None:
candidates = []
else:
candidates = list(model_ids)
if tokenizer_id is not None:
candidates.append(tokenizer_id)
unique: list[int] = []
for candidate in candidates:
if isinstance(candidate, int) and candidate not in unique:
unique.append(candidate)
if not unique:
return None
if len(unique) == 1:
return unique[0]
return unique
|