File size: 5,353 Bytes
78736d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Bounded source-conditioned windows, shared by future training and inference.
Original groups are never cropped or split. A bounded auxiliary source-tail copy
is unscored encoder context, not extra retained text or consumer input.
"""
import hashlib
from pathlib import Path
PROFILE = "source-tail-1536-capacity-v2"
CONTEXT_TOKENS = 1536
WINDOW_TOKENS = 8192
MAX_GROUPS = 512
MAX_WINDOWS = 64
MAX_TOTAL_TOKENS = MAX_WINDOWS * WINDOW_TOKENS
def code_hash():
return hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
def verify_profile(report):
recipe = report.get("recipe", {})
if (recipe.get("scoring_window_profile") != PROFILE
or recipe.get("scoring_context_token_cap") != CONTEXT_TOKENS
or recipe.get("scoring_window_code_sha256") != code_hash()):
raise ValueError("matching_trained_source_context_profile_required")
def pack_windows(tokenizer, config, request):
"""Return legacy-shaped windows and a non-qualifying packing receipt.
Tokenization is once per query/group. Each original group has one scored
range; auxiliary copies have no ranges. All admission finishes before forward.
"""
groups = request["groups"]
if not isinstance(groups, list) or not 1 <= len(groups) <= MAX_GROUPS:
raise ValueError("bounded_nonempty_groups_required")
if (not isinstance(request.get("query"), str) or not request["query"].strip()
or any(not isinstance(g, dict) or not isinstance(g.get("text"), str) or not g["text"] for g in groups)):
raise ValueError("original_query_and_group_text_required")
tokenizer.no_truncation(); tokenizer.no_padding()
def special(name):
value = config[name]
token = tokenizer.token_to_id(value["content"] if isinstance(value, dict) else value)
if token is None: raise ValueError("missing_special_token")
return token
cls, sep = special("cls_token"), special("sep_token")
query = [cls] + tokenizer.encode(request["query"], add_special_tokens=False).ids + [sep]
if len(query) >= WINDOW_TOKENS: raise ValueError("query_token_cap")
encoded = [tokenizer.encode(group["text"], add_special_tokens=False).ids for group in groups]
if any(not ids or len(query) + len(ids) + 1 > WINDOW_TOKENS for ids in encoded):
raise ValueError("whole_group_token_cap")
source_tokens = sum(len(ids) + 1 for ids in encoded)
if source_tokens + len(query) > MAX_TOTAL_TOKENS:
raise ValueError("total_token_cap")
context = []; capacity_checks = 0; compatible_allowance = 0; reduced = False
if source_tokens + len(query) > WINDOW_TOKENS:
allowance = min(CONTEXT_TOKENS, WINDOW_TOKENS - len(query) - max(map(len, encoded)) - 2)
if allowance < 1: raise ValueError("whole_group_leaves_no_source_context")
compatible_allowance = allowance
def demand(amount):
nonlocal capacity_checks
capacity_checks += 1
header = len(query) + amount + 1
count = 1; used = header
for content in encoded:
length = len(content) + 1
if used + length > WINDOW_TOKENS:
count += 1; used = header
used += length
return count, source_tokens + count * header
count, total = demand(allowance)
if count > MAX_WINDOWS or total > MAX_TOTAL_TOKENS:
count, total = demand(1)
if count > MAX_WINDOWS: raise ValueError("window_count_cap")
if total > MAX_TOTAL_TOKENS: raise ValueError("total_token_cap")
# Largest feasible auxiliary copy, never fewer original source tokens.
low, high = 1, allowance
while low < high:
mid = (low + high + 1) // 2
count, total = demand(mid)
if count <= MAX_WINDOWS and total <= MAX_TOTAL_TOKENS: low = mid
else: high = mid - 1
allowance = low; reduced = True
for ids in encoded:
context = (context + [sep] + ids)[-allowance:]
header = query + context + [sep] if context else query
windows = []
ids, ranges, indices = list(header), [], []
for index, content in enumerate(encoded):
if len(ids) + len(content) + 1 > WINDOW_TOKENS:
windows.append((ids, ranges, indices))
if len(windows) >= MAX_WINDOWS: raise ValueError("window_count_cap")
ids, ranges, indices = list(header), [], []
start = len(ids); ids.extend(content); ids.append(sep)
ranges.append((start, start + len(content))); indices.append(index)
windows.append((ids, ranges, indices))
total = sum(len(window[0]) for window in windows)
if total > MAX_TOTAL_TOKENS: raise ValueError("total_token_cap")
return windows, {"profile": PROFILE, "context_tokens_per_window": len(context),
"max_context_tokens": CONTEXT_TOKENS,
"whole_group_compatible_context_tokens": compatible_allowance,
"context_reduced_for_capacity": reduced, "capacity_checks": capacity_checks,
"scored_source_tokens_with_separators": source_tokens,
"total_encoded_tokens_including_copies": total,
"windows": len(windows), "groups": len(groups), "qualification": False}
|