GeoPOI / runtime /causal.py
Tiga233's picture
Release complete GeoPOI model with shared regional basis and 500-entity public catalog
49d63ea verified
Raw History Blame Contribute Delete
22.7 kB
from __future__ import annotations
from dataclasses import dataclass
import collections,hashlib,json,math,struct
SCHEMA = 'R8_service_clock_Entry18_Gate46_causal_helper_v1'
MAX_LOOKAHEAD_US = 1000000
ENTRY_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64']
GATE_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64', 'candidate_local_mapping_valid', 'candidate_unique_original_position', 'candidate_family_ambiguous', 'candidate_family_canonical_count', 'candidate_canonical_character_count', 'candidate_old_literal_character_count', 'candidate_current_literal_in_local_phone_family', 'candidate_unique_literal_mismatch', 'candidate_left_anchor_characters', 'candidate_right_anchor_characters', 'candidate_start_in_tail_div64', 'candidate_end_in_tail_div64', 'candidate_ranker_score_or0', 'ranker_none_or0', 'candidate_ranker_minus_none_or0', 'candidate_reader_acoustic_score_or0', 'candidate_retriever_coarse_score_or0', 'original_selected_rank_div10_or1', 'ranker_top1_minus_top2_or0', 'valid_span_count_div4', 'pool_unique_surfaces_log1p', 'candidate_consecutive_versions_cap8', 'candidate_anchor_consecutive_versions_cap8', 'candidate_source_local_fallback', 'candidate_R8_scores_available', 'candidate_phonefamily_uncertain', 'reserved_policy_independent_overlap_zero', 'candidate_original_shortlist_rank_fraction_or1']
CANONICAL_KEYS = ('candidate_id', 'variant_id', 'lexical_id', 'lexical_entity_id', 'cell', 'surface_id', 'canonical', 'decoder_token_ids')
def need(v, m):
if not v:
raise ValueError(m)
def digest(v):
return hashlib.sha256(json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode()).hexdigest()
def f32(v):
v = float(v)
need(math.isfinite(v), 'finite feature')
return struct.unpack('<f', struct.pack('<f', v))[0]
def integer(v, name):
if type(v) is not int or v < 0:
raise ValueError(name + ' must be a nonnegative integer')
def text(v, name):
if type(v) is not str:
raise TypeError(name + ' must be raw Unicode text')
v.encode('utf-8', errors='strict')
@dataclass(frozen=True)
class RawSnapshot:
version: int
audio_time_us: int
text: str
source_audio_eof: bool
asr_done: bool
def __post_init__(self):
integer(self.version, 'version')
integer(self.audio_time_us, 'audio_time_us')
text(self.text, 'text')
if type(self.source_audio_eof) is not bool or type(self.asr_done) is not bool:
raise TypeError('separate actual source EOF and ASR done booleans required')
@dataclass(frozen=True)
class LiteralProposal:
observed_version: int
start: int
end: int
old_literal: str
replacement: str
left_anchor: str
right_anchor: str
def __post_init__(self):
for key in ('observed_version', 'start', 'end'):
integer(getattr(self, key), key)
for key in ('old_literal', 'replacement', 'left_anchor', 'right_anchor'):
text(getattr(self, key), key)
if not self.old_literal or not self.replacement or self.end <= self.start or (self.end - self.start != len(self.old_literal)):
raise ValueError('exact nonempty observed character span and replacement required')
@dataclass(frozen=True)
class FrozenPatch:
sequence: int
proposal: LiteralProposal
proposal_audio_time_us: int
decision_audio_time_us: int
decision_version: int
@dataclass(frozen=True)
class FreezeDecision:
accepted: bool
reason: str
patch: FrozenPatch | None = None
@dataclass(frozen=True)
class PatchOutcome:
sequence: int
applied: bool
reason: str
@dataclass(frozen=True)
class RenderResult:
raw_final_text: str
corrected_text: str
outcomes: tuple
outside_applied_spans_byte_exact: bool
@dataclass(frozen=True)
class PartialRenderResult:
observed_version: int
raw_partial_text: str
corrected_text: str
outcomes: tuple
outside_applied_spans_byte_exact: bool
def matches(raw, p):
return p.start >= len(p.left_anchor) and p.end + len(p.right_anchor) <= len(raw) and (raw[p.start:p.end] == p.old_literal) and (raw[p.start - len(p.left_anchor):p.start] == p.left_anchor) and (raw[p.end:p.end + len(p.right_anchor)] == p.right_anchor)
def overlap(a, b):
return max(a.start, b.start) < min(a.end, b.end)
class ActiveLiteralPatcher:
def __init__(self):
self._snapshots = []
self._versions = {}
self._received_audio_us = 0
self._source_eof_seen = False
self._asr_done_seen = False
self._patches = []
self._invalidated = {}
self._rendered = False
@property
def raw_snapshots(self):
return tuple(self._snapshots)
@property
def frozen_patches(self):
return tuple(self._patches)
def observe_raw(self, snapshot, *, received_audio_time_us):
if type(snapshot) is not RawSnapshot:
raise TypeError('only original RawSnapshot inputs, never rendered results')
integer(received_audio_time_us, 'received_audio_time_us')
if self._rendered or self._asr_done_seen:
raise ValueError('ASR done ledger cannot accept more raw snapshots')
if snapshot.audio_time_us > received_audio_time_us or received_audio_time_us < self._received_audio_us:
raise ValueError('future or backward received-audio clock')
if self._snapshots:
prior = self._snapshots[-1]
if snapshot.version <= prior.version or snapshot.audio_time_us < prior.audio_time_us:
raise ValueError('raw versions and audio clocks must be ordered')
if self._source_eof_seen and (not snapshot.source_audio_eof):
raise ValueError('actual physical EOF flag cannot regress')
self._snapshots.append(snapshot)
self._versions[snapshot.version] = snapshot
self._received_audio_us = received_audio_time_us
self._source_eof_seen |= snapshot.source_audio_eof
self._asr_done_seen |= snapshot.asr_done
for p in self._patches:
if not matches(snapshot.text, p.proposal):
self._invalidated.setdefault(p.sequence, 'RAW_REVISION_INVALIDATED_ANCHOR')
def freeze(self, proposal, *, source_audio_eof, asr_done):
if type(proposal) is not LiteralProposal or type(source_audio_eof) is not bool or type(asr_done) is not bool:
raise TypeError('fixed proposal and two explicit actual lifecycle booleans required')
if self._rendered or self._asr_done_seen or asr_done:
return FreezeDecision(False, 'ASR_DONE_OR_FINAL_ALREADY_OBSERVED')
if not self._snapshots or proposal.observed_version not in self._versions:
return FreezeDecision(False, 'PROPOSAL_SNAPSHOT_NOT_OBSERVED')
current = self._snapshots[-1]
if source_audio_eof != current.source_audio_eof or asr_done != current.asr_done:
raise ValueError('physical EOF and ASR done must match actual current snapshot')
original = self._versions[proposal.observed_version]
if not 0 <= self._received_audio_us - original.audio_time_us <= MAX_LOOKAHEAD_US:
return FreezeDecision(False, 'PROPOSAL_WINDOW_EXCEEDED_ONE_SECOND')
if not matches(original.text, proposal):
return FreezeDecision(False, 'PROPOSAL_NOT_PRESENT_AT_OBSERVATION')
if not proposal.left_anchor and proposal.start != 0 or (not proposal.right_anchor and proposal.end != len(original.text)):
return FreezeDecision(False, 'EMPTY_ANCHOR_ONLY_AT_OBSERVED_TEXT_BOUNDARY')
if any((not matches(s.text, proposal) for s in self._snapshots if s.version >= original.version)):
return FreezeDecision(False, 'PROPOSAL_INVALIDATED_BEFORE_FREEZE')
if proposal.replacement == proposal.old_literal:
return FreezeDecision(False, 'NO_CHANGE')
if any((overlap(proposal, p.proposal) for p in self._patches if p.sequence not in self._invalidated)):
return FreezeDecision(False, 'OVERLAP_FIRST_FREEZE_WINS')
p = FrozenPatch(len(self._patches), proposal, original.audio_time_us, self._received_audio_us, current.version)
self._patches.append(p)
return FreezeDecision(True, 'FROZEN_BEFORE_ASR_DONE', p)
def render_current_partial(self):
if self._rendered or self._asr_done_seen or (not self._snapshots):
raise ValueError('observed nonfinal ASR partial required')
s = self._snapshots[-1]
corrected, outcomes = self._render_frozen(s)
return PartialRenderResult(s.version, s.text, corrected, outcomes, True)
def render_final(self, snapshot, *, received_audio_time_us):
if type(snapshot) is not RawSnapshot or not snapshot.asr_done:
raise ValueError('actual transcription.done snapshot required')
self.observe_raw(snapshot, received_audio_time_us=received_audio_time_us)
corrected, outcomes = self._render_frozen(snapshot, final_version=snapshot.version)
self._rendered = True
return RenderResult(snapshot.text, corrected, outcomes, True)
def _render_frozen(self, snapshot, *, final_version=None):
raw = snapshot.text
applied = []
outcomes = []
for patch in self._patches:
reason = self._invalidated.get(patch.sequence)
if final_version is not None and patch.decision_version >= final_version:
reason = 'DECISION_VERSION_NOT_BEFORE_ASR_DONE'
if reason is None and (not matches(raw, patch.proposal)):
reason = 'FINAL_FIXED_POSITION_OR_ANCHOR_MISMATCH' if final_version is not None else 'CURRENT_FIXED_POSITION_OR_ANCHOR_MISMATCH'
if reason is None:
applied.append(patch)
outcomes.append(PatchOutcome(patch.sequence, reason is None, reason or 'APPLIED'))
corrected = raw
for patch in sorted(applied, key=lambda p: p.proposal.start, reverse=True):
p = patch.proposal
corrected = corrected[:p.start] + p.replacement + corrected[p.end:]
actual = corrected.encode('utf-8')
raw_cursor = output_cursor = 0
for patch in sorted(applied, key=lambda p: p.proposal.start):
p = patch.proposal
unchanged = raw[raw_cursor:p.start].encode('utf-8')
replacement = p.replacement.encode('utf-8')
if actual[output_cursor:output_cursor + len(unchanged)] != unchanged:
raise AssertionError('outside frozen raw spans changed')
output_cursor += len(unchanged)
if actual[output_cursor:output_cursor + len(replacement)] != replacement:
raise AssertionError('fixed replacement differs')
output_cursor += len(replacement)
raw_cursor = p.end
if actual[output_cursor:] != raw[raw_cursor:].encode('utf-8'):
raise AssertionError('outside frozen raw spans changed')
return (corrected, tuple(outcomes))
def active_overlap(ledger, proposal):
return any((p.sequence not in ledger._invalidated and overlap(proposal, p.proposal) for p in ledger.frozen_patches))
def candidate_evidence(case, snapshot, value, pool):
if value is None:
return (None, [], [], None)
if 'evidence' in value:
e = value['evidence']
else:
e = value
row = value.get('original_physical_decision', value)
for key in ('observed_version', 'audio_time_us', 'received_audio_time_us', 'raw_partial_event_ref'):
if key in row:
need(row[key] == snapshot[key], 'candidate uses this actual observed partial ' + key)
if 'logical_original_global_row_index' in value:
need(value['logical_original_global_row_index'] == case['original_global_row_index'], 'logical view candidate identity')
origin = value.get('logical_pool', value.get('original_pool', case['pool']))
need(origin['sha256'] == case['pool']['sha256'] and origin['size'] == case['pool']['size'], 'same original logical pool body')
allrows = e['all_original_shortlist']
selected = e['selected']
need(type(allrows) is list and type(selected) is list and (len(selected) <= 10) and (e['whole_selected_list_preserved'] is True), 'whole original shortlist and orderedtop10')
for c in allrows:
i = c['index']
need(type(i) is int and 0 <= i < len(pool['rows']) and all((c[k] == pool['rows'][i][k] for k in CANONICAL_KEYS)), 'all candidate canonical/token/cell/row lineage is original pool')
identity = {k: c[k] for k in CANONICAL_KEYS}
need(c['candidate_identity_sha256'] == digest(identity) and c['candidate_binding_sha256'] == digest(dict(original_pool=origin, original_pool_row=i, identity=identity)), 'original frozen candidate SHA binding')
known = {c['candidate_binding_sha256']: c for c in allrows}
need(all((c['candidate_binding_sha256'] in known and known[c['candidate_binding_sha256']] == c for c in selected)), 'selected exact original shortlist records in order')
return (e, allrows, selected, origin)
class CaseFeatureState:
def __init__(self):
self.previous_tail = ''
self.previous_offset = 0
self.have_previous = False
self.mismatch_counts = {}
self.candidate_counts = {}
self.anchor_counts = {}
def entry(self, s, text, matches):
offset = max(0, len(text) - 64)
tail = text[offset:]
analysis = matches._analysis.get('matches', [])
lengths = [x['stop_character'] - x['start_character'] for x in analysis]
unique = {digest([offset + x['start_character'], offset + x['stop_character'], tail[x['start_character']:x['stop_character']], x['complete_phone_key']]) for x in analysis if x['unique_literal_mismatch']}
current = {k: min(8, self.mismatch_counts.get(k, 0) + 1) for k in unique}
stable = 0
if self.have_previous:
for i, ch in enumerate(tail):
j = offset + i - self.previous_offset
stable += int(0 <= j < len(self.previous_tail) and self.previous_tail[j] == ch)
values = [s['audio_time_us'] / 1000000.0, (s['received_audio_time_us'] - s['audio_time_us']) / 1000000.0, bool(s['eos_observed']), len(tail) / 64, bool(matches.local_valid), len(analysis), sum((bool(x['unique_literal_mismatch']) for x in analysis)), sum((bool(x['literal_in_local_phone_family']) for x in analysis)), sum((bool(x['ambiguous']) for x in analysis)), sum((bool(x['uncertain_literal_mapping']) for x in analysis)), min(lengths, default=0), max(lengths, default=0), max((x['local_distinct_canonical_count'] for x in analysis), default=0), len(matches._query_proof.get('scope', {}).get('mapped_region_ids', [])), bool(matches._query_proof), stable / len(tail) if tail and self.have_previous else 0, max(current.values(), default=0), min(s['observed_version'], 64)]
self.previous_tail = tail
self.previous_offset = offset
self.have_previous = True
self.mismatch_counts = current
return (list(map(f32, values)), tail, offset)
def candidates(self, actions):
current = {}
anchors = {}
for a in actions:
p = a['proposal']
key = digest([p['start'], p['end'], p['old_literal'], p['replacement'], a['candidate_identity_sha256']])
ak = digest([key, p['left_anchor'], p['right_anchor']])
a['_consecutive'] = min(8, self.candidate_counts.get(key, 0) + 1)
a['_anchor_consecutive'] = min(8, self.anchor_counts.get(ak, 0) + 1)
current[key] = a['_consecutive']
anchors[ak] = a['_anchor_consecutive']
self.candidate_counts = current
self.anchor_counts = anchors
def candidate_action(candidate, located, tail, offset, s, source, order):
if not located.valid:
return None
start, end = (located.start, located.end)
if offset > 0 and start == 0:
return None
left = tail[max(0, start - 1):start]
right = tail[end:end + 1]
p = LiteralProposal(s['observed_version'], offset + start, offset + end, located.old_literal, located.replacement, left, right)
if p.old_literal == p.replacement:
return None
return dict(proposal=vars(p), candidate_identity_sha256=candidate['candidate_identity_sha256'], candidate_binding_sha256=candidate['candidate_binding_sha256'], original_pool_row=candidate['index'], candidate={k: candidate[k] for k in CANONICAL_KEYS}, location_audit=located.audit(), source_local_fallback=source, original_order=order, original_selected_rank=order if not source else None)
def propose_snapshot(case, s, text, value, configuration, pool, state, front):
need(s['is_final'] is False, 'done text never a decision input')
need(type(s['eos_observed']) is bool and 0 <= s['audio_time_us'] <= s['received_audio_time_us'], 'actual causal source clock')
tail = text[max(0, len(text) - 64):]
matches = front.analyze_current_local_view(tail, observed_version=s['observed_version'], audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], asr_done=False, cells=case['geography']['cells'])
entry, tail, offset = state.entry(s, text, matches)
ev, allrows, selected, origin = candidate_evidence(case, s, value, pool)
audit = []
actions = []
primary_patches = set()
eligible = s['received_audio_time_us'] - s['audio_time_us'] <= MAX_LOOKAHEAD_US
for rank, c in enumerate(selected):
located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256'])
a = located.audit()
a.update(source_local_fallback=False, original_selected_rank=rank, NOCHANGE_is_baseline_not_truth_or_reservation=True)
act = candidate_action(c, located, tail, offset, s, False, rank)
if act is not None and eligible:
actions.append(act)
primary_patches.add((act['proposal']['start'], act['proposal']['end'], act['proposal']['replacement']))
a['eligible_action'] = act is not None and eligible
audit.append(a)
fallback = []
for match in matches._analysis.get('matches', []):
family = matches._families['families'][match['complete_phone_key']]
literals = family['literals']
if match['ambiguous'] or match['uncertain_literal_mapping'] or len(literals) != 1:
continue
name = literals[0]
if name not in pool['minimum_row_per_canonical']:
continue
c = pool['minimum_row_per_canonical'][name]
located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256'])
act = candidate_action(c, located, tail, offset, s, True, len(selected))
if act is None or not eligible:
continue
p = act['proposal']
if (p['start'], p['end'], p['replacement']) in primary_patches:
continue
fallback.append((p['start'], -(p['end'] - p['start']), hashlib.sha256(c['canonical'].encode()).hexdigest(), c['index'], act))
for order, item in enumerate(sorted(fallback, key=lambda x: x[:4])):
item[-1]['original_order'] = len(selected) + order
actions.append(item[-1])
state.candidates(actions)
shortlist_by_row = {c['index']: (i, c) for i, c in enumerate(allrows)}
top = sorted((float(c['ranker_score']) for c in allrows), reverse=True)
gap = top[0] - top[1] if len(top) > 1 else 0
none = float(ev['ranker_none']) if ev is not None else 0
valid_spans = float(ev['valid_span_count']) if ev is not None else 0
gate = []
for a in actions:
p = a['proposal']
l = a['location_audit']
record = shortlist_by_row.get(a['original_pool_row'])
available = record is not None
score = record[1] if available else {}
bs = float(score.get('ranker_score', 0))
ac = float(score.get('acoustic_score', 0))
co = float(score.get('coarse', 0))
rank = a['original_selected_rank']
extra = [bool(matches.local_valid), bool(l['unique_original_character_position']), bool(l['family_ambiguous']), l['family_canonical_count'] or 0, len(a['candidate']['canonical']), len(p['old_literal']), bool(l['literal_in_local_phone_family']), bool(l['original_unique_literal_mismatch']), len(p['left_anchor']), len(p['right_anchor']), (p['start'] - offset) / 64, (p['end'] - offset) / 64, bs, none, bs - none if available and ev is not None else 0, ac, co, rank / 10 if rank is not None else 1, gap, valid_spans / 4, math.log1p(pool['unique_surfaces']), a.pop('_consecutive'), a.pop('_anchor_consecutive'), a['source_local_fallback'], available, bool(l['family_uncertain']), 0, record[0] / max(1, len(allrows)) if available else 1]
need(len(extra) == 28, 'exact Gate28 suffix')
gate.append(entry + list(map(f32, extra)))
a.update(candidate_R8_scores_available=available, original_candidate_pool=origin if available else case['pool'], active_overlap_not_a_model_feature=True)
receipt = dict(schema=SCHEMA + '_decision', observed_version=s['observed_version'], is_final=False, audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], raw_partial_event_ref=s.get('raw_partial_event_ref'), tail_start_absolute_character=offset, tail_character_count=len(tail), local_mapping_audit=matches.audit(), candidate_audit=audit, action_count=len(actions), source_lookahead_within1s=eligible, NOCHANGE_has_no_action_row_and_does_not_block_other_mismatch=True, reserved_Gate_column44=0)
return (entry, gate, actions, receipt)