from __future__ import annotations from dataclasses import dataclass import collections,hashlib,json,math,struct SCHEMA = 'R8_service_clock_Entry18_Gate46_causal_helper_v1' MAX_LOOKAHEAD_US = 1000000 ENTRY_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64'] GATE_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64', 'candidate_local_mapping_valid', 'candidate_unique_original_position', 'candidate_family_ambiguous', 'candidate_family_canonical_count', 'candidate_canonical_character_count', 'candidate_old_literal_character_count', 'candidate_current_literal_in_local_phone_family', 'candidate_unique_literal_mismatch', 'candidate_left_anchor_characters', 'candidate_right_anchor_characters', 'candidate_start_in_tail_div64', 'candidate_end_in_tail_div64', 'candidate_ranker_score_or0', 'ranker_none_or0', 'candidate_ranker_minus_none_or0', 'candidate_reader_acoustic_score_or0', 'candidate_retriever_coarse_score_or0', 'original_selected_rank_div10_or1', 'ranker_top1_minus_top2_or0', 'valid_span_count_div4', 'pool_unique_surfaces_log1p', 'candidate_consecutive_versions_cap8', 'candidate_anchor_consecutive_versions_cap8', 'candidate_source_local_fallback', 'candidate_R8_scores_available', 'candidate_phonefamily_uncertain', 'reserved_policy_independent_overlap_zero', 'candidate_original_shortlist_rank_fraction_or1'] CANONICAL_KEYS = ('candidate_id', 'variant_id', 'lexical_id', 'lexical_entity_id', 'cell', 'surface_id', 'canonical', 'decoder_token_ids') def need(v, m): if not v: raise ValueError(m) def digest(v): return hashlib.sha256(json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode()).hexdigest() def f32(v): v = float(v) need(math.isfinite(v), 'finite feature') return struct.unpack('= len(p.left_anchor) and p.end + len(p.right_anchor) <= len(raw) and (raw[p.start:p.end] == p.old_literal) and (raw[p.start - len(p.left_anchor):p.start] == p.left_anchor) and (raw[p.end:p.end + len(p.right_anchor)] == p.right_anchor) def overlap(a, b): return max(a.start, b.start) < min(a.end, b.end) class ActiveLiteralPatcher: def __init__(self): self._snapshots = [] self._versions = {} self._received_audio_us = 0 self._source_eof_seen = False self._asr_done_seen = False self._patches = [] self._invalidated = {} self._rendered = False @property def raw_snapshots(self): return tuple(self._snapshots) @property def frozen_patches(self): return tuple(self._patches) def observe_raw(self, snapshot, *, received_audio_time_us): if type(snapshot) is not RawSnapshot: raise TypeError('only original RawSnapshot inputs, never rendered results') integer(received_audio_time_us, 'received_audio_time_us') if self._rendered or self._asr_done_seen: raise ValueError('ASR done ledger cannot accept more raw snapshots') if snapshot.audio_time_us > received_audio_time_us or received_audio_time_us < self._received_audio_us: raise ValueError('future or backward received-audio clock') if self._snapshots: prior = self._snapshots[-1] if snapshot.version <= prior.version or snapshot.audio_time_us < prior.audio_time_us: raise ValueError('raw versions and audio clocks must be ordered') if self._source_eof_seen and (not snapshot.source_audio_eof): raise ValueError('actual physical EOF flag cannot regress') self._snapshots.append(snapshot) self._versions[snapshot.version] = snapshot self._received_audio_us = received_audio_time_us self._source_eof_seen |= snapshot.source_audio_eof self._asr_done_seen |= snapshot.asr_done for p in self._patches: if not matches(snapshot.text, p.proposal): self._invalidated.setdefault(p.sequence, 'RAW_REVISION_INVALIDATED_ANCHOR') def freeze(self, proposal, *, source_audio_eof, asr_done): if type(proposal) is not LiteralProposal or type(source_audio_eof) is not bool or type(asr_done) is not bool: raise TypeError('fixed proposal and two explicit actual lifecycle booleans required') if self._rendered or self._asr_done_seen or asr_done: return FreezeDecision(False, 'ASR_DONE_OR_FINAL_ALREADY_OBSERVED') if not self._snapshots or proposal.observed_version not in self._versions: return FreezeDecision(False, 'PROPOSAL_SNAPSHOT_NOT_OBSERVED') current = self._snapshots[-1] if source_audio_eof != current.source_audio_eof or asr_done != current.asr_done: raise ValueError('physical EOF and ASR done must match actual current snapshot') original = self._versions[proposal.observed_version] if not 0 <= self._received_audio_us - original.audio_time_us <= MAX_LOOKAHEAD_US: return FreezeDecision(False, 'PROPOSAL_WINDOW_EXCEEDED_ONE_SECOND') if not matches(original.text, proposal): return FreezeDecision(False, 'PROPOSAL_NOT_PRESENT_AT_OBSERVATION') if not proposal.left_anchor and proposal.start != 0 or (not proposal.right_anchor and proposal.end != len(original.text)): return FreezeDecision(False, 'EMPTY_ANCHOR_ONLY_AT_OBSERVED_TEXT_BOUNDARY') if any((not matches(s.text, proposal) for s in self._snapshots if s.version >= original.version)): return FreezeDecision(False, 'PROPOSAL_INVALIDATED_BEFORE_FREEZE') if proposal.replacement == proposal.old_literal: return FreezeDecision(False, 'NO_CHANGE') if any((overlap(proposal, p.proposal) for p in self._patches if p.sequence not in self._invalidated)): return FreezeDecision(False, 'OVERLAP_FIRST_FREEZE_WINS') p = FrozenPatch(len(self._patches), proposal, original.audio_time_us, self._received_audio_us, current.version) self._patches.append(p) return FreezeDecision(True, 'FROZEN_BEFORE_ASR_DONE', p) def render_current_partial(self): if self._rendered or self._asr_done_seen or (not self._snapshots): raise ValueError('observed nonfinal ASR partial required') s = self._snapshots[-1] corrected, outcomes = self._render_frozen(s) return PartialRenderResult(s.version, s.text, corrected, outcomes, True) def render_final(self, snapshot, *, received_audio_time_us): if type(snapshot) is not RawSnapshot or not snapshot.asr_done: raise ValueError('actual transcription.done snapshot required') self.observe_raw(snapshot, received_audio_time_us=received_audio_time_us) corrected, outcomes = self._render_frozen(snapshot, final_version=snapshot.version) self._rendered = True return RenderResult(snapshot.text, corrected, outcomes, True) def _render_frozen(self, snapshot, *, final_version=None): raw = snapshot.text applied = [] outcomes = [] for patch in self._patches: reason = self._invalidated.get(patch.sequence) if final_version is not None and patch.decision_version >= final_version: reason = 'DECISION_VERSION_NOT_BEFORE_ASR_DONE' if reason is None and (not matches(raw, patch.proposal)): reason = 'FINAL_FIXED_POSITION_OR_ANCHOR_MISMATCH' if final_version is not None else 'CURRENT_FIXED_POSITION_OR_ANCHOR_MISMATCH' if reason is None: applied.append(patch) outcomes.append(PatchOutcome(patch.sequence, reason is None, reason or 'APPLIED')) corrected = raw for patch in sorted(applied, key=lambda p: p.proposal.start, reverse=True): p = patch.proposal corrected = corrected[:p.start] + p.replacement + corrected[p.end:] actual = corrected.encode('utf-8') raw_cursor = output_cursor = 0 for patch in sorted(applied, key=lambda p: p.proposal.start): p = patch.proposal unchanged = raw[raw_cursor:p.start].encode('utf-8') replacement = p.replacement.encode('utf-8') if actual[output_cursor:output_cursor + len(unchanged)] != unchanged: raise AssertionError('outside frozen raw spans changed') output_cursor += len(unchanged) if actual[output_cursor:output_cursor + len(replacement)] != replacement: raise AssertionError('fixed replacement differs') output_cursor += len(replacement) raw_cursor = p.end if actual[output_cursor:] != raw[raw_cursor:].encode('utf-8'): raise AssertionError('outside frozen raw spans changed') return (corrected, tuple(outcomes)) def active_overlap(ledger, proposal): return any((p.sequence not in ledger._invalidated and overlap(proposal, p.proposal) for p in ledger.frozen_patches)) def candidate_evidence(case, snapshot, value, pool): if value is None: return (None, [], [], None) if 'evidence' in value: e = value['evidence'] else: e = value row = value.get('original_physical_decision', value) for key in ('observed_version', 'audio_time_us', 'received_audio_time_us', 'raw_partial_event_ref'): if key in row: need(row[key] == snapshot[key], 'candidate uses this actual observed partial ' + key) if 'logical_original_global_row_index' in value: need(value['logical_original_global_row_index'] == case['original_global_row_index'], 'logical view candidate identity') origin = value.get('logical_pool', value.get('original_pool', case['pool'])) need(origin['sha256'] == case['pool']['sha256'] and origin['size'] == case['pool']['size'], 'same original logical pool body') allrows = e['all_original_shortlist'] selected = e['selected'] need(type(allrows) is list and type(selected) is list and (len(selected) <= 10) and (e['whole_selected_list_preserved'] is True), 'whole original shortlist and orderedtop10') for c in allrows: i = c['index'] need(type(i) is int and 0 <= i < len(pool['rows']) and all((c[k] == pool['rows'][i][k] for k in CANONICAL_KEYS)), 'all candidate canonical/token/cell/row lineage is original pool') identity = {k: c[k] for k in CANONICAL_KEYS} need(c['candidate_identity_sha256'] == digest(identity) and c['candidate_binding_sha256'] == digest(dict(original_pool=origin, original_pool_row=i, identity=identity)), 'original frozen candidate SHA binding') known = {c['candidate_binding_sha256']: c for c in allrows} need(all((c['candidate_binding_sha256'] in known and known[c['candidate_binding_sha256']] == c for c in selected)), 'selected exact original shortlist records in order') return (e, allrows, selected, origin) class CaseFeatureState: def __init__(self): self.previous_tail = '' self.previous_offset = 0 self.have_previous = False self.mismatch_counts = {} self.candidate_counts = {} self.anchor_counts = {} def entry(self, s, text, matches): offset = max(0, len(text) - 64) tail = text[offset:] analysis = matches._analysis.get('matches', []) lengths = [x['stop_character'] - x['start_character'] for x in analysis] unique = {digest([offset + x['start_character'], offset + x['stop_character'], tail[x['start_character']:x['stop_character']], x['complete_phone_key']]) for x in analysis if x['unique_literal_mismatch']} current = {k: min(8, self.mismatch_counts.get(k, 0) + 1) for k in unique} stable = 0 if self.have_previous: for i, ch in enumerate(tail): j = offset + i - self.previous_offset stable += int(0 <= j < len(self.previous_tail) and self.previous_tail[j] == ch) values = [s['audio_time_us'] / 1000000.0, (s['received_audio_time_us'] - s['audio_time_us']) / 1000000.0, bool(s['eos_observed']), len(tail) / 64, bool(matches.local_valid), len(analysis), sum((bool(x['unique_literal_mismatch']) for x in analysis)), sum((bool(x['literal_in_local_phone_family']) for x in analysis)), sum((bool(x['ambiguous']) for x in analysis)), sum((bool(x['uncertain_literal_mapping']) for x in analysis)), min(lengths, default=0), max(lengths, default=0), max((x['local_distinct_canonical_count'] for x in analysis), default=0), len(matches._query_proof.get('scope', {}).get('mapped_region_ids', [])), bool(matches._query_proof), stable / len(tail) if tail and self.have_previous else 0, max(current.values(), default=0), min(s['observed_version'], 64)] self.previous_tail = tail self.previous_offset = offset self.have_previous = True self.mismatch_counts = current return (list(map(f32, values)), tail, offset) def candidates(self, actions): current = {} anchors = {} for a in actions: p = a['proposal'] key = digest([p['start'], p['end'], p['old_literal'], p['replacement'], a['candidate_identity_sha256']]) ak = digest([key, p['left_anchor'], p['right_anchor']]) a['_consecutive'] = min(8, self.candidate_counts.get(key, 0) + 1) a['_anchor_consecutive'] = min(8, self.anchor_counts.get(ak, 0) + 1) current[key] = a['_consecutive'] anchors[ak] = a['_anchor_consecutive'] self.candidate_counts = current self.anchor_counts = anchors def candidate_action(candidate, located, tail, offset, s, source, order): if not located.valid: return None start, end = (located.start, located.end) if offset > 0 and start == 0: return None left = tail[max(0, start - 1):start] right = tail[end:end + 1] p = LiteralProposal(s['observed_version'], offset + start, offset + end, located.old_literal, located.replacement, left, right) if p.old_literal == p.replacement: return None return dict(proposal=vars(p), candidate_identity_sha256=candidate['candidate_identity_sha256'], candidate_binding_sha256=candidate['candidate_binding_sha256'], original_pool_row=candidate['index'], candidate={k: candidate[k] for k in CANONICAL_KEYS}, location_audit=located.audit(), source_local_fallback=source, original_order=order, original_selected_rank=order if not source else None) def propose_snapshot(case, s, text, value, configuration, pool, state, front): need(s['is_final'] is False, 'done text never a decision input') need(type(s['eos_observed']) is bool and 0 <= s['audio_time_us'] <= s['received_audio_time_us'], 'actual causal source clock') tail = text[max(0, len(text) - 64):] matches = front.analyze_current_local_view(tail, observed_version=s['observed_version'], audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], asr_done=False, cells=case['geography']['cells']) entry, tail, offset = state.entry(s, text, matches) ev, allrows, selected, origin = candidate_evidence(case, s, value, pool) audit = [] actions = [] primary_patches = set() eligible = s['received_audio_time_us'] - s['audio_time_us'] <= MAX_LOOKAHEAD_US for rank, c in enumerate(selected): located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256']) a = located.audit() a.update(source_local_fallback=False, original_selected_rank=rank, NOCHANGE_is_baseline_not_truth_or_reservation=True) act = candidate_action(c, located, tail, offset, s, False, rank) if act is not None and eligible: actions.append(act) primary_patches.add((act['proposal']['start'], act['proposal']['end'], act['proposal']['replacement'])) a['eligible_action'] = act is not None and eligible audit.append(a) fallback = [] for match in matches._analysis.get('matches', []): family = matches._families['families'][match['complete_phone_key']] literals = family['literals'] if match['ambiguous'] or match['uncertain_literal_mapping'] or len(literals) != 1: continue name = literals[0] if name not in pool['minimum_row_per_canonical']: continue c = pool['minimum_row_per_canonical'][name] located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256']) act = candidate_action(c, located, tail, offset, s, True, len(selected)) if act is None or not eligible: continue p = act['proposal'] if (p['start'], p['end'], p['replacement']) in primary_patches: continue fallback.append((p['start'], -(p['end'] - p['start']), hashlib.sha256(c['canonical'].encode()).hexdigest(), c['index'], act)) for order, item in enumerate(sorted(fallback, key=lambda x: x[:4])): item[-1]['original_order'] = len(selected) + order actions.append(item[-1]) state.candidates(actions) shortlist_by_row = {c['index']: (i, c) for i, c in enumerate(allrows)} top = sorted((float(c['ranker_score']) for c in allrows), reverse=True) gap = top[0] - top[1] if len(top) > 1 else 0 none = float(ev['ranker_none']) if ev is not None else 0 valid_spans = float(ev['valid_span_count']) if ev is not None else 0 gate = [] for a in actions: p = a['proposal'] l = a['location_audit'] record = shortlist_by_row.get(a['original_pool_row']) available = record is not None score = record[1] if available else {} bs = float(score.get('ranker_score', 0)) ac = float(score.get('acoustic_score', 0)) co = float(score.get('coarse', 0)) rank = a['original_selected_rank'] extra = [bool(matches.local_valid), bool(l['unique_original_character_position']), bool(l['family_ambiguous']), l['family_canonical_count'] or 0, len(a['candidate']['canonical']), len(p['old_literal']), bool(l['literal_in_local_phone_family']), bool(l['original_unique_literal_mismatch']), len(p['left_anchor']), len(p['right_anchor']), (p['start'] - offset) / 64, (p['end'] - offset) / 64, bs, none, bs - none if available and ev is not None else 0, ac, co, rank / 10 if rank is not None else 1, gap, valid_spans / 4, math.log1p(pool['unique_surfaces']), a.pop('_consecutive'), a.pop('_anchor_consecutive'), a['source_local_fallback'], available, bool(l['family_uncertain']), 0, record[0] / max(1, len(allrows)) if available else 1] need(len(extra) == 28, 'exact Gate28 suffix') gate.append(entry + list(map(f32, extra))) a.update(candidate_R8_scores_available=available, original_candidate_pool=origin if available else case['pool'], active_overlap_not_a_model_feature=True) receipt = dict(schema=SCHEMA + '_decision', observed_version=s['observed_version'], is_final=False, audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], raw_partial_event_ref=s.get('raw_partial_event_ref'), tail_start_absolute_character=offset, tail_character_count=len(tail), local_mapping_audit=matches.audit(), candidate_audit=audit, action_count=len(actions), source_lookahead_within1s=eligible, NOCHANGE_has_no_action_row_and_does_not_block_other_mismatch=True, reserved_Gate_column44=0) return (entry, gate, actions, receipt)