Download runtime/causal.py from guga-guga/GeoPOI: direct link, hf CLI and curl.
- Browser
- Download file 22.7 kB
-
https://huggingface.co/guga-guga/GeoPOI/resolve/main/runtime/causal.py
- Command line
-
hf download hf://guga-guga/GeoPOI/runtime/causal.py
-
curl -L -o causal.py https://huggingface.co/guga-guga/GeoPOI/resolve/main/runtime/causal.py
22.7 kB
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import collections,hashlib,json,math,struct | |
| SCHEMA = 'R8_service_clock_Entry18_Gate46_causal_helper_v1' | |
| MAX_LOOKAHEAD_US = 1000000 | |
| ENTRY_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64'] | |
| GATE_NAMES = ['audio_time_s', 'received_minus_audio_s', 'source_audio_eof', 'tail_character_count_div64', 'local_mapping_valid', 'local_matching_span_count', 'local_unique_literal_mismatch_count', 'local_literal_already_present_count', 'local_ambiguous_span_count', 'local_uncertain_span_count', 'local_min_match_characters_or0', 'local_max_match_characters_or0', 'local_max_family_canonical_count_or0', 'mapped_region_count', 'local_query_completed', 'same_absolute_position_tail_stability_fraction', 'maximum_unique_mismatch_consecutive_versions_cap8', 'partial_observed_version_cap64', 'candidate_local_mapping_valid', 'candidate_unique_original_position', 'candidate_family_ambiguous', 'candidate_family_canonical_count', 'candidate_canonical_character_count', 'candidate_old_literal_character_count', 'candidate_current_literal_in_local_phone_family', 'candidate_unique_literal_mismatch', 'candidate_left_anchor_characters', 'candidate_right_anchor_characters', 'candidate_start_in_tail_div64', 'candidate_end_in_tail_div64', 'candidate_ranker_score_or0', 'ranker_none_or0', 'candidate_ranker_minus_none_or0', 'candidate_reader_acoustic_score_or0', 'candidate_retriever_coarse_score_or0', 'original_selected_rank_div10_or1', 'ranker_top1_minus_top2_or0', 'valid_span_count_div4', 'pool_unique_surfaces_log1p', 'candidate_consecutive_versions_cap8', 'candidate_anchor_consecutive_versions_cap8', 'candidate_source_local_fallback', 'candidate_R8_scores_available', 'candidate_phonefamily_uncertain', 'reserved_policy_independent_overlap_zero', 'candidate_original_shortlist_rank_fraction_or1'] | |
| CANONICAL_KEYS = ('candidate_id', 'variant_id', 'lexical_id', 'lexical_entity_id', 'cell', 'surface_id', 'canonical', 'decoder_token_ids') | |
| def need(v, m): | |
| if not v: | |
| raise ValueError(m) | |
| def digest(v): | |
| return hashlib.sha256(json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode()).hexdigest() | |
| def f32(v): | |
| v = float(v) | |
| need(math.isfinite(v), 'finite feature') | |
| return struct.unpack('<f', struct.pack('<f', v))[0] | |
| def integer(v, name): | |
| if type(v) is not int or v < 0: | |
| raise ValueError(name + ' must be a nonnegative integer') | |
| def text(v, name): | |
| if type(v) is not str: | |
| raise TypeError(name + ' must be raw Unicode text') | |
| v.encode('utf-8', errors='strict') | |
| class RawSnapshot: | |
| version: int | |
| audio_time_us: int | |
| text: str | |
| source_audio_eof: bool | |
| asr_done: bool | |
| def __post_init__(self): | |
| integer(self.version, 'version') | |
| integer(self.audio_time_us, 'audio_time_us') | |
| text(self.text, 'text') | |
| if type(self.source_audio_eof) is not bool or type(self.asr_done) is not bool: | |
| raise TypeError('separate actual source EOF and ASR done booleans required') | |
| class LiteralProposal: | |
| observed_version: int | |
| start: int | |
| end: int | |
| old_literal: str | |
| replacement: str | |
| left_anchor: str | |
| right_anchor: str | |
| def __post_init__(self): | |
| for key in ('observed_version', 'start', 'end'): | |
| integer(getattr(self, key), key) | |
| for key in ('old_literal', 'replacement', 'left_anchor', 'right_anchor'): | |
| text(getattr(self, key), key) | |
| if not self.old_literal or not self.replacement or self.end <= self.start or (self.end - self.start != len(self.old_literal)): | |
| raise ValueError('exact nonempty observed character span and replacement required') | |
| class FrozenPatch: | |
| sequence: int | |
| proposal: LiteralProposal | |
| proposal_audio_time_us: int | |
| decision_audio_time_us: int | |
| decision_version: int | |
| class FreezeDecision: | |
| accepted: bool | |
| reason: str | |
| patch: FrozenPatch | None = None | |
| class PatchOutcome: | |
| sequence: int | |
| applied: bool | |
| reason: str | |
| class RenderResult: | |
| raw_final_text: str | |
| corrected_text: str | |
| outcomes: tuple | |
| outside_applied_spans_byte_exact: bool | |
| class PartialRenderResult: | |
| observed_version: int | |
| raw_partial_text: str | |
| corrected_text: str | |
| outcomes: tuple | |
| outside_applied_spans_byte_exact: bool | |
| def matches(raw, p): | |
| return p.start >= len(p.left_anchor) and p.end + len(p.right_anchor) <= len(raw) and (raw[p.start:p.end] == p.old_literal) and (raw[p.start - len(p.left_anchor):p.start] == p.left_anchor) and (raw[p.end:p.end + len(p.right_anchor)] == p.right_anchor) | |
| def overlap(a, b): | |
| return max(a.start, b.start) < min(a.end, b.end) | |
| class ActiveLiteralPatcher: | |
| def __init__(self): | |
| self._snapshots = [] | |
| self._versions = {} | |
| self._received_audio_us = 0 | |
| self._source_eof_seen = False | |
| self._asr_done_seen = False | |
| self._patches = [] | |
| self._invalidated = {} | |
| self._rendered = False | |
| def raw_snapshots(self): | |
| return tuple(self._snapshots) | |
| def frozen_patches(self): | |
| return tuple(self._patches) | |
| def observe_raw(self, snapshot, *, received_audio_time_us): | |
| if type(snapshot) is not RawSnapshot: | |
| raise TypeError('only original RawSnapshot inputs, never rendered results') | |
| integer(received_audio_time_us, 'received_audio_time_us') | |
| if self._rendered or self._asr_done_seen: | |
| raise ValueError('ASR done ledger cannot accept more raw snapshots') | |
| if snapshot.audio_time_us > received_audio_time_us or received_audio_time_us < self._received_audio_us: | |
| raise ValueError('future or backward received-audio clock') | |
| if self._snapshots: | |
| prior = self._snapshots[-1] | |
| if snapshot.version <= prior.version or snapshot.audio_time_us < prior.audio_time_us: | |
| raise ValueError('raw versions and audio clocks must be ordered') | |
| if self._source_eof_seen and (not snapshot.source_audio_eof): | |
| raise ValueError('actual physical EOF flag cannot regress') | |
| self._snapshots.append(snapshot) | |
| self._versions[snapshot.version] = snapshot | |
| self._received_audio_us = received_audio_time_us | |
| self._source_eof_seen |= snapshot.source_audio_eof | |
| self._asr_done_seen |= snapshot.asr_done | |
| for p in self._patches: | |
| if not matches(snapshot.text, p.proposal): | |
| self._invalidated.setdefault(p.sequence, 'RAW_REVISION_INVALIDATED_ANCHOR') | |
| def freeze(self, proposal, *, source_audio_eof, asr_done): | |
| if type(proposal) is not LiteralProposal or type(source_audio_eof) is not bool or type(asr_done) is not bool: | |
| raise TypeError('fixed proposal and two explicit actual lifecycle booleans required') | |
| if self._rendered or self._asr_done_seen or asr_done: | |
| return FreezeDecision(False, 'ASR_DONE_OR_FINAL_ALREADY_OBSERVED') | |
| if not self._snapshots or proposal.observed_version not in self._versions: | |
| return FreezeDecision(False, 'PROPOSAL_SNAPSHOT_NOT_OBSERVED') | |
| current = self._snapshots[-1] | |
| if source_audio_eof != current.source_audio_eof or asr_done != current.asr_done: | |
| raise ValueError('physical EOF and ASR done must match actual current snapshot') | |
| original = self._versions[proposal.observed_version] | |
| if not 0 <= self._received_audio_us - original.audio_time_us <= MAX_LOOKAHEAD_US: | |
| return FreezeDecision(False, 'PROPOSAL_WINDOW_EXCEEDED_ONE_SECOND') | |
| if not matches(original.text, proposal): | |
| return FreezeDecision(False, 'PROPOSAL_NOT_PRESENT_AT_OBSERVATION') | |
| if not proposal.left_anchor and proposal.start != 0 or (not proposal.right_anchor and proposal.end != len(original.text)): | |
| return FreezeDecision(False, 'EMPTY_ANCHOR_ONLY_AT_OBSERVED_TEXT_BOUNDARY') | |
| if any((not matches(s.text, proposal) for s in self._snapshots if s.version >= original.version)): | |
| return FreezeDecision(False, 'PROPOSAL_INVALIDATED_BEFORE_FREEZE') | |
| if proposal.replacement == proposal.old_literal: | |
| return FreezeDecision(False, 'NO_CHANGE') | |
| if any((overlap(proposal, p.proposal) for p in self._patches if p.sequence not in self._invalidated)): | |
| return FreezeDecision(False, 'OVERLAP_FIRST_FREEZE_WINS') | |
| p = FrozenPatch(len(self._patches), proposal, original.audio_time_us, self._received_audio_us, current.version) | |
| self._patches.append(p) | |
| return FreezeDecision(True, 'FROZEN_BEFORE_ASR_DONE', p) | |
| def render_current_partial(self): | |
| if self._rendered or self._asr_done_seen or (not self._snapshots): | |
| raise ValueError('observed nonfinal ASR partial required') | |
| s = self._snapshots[-1] | |
| corrected, outcomes = self._render_frozen(s) | |
| return PartialRenderResult(s.version, s.text, corrected, outcomes, True) | |
| def render_final(self, snapshot, *, received_audio_time_us): | |
| if type(snapshot) is not RawSnapshot or not snapshot.asr_done: | |
| raise ValueError('actual transcription.done snapshot required') | |
| self.observe_raw(snapshot, received_audio_time_us=received_audio_time_us) | |
| corrected, outcomes = self._render_frozen(snapshot, final_version=snapshot.version) | |
| self._rendered = True | |
| return RenderResult(snapshot.text, corrected, outcomes, True) | |
| def _render_frozen(self, snapshot, *, final_version=None): | |
| raw = snapshot.text | |
| applied = [] | |
| outcomes = [] | |
| for patch in self._patches: | |
| reason = self._invalidated.get(patch.sequence) | |
| if final_version is not None and patch.decision_version >= final_version: | |
| reason = 'DECISION_VERSION_NOT_BEFORE_ASR_DONE' | |
| if reason is None and (not matches(raw, patch.proposal)): | |
| reason = 'FINAL_FIXED_POSITION_OR_ANCHOR_MISMATCH' if final_version is not None else 'CURRENT_FIXED_POSITION_OR_ANCHOR_MISMATCH' | |
| if reason is None: | |
| applied.append(patch) | |
| outcomes.append(PatchOutcome(patch.sequence, reason is None, reason or 'APPLIED')) | |
| corrected = raw | |
| for patch in sorted(applied, key=lambda p: p.proposal.start, reverse=True): | |
| p = patch.proposal | |
| corrected = corrected[:p.start] + p.replacement + corrected[p.end:] | |
| actual = corrected.encode('utf-8') | |
| raw_cursor = output_cursor = 0 | |
| for patch in sorted(applied, key=lambda p: p.proposal.start): | |
| p = patch.proposal | |
| unchanged = raw[raw_cursor:p.start].encode('utf-8') | |
| replacement = p.replacement.encode('utf-8') | |
| if actual[output_cursor:output_cursor + len(unchanged)] != unchanged: | |
| raise AssertionError('outside frozen raw spans changed') | |
| output_cursor += len(unchanged) | |
| if actual[output_cursor:output_cursor + len(replacement)] != replacement: | |
| raise AssertionError('fixed replacement differs') | |
| output_cursor += len(replacement) | |
| raw_cursor = p.end | |
| if actual[output_cursor:] != raw[raw_cursor:].encode('utf-8'): | |
| raise AssertionError('outside frozen raw spans changed') | |
| return (corrected, tuple(outcomes)) | |
| def active_overlap(ledger, proposal): | |
| return any((p.sequence not in ledger._invalidated and overlap(proposal, p.proposal) for p in ledger.frozen_patches)) | |
| def candidate_evidence(case, snapshot, value, pool): | |
| if value is None: | |
| return (None, [], [], None) | |
| if 'evidence' in value: | |
| e = value['evidence'] | |
| else: | |
| e = value | |
| row = value.get('original_physical_decision', value) | |
| for key in ('observed_version', 'audio_time_us', 'received_audio_time_us', 'raw_partial_event_ref'): | |
| if key in row: | |
| need(row[key] == snapshot[key], 'candidate uses this actual observed partial ' + key) | |
| if 'logical_original_global_row_index' in value: | |
| need(value['logical_original_global_row_index'] == case['original_global_row_index'], 'logical view candidate identity') | |
| origin = value.get('logical_pool', value.get('original_pool', case['pool'])) | |
| need(origin['sha256'] == case['pool']['sha256'] and origin['size'] == case['pool']['size'], 'same original logical pool body') | |
| allrows = e['all_original_shortlist'] | |
| selected = e['selected'] | |
| need(type(allrows) is list and type(selected) is list and (len(selected) <= 10) and (e['whole_selected_list_preserved'] is True), 'whole original shortlist and orderedtop10') | |
| for c in allrows: | |
| i = c['index'] | |
| need(type(i) is int and 0 <= i < len(pool['rows']) and all((c[k] == pool['rows'][i][k] for k in CANONICAL_KEYS)), 'all candidate canonical/token/cell/row lineage is original pool') | |
| identity = {k: c[k] for k in CANONICAL_KEYS} | |
| need(c['candidate_identity_sha256'] == digest(identity) and c['candidate_binding_sha256'] == digest(dict(original_pool=origin, original_pool_row=i, identity=identity)), 'original frozen candidate SHA binding') | |
| known = {c['candidate_binding_sha256']: c for c in allrows} | |
| need(all((c['candidate_binding_sha256'] in known and known[c['candidate_binding_sha256']] == c for c in selected)), 'selected exact original shortlist records in order') | |
| return (e, allrows, selected, origin) | |
| class CaseFeatureState: | |
| def __init__(self): | |
| self.previous_tail = '' | |
| self.previous_offset = 0 | |
| self.have_previous = False | |
| self.mismatch_counts = {} | |
| self.candidate_counts = {} | |
| self.anchor_counts = {} | |
| def entry(self, s, text, matches): | |
| offset = max(0, len(text) - 64) | |
| tail = text[offset:] | |
| analysis = matches._analysis.get('matches', []) | |
| lengths = [x['stop_character'] - x['start_character'] for x in analysis] | |
| unique = {digest([offset + x['start_character'], offset + x['stop_character'], tail[x['start_character']:x['stop_character']], x['complete_phone_key']]) for x in analysis if x['unique_literal_mismatch']} | |
| current = {k: min(8, self.mismatch_counts.get(k, 0) + 1) for k in unique} | |
| stable = 0 | |
| if self.have_previous: | |
| for i, ch in enumerate(tail): | |
| j = offset + i - self.previous_offset | |
| stable += int(0 <= j < len(self.previous_tail) and self.previous_tail[j] == ch) | |
| values = [s['audio_time_us'] / 1000000.0, (s['received_audio_time_us'] - s['audio_time_us']) / 1000000.0, bool(s['eos_observed']), len(tail) / 64, bool(matches.local_valid), len(analysis), sum((bool(x['unique_literal_mismatch']) for x in analysis)), sum((bool(x['literal_in_local_phone_family']) for x in analysis)), sum((bool(x['ambiguous']) for x in analysis)), sum((bool(x['uncertain_literal_mapping']) for x in analysis)), min(lengths, default=0), max(lengths, default=0), max((x['local_distinct_canonical_count'] for x in analysis), default=0), len(matches._query_proof.get('scope', {}).get('mapped_region_ids', [])), bool(matches._query_proof), stable / len(tail) if tail and self.have_previous else 0, max(current.values(), default=0), min(s['observed_version'], 64)] | |
| self.previous_tail = tail | |
| self.previous_offset = offset | |
| self.have_previous = True | |
| self.mismatch_counts = current | |
| return (list(map(f32, values)), tail, offset) | |
| def candidates(self, actions): | |
| current = {} | |
| anchors = {} | |
| for a in actions: | |
| p = a['proposal'] | |
| key = digest([p['start'], p['end'], p['old_literal'], p['replacement'], a['candidate_identity_sha256']]) | |
| ak = digest([key, p['left_anchor'], p['right_anchor']]) | |
| a['_consecutive'] = min(8, self.candidate_counts.get(key, 0) + 1) | |
| a['_anchor_consecutive'] = min(8, self.anchor_counts.get(ak, 0) + 1) | |
| current[key] = a['_consecutive'] | |
| anchors[ak] = a['_anchor_consecutive'] | |
| self.candidate_counts = current | |
| self.anchor_counts = anchors | |
| def candidate_action(candidate, located, tail, offset, s, source, order): | |
| if not located.valid: | |
| return None | |
| start, end = (located.start, located.end) | |
| if offset > 0 and start == 0: | |
| return None | |
| left = tail[max(0, start - 1):start] | |
| right = tail[end:end + 1] | |
| p = LiteralProposal(s['observed_version'], offset + start, offset + end, located.old_literal, located.replacement, left, right) | |
| if p.old_literal == p.replacement: | |
| return None | |
| return dict(proposal=vars(p), candidate_identity_sha256=candidate['candidate_identity_sha256'], candidate_binding_sha256=candidate['candidate_binding_sha256'], original_pool_row=candidate['index'], candidate={k: candidate[k] for k in CANONICAL_KEYS}, location_audit=located.audit(), source_local_fallback=source, original_order=order, original_selected_rank=order if not source else None) | |
| def propose_snapshot(case, s, text, value, configuration, pool, state, front): | |
| need(s['is_final'] is False, 'done text never a decision input') | |
| need(type(s['eos_observed']) is bool and 0 <= s['audio_time_us'] <= s['received_audio_time_us'], 'actual causal source clock') | |
| tail = text[max(0, len(text) - 64):] | |
| matches = front.analyze_current_local_view(tail, observed_version=s['observed_version'], audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], asr_done=False, cells=case['geography']['cells']) | |
| entry, tail, offset = state.entry(s, text, matches) | |
| ev, allrows, selected, origin = candidate_evidence(case, s, value, pool) | |
| audit = [] | |
| actions = [] | |
| primary_patches = set() | |
| eligible = s['received_audio_time_us'] - s['audio_time_us'] <= MAX_LOOKAHEAD_US | |
| for rank, c in enumerate(selected): | |
| located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256']) | |
| a = located.audit() | |
| a.update(source_local_fallback=False, original_selected_rank=rank, NOCHANGE_is_baseline_not_truth_or_reservation=True) | |
| act = candidate_action(c, located, tail, offset, s, False, rank) | |
| if act is not None and eligible: | |
| actions.append(act) | |
| primary_patches.add((act['proposal']['start'], act['proposal']['end'], act['proposal']['replacement'])) | |
| a['eligible_action'] = act is not None and eligible | |
| audit.append(a) | |
| fallback = [] | |
| for match in matches._analysis.get('matches', []): | |
| family = matches._families['families'][match['complete_phone_key']] | |
| literals = family['literals'] | |
| if match['ambiguous'] or match['uncertain_literal_mapping'] or len(literals) != 1: | |
| continue | |
| name = literals[0] | |
| if name not in pool['minimum_row_per_canonical']: | |
| continue | |
| c = pool['minimum_row_per_canonical'][name] | |
| located = matches.locate_selected(c, candidate_binding_sha256=c['candidate_binding_sha256']) | |
| act = candidate_action(c, located, tail, offset, s, True, len(selected)) | |
| if act is None or not eligible: | |
| continue | |
| p = act['proposal'] | |
| if (p['start'], p['end'], p['replacement']) in primary_patches: | |
| continue | |
| fallback.append((p['start'], -(p['end'] - p['start']), hashlib.sha256(c['canonical'].encode()).hexdigest(), c['index'], act)) | |
| for order, item in enumerate(sorted(fallback, key=lambda x: x[:4])): | |
| item[-1]['original_order'] = len(selected) + order | |
| actions.append(item[-1]) | |
| state.candidates(actions) | |
| shortlist_by_row = {c['index']: (i, c) for i, c in enumerate(allrows)} | |
| top = sorted((float(c['ranker_score']) for c in allrows), reverse=True) | |
| gap = top[0] - top[1] if len(top) > 1 else 0 | |
| none = float(ev['ranker_none']) if ev is not None else 0 | |
| valid_spans = float(ev['valid_span_count']) if ev is not None else 0 | |
| gate = [] | |
| for a in actions: | |
| p = a['proposal'] | |
| l = a['location_audit'] | |
| record = shortlist_by_row.get(a['original_pool_row']) | |
| available = record is not None | |
| score = record[1] if available else {} | |
| bs = float(score.get('ranker_score', 0)) | |
| ac = float(score.get('acoustic_score', 0)) | |
| co = float(score.get('coarse', 0)) | |
| rank = a['original_selected_rank'] | |
| extra = [bool(matches.local_valid), bool(l['unique_original_character_position']), bool(l['family_ambiguous']), l['family_canonical_count'] or 0, len(a['candidate']['canonical']), len(p['old_literal']), bool(l['literal_in_local_phone_family']), bool(l['original_unique_literal_mismatch']), len(p['left_anchor']), len(p['right_anchor']), (p['start'] - offset) / 64, (p['end'] - offset) / 64, bs, none, bs - none if available and ev is not None else 0, ac, co, rank / 10 if rank is not None else 1, gap, valid_spans / 4, math.log1p(pool['unique_surfaces']), a.pop('_consecutive'), a.pop('_anchor_consecutive'), a['source_local_fallback'], available, bool(l['family_uncertain']), 0, record[0] / max(1, len(allrows)) if available else 1] | |
| need(len(extra) == 28, 'exact Gate28 suffix') | |
| gate.append(entry + list(map(f32, extra))) | |
| a.update(candidate_R8_scores_available=available, original_candidate_pool=origin if available else case['pool'], active_overlap_not_a_model_feature=True) | |
| receipt = dict(schema=SCHEMA + '_decision', observed_version=s['observed_version'], is_final=False, audio_time_us=s['audio_time_us'], received_audio_time_us=s['received_audio_time_us'], source_audio_eof=s['eos_observed'], raw_partial_event_ref=s.get('raw_partial_event_ref'), tail_start_absolute_character=offset, tail_character_count=len(tail), local_mapping_audit=matches.audit(), candidate_audit=audit, action_count=len(actions), source_lookahead_within1s=eligible, NOCHANGE_has_no_action_row_and_does_not_block_other_mismatch=True, reserved_Gate_column44=0) | |
| return (entry, gate, actions, receipt) | |