"""Complete trained GeoPOI model with an offline causal-prefix HF backend.""" from __future__ import annotations import json from pathlib import Path import threading import time import numpy as np import soundfile as sf import torch from components.router import AudioSemanticRouter from .regional import RegionalAdapter from .cpu import CompleteCPU class GeoPOI: """All learned components; the released directory alone contains 500 POIs. The offline HTTP caller provides one complete WAV. This backend evaluates real causal audio prefixes with the trained decision protocol. It recomputes each prefix with Transformers rather than reusing the production vLLM cache. """ def __init__(self, directory, *, device='cuda'): from transformers import AutoProcessor, Qwen3ASRForConditionalGeneration self.directory = Path(directory) self.device = torch.device(device) self.dtype = torch.bfloat16 if self.device.type == 'cuda' else torch.float32 base = self.directory / 'base_model' self.processor = AutoProcessor.from_pretrained(base, local_files_only=True, trust_remote_code=False) self.model = Qwen3ASRForConditionalGeneration.from_pretrained(base, dtype=self.dtype, attn_implementation='sdpa', local_files_only=True, trust_remote_code=False) self.model.to(self.device).eval().requires_grad_(False) self.adapter = RegionalAdapter.from_pretrained(self.directory / 'adapter', device=self.device) self.adapter.attach(self.model) self.router = AudioSemanticRouter(self.directory / 'components/router', 'cpu') self.cpu = CompleteCPU(self.directory) self.region_to_id = {value: i + 1 for i, value in enumerate(self.adapter.regions)} self._current = None self._projector_hook = self.model.model.multi_modal_projector.register_forward_hook(self._projected) self._lock = threading.Lock() self.last_trace = None self.loaded_components = ['Qwen3-ASR', 'regional_adapter', 'reader', 'span', 'retriever', 'ranker', 'Entry', 'benefit', 'harm', 'gain', 'Hcal3', 'B49cost3', 'audio_semantic_router'] @classmethod def from_pretrained(cls, path_or_repo, *, device='cuda', revision=None, local_files_only=False): directory = Path(path_or_repo) if not directory.is_dir(): from huggingface_hub import snapshot_download directory = Path(snapshot_download(str(path_or_repo), revision=revision, local_files_only=local_files_only, allow_patterns=['base_model/*', 'adapter/*', 'components/*', 'assets/*', 'pois.json'])) return cls(directory, device=device) def _context(self, context, poi_id): if context is not None and poi_id is not None: raise ValueError('Supply one observed context or one released context ID') if poi_id is not None: context = self.cpu.catalog.pois[poi_id] if context is None: return None, [] if not isinstance(context, dict) or type(context.get('region_id')) is not int: raise ValueError('Context needs an integer region_id') center = context['region_id'] neighbors = list(context.get('neighborhood_region_ids', [center])) if center not in neighbors or len(neighbors) != len(set(neighbors)) or len(neighbors) > 9: raise ValueError('A context needs its center and at most nine distinct regions') if any(type(i) is not int or not 1 <= i <= len(self.adapter.regions) for i in neighbors): raise ValueError('Context outside the trained regional bank') return self.adapter.regions[center - 1], [self.adapter.regions[i - 1] for i in neighbors] def _projected(self, module, inputs, output): current = self._current if current is None: return output if current.get('projection_seen'): raise RuntimeError('One real audio projection is expected per INPUT') frames = output if frames.ndim == 3 and frames.shape[0] == 1: frames = frames[0] if frames.ndim != 2 or frames.shape[1] != 2048: raise ValueError('Expected actual projected audio states [T,2048]') current['projection_seen'] = True frames = frames.detach().to('cpu').contiguous() selection = self.router.select(frames, current['allowed_cells'], current['center']) scope = selection['cells'] if current['allowed_cells']: current['router_calls'] += 1 pool, ref, metadata = self.cpu.catalog.pool(scope) evidence = self.cpu.evidence(frames, pool, ref) permission = current['session'].begin_input(current['sequence'], evidence, self.region_to_id) route = permission.native_route if route is not None and permission.enabled: self.adapter._ids = [list(route['region_ids'])] self.adapter._weights = [list(route['weights'])] if route['weights'] is not None else None else: self.adapter._ids = self.adapter._weights = None current.update(scope=scope, pool_ref=ref, metadata=metadata, evidence=evidence, router=selection, permission_reason=permission.reason, adapter_permission=bool(permission.enabled), projected_frames=len(frames)) return output def _generate_prefix(self, audio, *, max_new_tokens): messages = [ {'role': 'system', 'content': [{'type': 'text', 'text': '请将语音转写为中文文本,只输出转写内容。'}]}, {'role': 'user', 'content': [{'type': 'audio', 'audio': audio}]}, ] batch = self.processor.apply_chat_template([messages], tokenize=True, return_dict=True, add_generation_prompt=True, processor_kwargs={'output_labels': False, 'text_kwargs': {'padding': True, 'padding_side': 'left'}, 'audio_kwargs': {'sampling_rate': 16000, 'padding': True}}) keys = ('input_ids', 'attention_mask', 'input_features', 'input_features_mask') inputs = {key: batch[key].to(device=self.device, dtype=self.dtype) if key == 'input_features' else batch[key].to(self.device) for key in keys} width = inputs['input_ids'].shape[-1] with self.adapter.route(None): generated = self.model.generate(**inputs, do_sample=False, num_beams=1, repetition_penalty=1.1, max_new_tokens=max_new_tokens, use_cache=True) return self.processor.decode(generated[0, width:].detach().cpu(), return_format='transcription_only').strip() def transcribe(self, audio_file, *, context=None, poi_id=None, max_new_tokens=128): """Transcribe one complete mono 16 kHz WAV through the complete model.""" if type(max_new_tokens) is not int or not 1 <= max_new_tokens <= 512: raise ValueError('Invalid decoding token budget') audio, rate = sf.read(audio_file, dtype='float32', always_2d=False) if rate != 16000 or audio.ndim != 1 or not 0 < len(audio) <= rate * 60 or not np.isfinite(audio).all(): raise ValueError('Provide finite mono16k WAV audio of at most60seconds') center, cells = self._context(context, poi_id) stops = [min(len(audio), 8000)] while stops[-1] < len(audio): stops.append(min(len(audio), stops[-1] + 16000)) with self._lock, torch.inference_mode(): self.cpu.reset() session = self.cpu.session(cells) per_input = [] raw_text = '' began = time.perf_counter() try: for sequence, stop in enumerate(stops): current = dict(sequence=sequence, session=session, center=center, allowed_cells=cells, projection_seen=False, router_calls=0) self._current = current raw_text = self._generate_prefix(np.ascontiguousarray(audio[:stop]), max_new_tokens=max_new_tokens) if not current['projection_seen']: raise RuntimeError('Actual audio projection was not observed') eof = stop == len(audio) audio_us = int(round(stop / rate * 1e6)) session.partial(sequence, raw_text, audio_us, eof, current['scope'], current['pool_ref'], current['metadata'], current['evidence']) per_input.append(dict(sequence=sequence, audio_end_samples=stop, source_audio_eof=eof, projected_frames=current['projected_frames'], router=current['router'], pool_rows=len(current['metadata']['rows']), permission_reason=current['permission_reason'], adapter_permission=current['adapter_permission'], adapter=dict(self.adapter.last_forward_stats))) result = session.done(len(stops), raw_text, int(round(len(audio) / rate * 1e6))) finally: self._current = None self.adapter._ids = self.adapter._weights = None trace = dict(loaded_components=list(self.loaded_components), all_regional_modules_loaded=len(self.adapter.regions), candidate_calls=dict(self.cpu.calls), decision_calls=dict(self.cpu.policy.calls), audio_router_calls=sum(bool(row['router'].get('probabilities')) for row in per_input), asr_passes=len(stops), projected_passes=len(per_input), inputs=per_input, decision_inputs=session.trace, first_input_adapter_off=not per_input[0]['adapter_permission'], adapter_active_input_count=sum(row['adapter']['nonzero_residual_calls'] > 0 for row in per_input), source_EOF_and_ASR_done_separate=True, raw_ASR_never_receives_corrected_text=True, learned_heads_retrained=False, execution_backend='Transformers causal prefixes; recomputed audio prefixes, not native vLLM KV reuse', full_WAV_offline=True, elapsed_seconds=time.perf_counter() - began) self.last_trace = trace return dict(text=result['corrected_final_text'], raw_text=result['raw_final_text'], trace=trace, asr_passes=len(stops), full_WAV_offline=True) def close(self): self.cpu.close() self._projector_hook.remove() GeoPOIComplete = GeoPOI