File size: 10,547 Bytes
49d63ea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
"""Complete trained GeoPOI model with an offline causal-prefix HF backend."""
from __future__ import annotations
import json
from pathlib import Path
import threading
import time
import numpy as np
import soundfile as sf
import torch
from components.router import AudioSemanticRouter
from .regional import RegionalAdapter
from .cpu import CompleteCPU


class GeoPOI:
    """All learned components; the released directory alone contains 500 POIs.

    The offline HTTP caller provides one complete WAV. This backend evaluates
    real causal audio prefixes with the trained decision protocol. It recomputes
    each prefix with Transformers rather than reusing the production vLLM cache.
    """

    def __init__(self, directory, *, device='cuda'):
        from transformers import AutoProcessor, Qwen3ASRForConditionalGeneration
        self.directory = Path(directory)
        self.device = torch.device(device)
        self.dtype = torch.bfloat16 if self.device.type == 'cuda' else torch.float32
        base = self.directory / 'base_model'
        self.processor = AutoProcessor.from_pretrained(base, local_files_only=True, trust_remote_code=False)
        self.model = Qwen3ASRForConditionalGeneration.from_pretrained(base, dtype=self.dtype,
            attn_implementation='sdpa', local_files_only=True, trust_remote_code=False)
        self.model.to(self.device).eval().requires_grad_(False)
        self.adapter = RegionalAdapter.from_pretrained(self.directory / 'adapter', device=self.device)
        self.adapter.attach(self.model)
        self.router = AudioSemanticRouter(self.directory / 'components/router', 'cpu')
        self.cpu = CompleteCPU(self.directory)
        self.region_to_id = {value: i + 1 for i, value in enumerate(self.adapter.regions)}
        self._current = None
        self._projector_hook = self.model.model.multi_modal_projector.register_forward_hook(self._projected)
        self._lock = threading.Lock()
        self.last_trace = None
        self.loaded_components = ['Qwen3-ASR', 'regional_adapter', 'reader', 'span', 'retriever',
            'ranker', 'Entry', 'benefit', 'harm', 'gain', 'Hcal3', 'B49cost3', 'audio_semantic_router']

    @classmethod
    def from_pretrained(cls, path_or_repo, *, device='cuda', revision=None, local_files_only=False):
        directory = Path(path_or_repo)
        if not directory.is_dir():
            from huggingface_hub import snapshot_download
            directory = Path(snapshot_download(str(path_or_repo), revision=revision,
                local_files_only=local_files_only, allow_patterns=['base_model/*', 'adapter/*',
                    'components/*', 'assets/*', 'pois.json']))
        return cls(directory, device=device)

    def _context(self, context, poi_id):
        if context is not None and poi_id is not None:
            raise ValueError('Supply one observed context or one released context ID')
        if poi_id is not None:
            context = self.cpu.catalog.pois[poi_id]
        if context is None:
            return None, []
        if not isinstance(context, dict) or type(context.get('region_id')) is not int:
            raise ValueError('Context needs an integer region_id')
        center = context['region_id']
        neighbors = list(context.get('neighborhood_region_ids', [center]))
        if center not in neighbors or len(neighbors) != len(set(neighbors)) or len(neighbors) > 9:
            raise ValueError('A context needs its center and at most nine distinct regions')
        if any(type(i) is not int or not 1 <= i <= len(self.adapter.regions) for i in neighbors):
            raise ValueError('Context outside the trained regional bank')
        return self.adapter.regions[center - 1], [self.adapter.regions[i - 1] for i in neighbors]

    def _projected(self, module, inputs, output):
        current = self._current
        if current is None:
            return output
        if current.get('projection_seen'):
            raise RuntimeError('One real audio projection is expected per INPUT')
        frames = output
        if frames.ndim == 3 and frames.shape[0] == 1:
            frames = frames[0]
        if frames.ndim != 2 or frames.shape[1] != 2048:
            raise ValueError('Expected actual projected audio states [T,2048]')
        current['projection_seen'] = True
        frames = frames.detach().to('cpu').contiguous()
        selection = self.router.select(frames, current['allowed_cells'], current['center'])
        scope = selection['cells']
        if current['allowed_cells']:
            current['router_calls'] += 1
        pool, ref, metadata = self.cpu.catalog.pool(scope)
        evidence = self.cpu.evidence(frames, pool, ref)
        permission = current['session'].begin_input(current['sequence'], evidence, self.region_to_id)
        route = permission.native_route
        if route is not None and permission.enabled:
            self.adapter._ids = [list(route['region_ids'])]
            self.adapter._weights = [list(route['weights'])] if route['weights'] is not None else None
        else:
            self.adapter._ids = self.adapter._weights = None
        current.update(scope=scope, pool_ref=ref, metadata=metadata, evidence=evidence,
            router=selection, permission_reason=permission.reason,
            adapter_permission=bool(permission.enabled), projected_frames=len(frames))
        return output

    def _generate_prefix(self, audio, *, max_new_tokens):
        messages = [
            {'role': 'system', 'content': [{'type': 'text', 'text': '请将语音转写为中文文本,只输出转写内容。'}]},
            {'role': 'user', 'content': [{'type': 'audio', 'audio': audio}]},
        ]
        batch = self.processor.apply_chat_template([messages], tokenize=True, return_dict=True,
            add_generation_prompt=True, processor_kwargs={'output_labels': False,
                'text_kwargs': {'padding': True, 'padding_side': 'left'},
                'audio_kwargs': {'sampling_rate': 16000, 'padding': True}})
        keys = ('input_ids', 'attention_mask', 'input_features', 'input_features_mask')
        inputs = {key: batch[key].to(device=self.device, dtype=self.dtype) if key == 'input_features'
                  else batch[key].to(self.device) for key in keys}
        width = inputs['input_ids'].shape[-1]
        with self.adapter.route(None):
            generated = self.model.generate(**inputs, do_sample=False, num_beams=1,
                repetition_penalty=1.1, max_new_tokens=max_new_tokens, use_cache=True)
        return self.processor.decode(generated[0, width:].detach().cpu(), return_format='transcription_only').strip()

    def transcribe(self, audio_file, *, context=None, poi_id=None, max_new_tokens=128):
        """Transcribe one complete mono 16 kHz WAV through the complete model."""
        if type(max_new_tokens) is not int or not 1 <= max_new_tokens <= 512:
            raise ValueError('Invalid decoding token budget')
        audio, rate = sf.read(audio_file, dtype='float32', always_2d=False)
        if rate != 16000 or audio.ndim != 1 or not 0 < len(audio) <= rate * 60 or not np.isfinite(audio).all():
            raise ValueError('Provide finite mono16k WAV audio of at most60seconds')
        center, cells = self._context(context, poi_id)
        stops = [min(len(audio), 8000)]
        while stops[-1] < len(audio):
            stops.append(min(len(audio), stops[-1] + 16000))
        with self._lock, torch.inference_mode():
            self.cpu.reset()
            session = self.cpu.session(cells)
            per_input = []
            raw_text = ''
            began = time.perf_counter()
            try:
                for sequence, stop in enumerate(stops):
                    current = dict(sequence=sequence, session=session, center=center,
                        allowed_cells=cells, projection_seen=False, router_calls=0)
                    self._current = current
                    raw_text = self._generate_prefix(np.ascontiguousarray(audio[:stop]), max_new_tokens=max_new_tokens)
                    if not current['projection_seen']:
                        raise RuntimeError('Actual audio projection was not observed')
                    eof = stop == len(audio)
                    audio_us = int(round(stop / rate * 1e6))
                    session.partial(sequence, raw_text, audio_us, eof, current['scope'],
                        current['pool_ref'], current['metadata'], current['evidence'])
                    per_input.append(dict(sequence=sequence, audio_end_samples=stop,
                        source_audio_eof=eof, projected_frames=current['projected_frames'],
                        router=current['router'], pool_rows=len(current['metadata']['rows']),
                        permission_reason=current['permission_reason'],
                        adapter_permission=current['adapter_permission'],
                        adapter=dict(self.adapter.last_forward_stats)))
                result = session.done(len(stops), raw_text, int(round(len(audio) / rate * 1e6)))
            finally:
                self._current = None
                self.adapter._ids = self.adapter._weights = None
            trace = dict(loaded_components=list(self.loaded_components),
                all_regional_modules_loaded=len(self.adapter.regions),
                candidate_calls=dict(self.cpu.calls), decision_calls=dict(self.cpu.policy.calls),
                audio_router_calls=sum(bool(row['router'].get('probabilities')) for row in per_input),
                asr_passes=len(stops), projected_passes=len(per_input), inputs=per_input,
                decision_inputs=session.trace,
                first_input_adapter_off=not per_input[0]['adapter_permission'],
                adapter_active_input_count=sum(row['adapter']['nonzero_residual_calls'] > 0 for row in per_input),
                source_EOF_and_ASR_done_separate=True,
                raw_ASR_never_receives_corrected_text=True,
                learned_heads_retrained=False,
                execution_backend='Transformers causal prefixes; recomputed audio prefixes, not native vLLM KV reuse',
                full_WAV_offline=True, elapsed_seconds=time.perf_counter() - began)
            self.last_trace = trace
            return dict(text=result['corrected_final_text'], raw_text=result['raw_final_text'],
                trace=trace, asr_passes=len(stops), full_WAV_offline=True)

    def close(self):
        self.cpu.close()
        self._projector_hook.remove()


GeoPOIComplete = GeoPOI