"""Track the appearance and position of one observed object in numerical memory.""" from collections import deque from dataclasses import dataclass import math import torch from torch import nn from torch.nn import functional as F @dataclass class ObjectState: identity: torch.Tensor recent: torch.Tensor position: torch.Tensor velocity: torch.Tensor class ObjectMemory(nn.Module): def __init__(self, grounder, input_width=64): super().__init__() self.grounder = grounder.eval().requires_grad_(False) self.input_width = input_width self.visual = nn.Sequential(nn.LayerNorm(input_width), nn.Linear(input_width, 80), nn.SiLU(), nn.Linear(80, 64)) self.sound = nn.Sequential(nn.LayerNorm(256), nn.Linear(256, 80), nn.SiLU(), nn.Linear(80, 64)) self.match = nn.Sequential(nn.Linear(6, 28), nn.SiLU(), nn.Linear(28, 1)) self.update = nn.Sequential(nn.Linear(4, 16), nn.SiLU(), nn.Linear(16, 1)) nn.init.zeros_(self.match[-1].weight) nn.init.zeros_(self.match[-1].bias) def encode(self, vision, audio): original = vision[..., :64].float() keys = F.normalize(F.normalize(original, dim=-1) + self.visual(vision.float()), dim=-1) query = F.normalize(self.sound(audio.float()), dim=-1) prior = F.normalize(self.grounder.query(audio.float()), dim=-1) reference = F.normalize(self.grounder.key(original), dim=-1) audio_logits = 7 * (keys * query[..., None, :]).sum(-1) + 12 * (reference * prior[..., None, :]).sum(-1) return keys, audio_logits @staticmethod def seed(keys, xy, weights): identity = F.normalize((weights[..., None] * keys).sum(1), dim=-1) position = (weights[..., None] * xy).sum(1) return ObjectState(identity, identity, position, torch.zeros_like(position)) def step(self, state, keys, audio_logits, xy, valid, elapsed=.5, *, static=False, appearance=True): predicted = (state.position + state.velocity * elapsed).clamp(0, 1) offset = xy - predicted[:, None] distance = offset.square().sum(-1) anchored = (keys * state.identity[:, None]).sum(-1) recent = (keys * state.recent[:, None]).sum(-1) if not appearance: anchored = recent = torch.zeros_like(anchored) features = torch.cat((offset, torch.stack((anchored, recent, distance, audio_logits / 16), dim=-1)), dim=-1) scores = (7 * anchored + 4 * recent - 6 * distance + self.match(features).squeeze(-1)) scores = scores.masked_fill(~valid, -1e4) weights = scores.softmax(1) next_pos = (weights[..., None] * xy).sum(1) current = F.normalize((weights[..., None] * keys).sum(1), dim=-1) best = torch.stack((weights.max(1).values, (weights * anchored).sum(1), (weights * recent).sum(1), (weights * distance).sum(1)), dim=1) blend = self.update(best).sigmoid() * .45 next_recent = F.normalize(state.recent.lerp(current, blend), dim=-1) velocity = state.velocity.lerp((next_pos - state.position) / elapsed, .40).clamp(-.9, .9) updated = ObjectState(state.identity, next_recent, next_pos, velocity) return scores, state if static else updated @dataclass class Hypotheses: states: ObjectState log_weights: torch.Tensor class HypothesisMemory(ObjectMemory): """Preserve several plausible objects when sight and sound do not resolve a region.""" def __init__(self, grounder, count=4, input_width=64): super().__init__(grounder, input_width) self.count = count self.acquire = nn.Sequential(nn.Linear(6, 24), nn.SiLU(), nn.Linear(24, 1)) self.compare = nn.Sequential(nn.Linear(5, 24), nn.SiLU(), nn.Linear(24, 1)) for layer in (self.visual[-1], self.acquire[-1], self.compare[-1]): nn.init.zeros_(layer.weight) nn.init.zeros_(layer.bias) def anchor_scores(self, keys, direct, prior_keys, prior_direct, xy, valid, prior_valid): affinity = keys @ prior_keys.transpose(-1, -2) distance = (xy[:, :, None] - xy[:, None]).square().sum(-1) matched = (affinity - 4 * distance).masked_fill(~prior_valid[:, None], -1e4).max(-1).values departure = 1 - (keys * prior_keys).sum(-1) evidence = torch.stack((direct / 16, prior_direct / 16, matched, departure, xy[..., 0], xy[..., 1]), dim=-1) scores = direct + self.acquire(evidence).squeeze(-1) return scores.masked_fill(~valid, -1e4) def initialize(self, keys, scores, xy, valid): b, n, d = keys.shape choices = scores selected = [] batch = torch.arange(b, device=keys.device) for _ in range(self.count): peak = choices.argmax(-1) selected.append(peak) center = xy[batch, peak] distance = (xy - center[:, None]).square().sum(-1) choices = choices - 8 * torch.exp(-distance / .025) indices = torch.stack(selected, dim=1) centers = xy.gather(1, indices[..., None].expand(-1, -1, 2)) distance = (xy[:, None] - centers[:, :, None]).square().sum(-1) local = scores[:, None] - 110 * distance weights = local.masked_fill(~valid[:, None], -1e4).softmax(-1) state = self.seed(keys[:, None].expand(-1, self.count, -1, -1).reshape(-1, n, d), xy[:, None].expand(-1, self.count, -1, -1).reshape(-1, n, 2), weights.reshape(-1, n)) log_weights = (scores.gather(1, indices) * .35).log_softmax(-1) return Hypotheses(state, log_weights) def reconsider(self, hypothesis, keys, direct, xy, valid, elapsed=.5): b, n, d = keys.shape k = self.count def expand(value): return value[:, None].expand(-1, k, *value.shape[1:]).reshape(b*k, *value.shape[1:]) scores, states = self.step(hypothesis.states, expand(keys), expand(direct), expand(xy), expand(valid), elapsed) values = scores.reshape(b, k, n).log_softmax(-1) attended = values.exp() audio = direct.masked_fill(~valid, -1e4).log_softmax(-1) overlap = (attended * audio.exp()[:, None]).sum(-1) displacement = (states.position - hypothesis.states.position).norm(dim=-1).reshape(b, k) identity = (states.identity * states.recent).sum(-1).reshape(b, k) confidence = attended.max(-1).values inputs = torch.stack((overlap, displacement, identity, confidence, hypothesis.log_weights), dim=-1) adjustment = self.compare(inputs).squeeze(-1).clamp(-1.5, 1.5) log_weights = (hypothesis.log_weights + adjustment).log_softmax(-1) combined = (values + log_weights[..., None]).logsumexp(1) return combined, Hypotheses(states, log_weights) class EvidenceChoice(nn.Module): """Choose whether past or current evidence better locates the current object.""" def __init__(self): super().__init__() self.gain = nn.Sequential(nn.Linear(12, 32), nn.LayerNorm(32), nn.SiLU(), nn.Linear(32, 1)) @staticmethod def features(remembered, arrived, xy, valid, audio, previous_audio): current = arrived.masked_fill(~valid, -1e4).softmax(-1) memory = remembered.exp() diff = ((memory - current)[..., None] * xy).sum(1) agreement = (memory * current).sum(-1) current_peak, memory_peak = current.max(-1).values, memory.max(-1).values audio_match = F.cosine_similarity(audio, previous_audio, dim=-1) magnitude = audio.norm(dim=-1) / audio.shape[-1] ** .5 old_magnitude = previous_audio.norm(dim=-1) / previous_audio.shape[-1] ** .5 entropy = lambda p: -(p * p.clamp_min(1e-8).log()).sum(-1) return torch.cat((diff.abs(), torch.stack((agreement, current_peak, memory_peak, audio_match, magnitude, old_magnitude, entropy(current), entropy(memory), arrived.max(-1).values, diff.norm(dim=-1)), dim=-1)), dim=-1) def forward(self, features): return self.gain(features).squeeze(-1) class EventAcquisition(nn.Module): """Recognize arrived evidence that can start or revise an object hypothesis.""" def __init__(self): super().__init__() self.sound = nn.Sequential(nn.LayerNorm(256), nn.Linear(256, 32), nn.SiLU()) self.policy = nn.Sequential(nn.Linear(72, 32), nn.SiLU(), nn.Linear(32, 1)) def forward(self, audio, old_audio, region, old_region, valid): present, prior = self.sound(audio), self.sound(old_audio) current = region.masked_fill(~valid, -1e4).softmax(-1) last = old_region.masked_fill(~valid, -1e4).softmax(-1) summaries = torch.stack((current.max(1).values, last.max(1).values, (current * last).sum(1), (current - last).abs().sum(1), F.cosine_similarity(audio, old_audio), region.max(1).values, old_region.max(1).values, (audio - old_audio).square().mean(1)), dim=1) return self.policy(torch.cat((present, (present-prior).abs(), summaries), dim=1)).squeeze(-1) class LiveObjectMemory: def __init__(self, model, choice, cutoff, acquisition=None): self.model, self.choice, self.cutoff = model, choice, cutoff self.acquisition = acquisition self.previous = self.state = self.anchor_time = self.focus = None self.evaluated = self.carrying = self.reanchored = self.used = 0 self.acquired = 0 self.sound_active = self.anchored_by_sound = False self.events = deque(maxlen=360) @torch.inference_mode() def observe(self, time, vision, xy, audio): audio = audio.float() keys, direct = self.model.encode(vision[None], audio[None]) valid = torch.ones_like(direct, dtype=torch.bool) xy = xy[None] previous = self.previous self.previous = (keys, direct, xy, audio[None], time) if previous is None or not torch.equal(previous[2], xy) or time - previous[4] > 1.25: self.state = self.model.initialize(keys, direct, xy, valid) self.anchor_time = time self.focus = (time, self.anchor_time, direct[0], False) self.sound_active = self.anchored_by_sound = False return old_keys, old_direct, _, old_audio, old_time = previous elapsed = time - old_time if elapsed <= 0: self.previous = previous return local = self.model.anchor_scores(keys, direct, old_keys, old_direct, xy, valid, valid) memory, proposed = self.model.reconsider(self.state, keys, direct, xy, valid, elapsed) features = self.choice.features(memory, local, xy, valid, audio[None], old_audio) gain = self.choice(features)[0] carry = bool(gain >= self.cutoff) new_event = False if self.acquisition is not None: scout, threshold, retention_floor = self.acquisition active = bool(scout(audio[None], old_audio, local, old_direct, valid)[0] >= threshold) new_event = active and not self.sound_active self.sound_active = active carry = (carry or (self.anchored_by_sound and bool(gain >= retention_floor))) and not new_event if carry: self.state = proposed self.carrying += 1 else: self.state = self.model.initialize(keys, local, xy, valid) self.anchor_time = time self.reanchored += 1 self.anchored_by_sound = new_event self.acquired += int(new_event) self.evaluated += 1 self.focus = (time, self.anchor_time, memory[0] if carry else local[0], carry) self.events.append({'time': time, 'selected': self.anchor_time, 'retained': carry, 'acquired_from_sound': new_event, 'current_xy': xy[0, int(self.focus[2].argmax())].tolist(), 'profile': 'object'}) observe_sensors = observe def for_speech(self, time): if self.focus is None: return None observed, anchor, _, carry = self.focus if carry and .9 <= observed - anchor and 0 <= time - observed <= .9: return {'time': observed, 'selected': anchor, 'profile': 'object'} return None def summary(self): return {'profile': 'object', 'evaluated': self.evaluated, 'retained': self.carrying, 'reanchored': self.reanchored, 'used_in_speech': self.used, 'acquired_from_sound': self.acquired, 'events': list(self.events)} def attention_bias(self, times, kinds): if self.focus is None: return None time, _, logits, carry = self.focus if not carry: return None mask = (kinds == 0) & torch.isclose(times, times.new_tensor(time), atol=1e-4, rtol=0) if int(mask.sum()) != len(logits): return None bias = torch.zeros_like(times) bias[mask] = (logits.log_softmax(0) + math.log(len(logits))).clamp(0, 2) return bias