Download lovemachine/object_memory.py from Koppen9029/LoveMachine: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/Koppen9029/LoveMachine/resolve/main/lovemachine/object_memory.py
- Command line
-
hf download hf://Koppen9029/LoveMachine/lovemachine/object_memory.py
-
curl -L -o object_memory.py https://huggingface.co/Koppen9029/LoveMachine/resolve/main/lovemachine/object_memory.py
13.2 kB
| """Track the appearance and position of one observed object in numerical memory.""" | |
| from collections import deque | |
| from dataclasses import dataclass | |
| import math | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class ObjectState: | |
| identity: torch.Tensor | |
| recent: torch.Tensor | |
| position: torch.Tensor | |
| velocity: torch.Tensor | |
| class ObjectMemory(nn.Module): | |
| def __init__(self, grounder, input_width=64): | |
| super().__init__() | |
| self.grounder = grounder.eval().requires_grad_(False) | |
| self.input_width = input_width | |
| self.visual = nn.Sequential(nn.LayerNorm(input_width), nn.Linear(input_width, 80), nn.SiLU(), nn.Linear(80, 64)) | |
| self.sound = nn.Sequential(nn.LayerNorm(256), nn.Linear(256, 80), nn.SiLU(), nn.Linear(80, 64)) | |
| self.match = nn.Sequential(nn.Linear(6, 28), nn.SiLU(), nn.Linear(28, 1)) | |
| self.update = nn.Sequential(nn.Linear(4, 16), nn.SiLU(), nn.Linear(16, 1)) | |
| nn.init.zeros_(self.match[-1].weight) | |
| nn.init.zeros_(self.match[-1].bias) | |
| def encode(self, vision, audio): | |
| original = vision[..., :64].float() | |
| keys = F.normalize(F.normalize(original, dim=-1) + self.visual(vision.float()), dim=-1) | |
| query = F.normalize(self.sound(audio.float()), dim=-1) | |
| prior = F.normalize(self.grounder.query(audio.float()), dim=-1) | |
| reference = F.normalize(self.grounder.key(original), dim=-1) | |
| audio_logits = 7 * (keys * query[..., None, :]).sum(-1) + 12 * (reference * prior[..., None, :]).sum(-1) | |
| return keys, audio_logits | |
| def seed(keys, xy, weights): | |
| identity = F.normalize((weights[..., None] * keys).sum(1), dim=-1) | |
| position = (weights[..., None] * xy).sum(1) | |
| return ObjectState(identity, identity, position, torch.zeros_like(position)) | |
| def step(self, state, keys, audio_logits, xy, valid, elapsed=.5, *, static=False, appearance=True): | |
| predicted = (state.position + state.velocity * elapsed).clamp(0, 1) | |
| offset = xy - predicted[:, None] | |
| distance = offset.square().sum(-1) | |
| anchored = (keys * state.identity[:, None]).sum(-1) | |
| recent = (keys * state.recent[:, None]).sum(-1) | |
| if not appearance: | |
| anchored = recent = torch.zeros_like(anchored) | |
| features = torch.cat((offset, torch.stack((anchored, recent, distance, audio_logits / 16), dim=-1)), dim=-1) | |
| scores = (7 * anchored + 4 * recent - 6 * distance + self.match(features).squeeze(-1)) | |
| scores = scores.masked_fill(~valid, -1e4) | |
| weights = scores.softmax(1) | |
| next_pos = (weights[..., None] * xy).sum(1) | |
| current = F.normalize((weights[..., None] * keys).sum(1), dim=-1) | |
| best = torch.stack((weights.max(1).values, (weights * anchored).sum(1), | |
| (weights * recent).sum(1), (weights * distance).sum(1)), dim=1) | |
| blend = self.update(best).sigmoid() * .45 | |
| next_recent = F.normalize(state.recent.lerp(current, blend), dim=-1) | |
| velocity = state.velocity.lerp((next_pos - state.position) / elapsed, .40).clamp(-.9, .9) | |
| updated = ObjectState(state.identity, next_recent, next_pos, velocity) | |
| return scores, state if static else updated | |
| class Hypotheses: | |
| states: ObjectState | |
| log_weights: torch.Tensor | |
| class HypothesisMemory(ObjectMemory): | |
| """Preserve several plausible objects when sight and sound do not resolve a region.""" | |
| def __init__(self, grounder, count=4, input_width=64): | |
| super().__init__(grounder, input_width) | |
| self.count = count | |
| self.acquire = nn.Sequential(nn.Linear(6, 24), nn.SiLU(), nn.Linear(24, 1)) | |
| self.compare = nn.Sequential(nn.Linear(5, 24), nn.SiLU(), nn.Linear(24, 1)) | |
| for layer in (self.visual[-1], self.acquire[-1], self.compare[-1]): | |
| nn.init.zeros_(layer.weight) | |
| nn.init.zeros_(layer.bias) | |
| def anchor_scores(self, keys, direct, prior_keys, prior_direct, xy, valid, prior_valid): | |
| affinity = keys @ prior_keys.transpose(-1, -2) | |
| distance = (xy[:, :, None] - xy[:, None]).square().sum(-1) | |
| matched = (affinity - 4 * distance).masked_fill(~prior_valid[:, None], -1e4).max(-1).values | |
| departure = 1 - (keys * prior_keys).sum(-1) | |
| evidence = torch.stack((direct / 16, prior_direct / 16, matched, departure, xy[..., 0], xy[..., 1]), dim=-1) | |
| scores = direct + self.acquire(evidence).squeeze(-1) | |
| return scores.masked_fill(~valid, -1e4) | |
| def initialize(self, keys, scores, xy, valid): | |
| b, n, d = keys.shape | |
| choices = scores | |
| selected = [] | |
| batch = torch.arange(b, device=keys.device) | |
| for _ in range(self.count): | |
| peak = choices.argmax(-1) | |
| selected.append(peak) | |
| center = xy[batch, peak] | |
| distance = (xy - center[:, None]).square().sum(-1) | |
| choices = choices - 8 * torch.exp(-distance / .025) | |
| indices = torch.stack(selected, dim=1) | |
| centers = xy.gather(1, indices[..., None].expand(-1, -1, 2)) | |
| distance = (xy[:, None] - centers[:, :, None]).square().sum(-1) | |
| local = scores[:, None] - 110 * distance | |
| weights = local.masked_fill(~valid[:, None], -1e4).softmax(-1) | |
| state = self.seed(keys[:, None].expand(-1, self.count, -1, -1).reshape(-1, n, d), | |
| xy[:, None].expand(-1, self.count, -1, -1).reshape(-1, n, 2), | |
| weights.reshape(-1, n)) | |
| log_weights = (scores.gather(1, indices) * .35).log_softmax(-1) | |
| return Hypotheses(state, log_weights) | |
| def reconsider(self, hypothesis, keys, direct, xy, valid, elapsed=.5): | |
| b, n, d = keys.shape | |
| k = self.count | |
| def expand(value): | |
| return value[:, None].expand(-1, k, *value.shape[1:]).reshape(b*k, *value.shape[1:]) | |
| scores, states = self.step(hypothesis.states, expand(keys), expand(direct), | |
| expand(xy), expand(valid), elapsed) | |
| values = scores.reshape(b, k, n).log_softmax(-1) | |
| attended = values.exp() | |
| audio = direct.masked_fill(~valid, -1e4).log_softmax(-1) | |
| overlap = (attended * audio.exp()[:, None]).sum(-1) | |
| displacement = (states.position - hypothesis.states.position).norm(dim=-1).reshape(b, k) | |
| identity = (states.identity * states.recent).sum(-1).reshape(b, k) | |
| confidence = attended.max(-1).values | |
| inputs = torch.stack((overlap, displacement, identity, confidence, hypothesis.log_weights), dim=-1) | |
| adjustment = self.compare(inputs).squeeze(-1).clamp(-1.5, 1.5) | |
| log_weights = (hypothesis.log_weights + adjustment).log_softmax(-1) | |
| combined = (values + log_weights[..., None]).logsumexp(1) | |
| return combined, Hypotheses(states, log_weights) | |
| class EvidenceChoice(nn.Module): | |
| """Choose whether past or current evidence better locates the current object.""" | |
| def __init__(self): | |
| super().__init__() | |
| self.gain = nn.Sequential(nn.Linear(12, 32), nn.LayerNorm(32), nn.SiLU(), nn.Linear(32, 1)) | |
| def features(remembered, arrived, xy, valid, audio, previous_audio): | |
| current = arrived.masked_fill(~valid, -1e4).softmax(-1) | |
| memory = remembered.exp() | |
| diff = ((memory - current)[..., None] * xy).sum(1) | |
| agreement = (memory * current).sum(-1) | |
| current_peak, memory_peak = current.max(-1).values, memory.max(-1).values | |
| audio_match = F.cosine_similarity(audio, previous_audio, dim=-1) | |
| magnitude = audio.norm(dim=-1) / audio.shape[-1] ** .5 | |
| old_magnitude = previous_audio.norm(dim=-1) / previous_audio.shape[-1] ** .5 | |
| entropy = lambda p: -(p * p.clamp_min(1e-8).log()).sum(-1) | |
| return torch.cat((diff.abs(), torch.stack((agreement, current_peak, memory_peak, audio_match, | |
| magnitude, old_magnitude, entropy(current), entropy(memory), arrived.max(-1).values, | |
| diff.norm(dim=-1)), dim=-1)), dim=-1) | |
| def forward(self, features): | |
| return self.gain(features).squeeze(-1) | |
| class EventAcquisition(nn.Module): | |
| """Recognize arrived evidence that can start or revise an object hypothesis.""" | |
| def __init__(self): | |
| super().__init__() | |
| self.sound = nn.Sequential(nn.LayerNorm(256), nn.Linear(256, 32), nn.SiLU()) | |
| self.policy = nn.Sequential(nn.Linear(72, 32), nn.SiLU(), nn.Linear(32, 1)) | |
| def forward(self, audio, old_audio, region, old_region, valid): | |
| present, prior = self.sound(audio), self.sound(old_audio) | |
| current = region.masked_fill(~valid, -1e4).softmax(-1) | |
| last = old_region.masked_fill(~valid, -1e4).softmax(-1) | |
| summaries = torch.stack((current.max(1).values, last.max(1).values, (current * last).sum(1), | |
| (current - last).abs().sum(1), F.cosine_similarity(audio, old_audio), | |
| region.max(1).values, old_region.max(1).values, | |
| (audio - old_audio).square().mean(1)), dim=1) | |
| return self.policy(torch.cat((present, (present-prior).abs(), summaries), dim=1)).squeeze(-1) | |
| class LiveObjectMemory: | |
| def __init__(self, model, choice, cutoff, acquisition=None): | |
| self.model, self.choice, self.cutoff = model, choice, cutoff | |
| self.acquisition = acquisition | |
| self.previous = self.state = self.anchor_time = self.focus = None | |
| self.evaluated = self.carrying = self.reanchored = self.used = 0 | |
| self.acquired = 0 | |
| self.sound_active = self.anchored_by_sound = False | |
| self.events = deque(maxlen=360) | |
| def observe(self, time, vision, xy, audio): | |
| audio = audio.float() | |
| keys, direct = self.model.encode(vision[None], audio[None]) | |
| valid = torch.ones_like(direct, dtype=torch.bool) | |
| xy = xy[None] | |
| previous = self.previous | |
| self.previous = (keys, direct, xy, audio[None], time) | |
| if previous is None or not torch.equal(previous[2], xy) or time - previous[4] > 1.25: | |
| self.state = self.model.initialize(keys, direct, xy, valid) | |
| self.anchor_time = time | |
| self.focus = (time, self.anchor_time, direct[0], False) | |
| self.sound_active = self.anchored_by_sound = False | |
| return | |
| old_keys, old_direct, _, old_audio, old_time = previous | |
| elapsed = time - old_time | |
| if elapsed <= 0: | |
| self.previous = previous | |
| return | |
| local = self.model.anchor_scores(keys, direct, old_keys, old_direct, xy, valid, valid) | |
| memory, proposed = self.model.reconsider(self.state, keys, direct, xy, valid, elapsed) | |
| features = self.choice.features(memory, local, xy, valid, audio[None], old_audio) | |
| gain = self.choice(features)[0] | |
| carry = bool(gain >= self.cutoff) | |
| new_event = False | |
| if self.acquisition is not None: | |
| scout, threshold, retention_floor = self.acquisition | |
| active = bool(scout(audio[None], old_audio, local, old_direct, valid)[0] >= threshold) | |
| new_event = active and not self.sound_active | |
| self.sound_active = active | |
| carry = (carry or (self.anchored_by_sound and bool(gain >= retention_floor))) and not new_event | |
| if carry: | |
| self.state = proposed | |
| self.carrying += 1 | |
| else: | |
| self.state = self.model.initialize(keys, local, xy, valid) | |
| self.anchor_time = time | |
| self.reanchored += 1 | |
| self.anchored_by_sound = new_event | |
| self.acquired += int(new_event) | |
| self.evaluated += 1 | |
| self.focus = (time, self.anchor_time, memory[0] if carry else local[0], carry) | |
| self.events.append({'time': time, 'selected': self.anchor_time, 'retained': carry, | |
| 'acquired_from_sound': new_event, | |
| 'current_xy': xy[0, int(self.focus[2].argmax())].tolist(), 'profile': 'object'}) | |
| observe_sensors = observe | |
| def for_speech(self, time): | |
| if self.focus is None: | |
| return None | |
| observed, anchor, _, carry = self.focus | |
| if carry and .9 <= observed - anchor and 0 <= time - observed <= .9: | |
| return {'time': observed, 'selected': anchor, 'profile': 'object'} | |
| return None | |
| def summary(self): | |
| return {'profile': 'object', 'evaluated': self.evaluated, 'retained': self.carrying, | |
| 'reanchored': self.reanchored, 'used_in_speech': self.used, | |
| 'acquired_from_sound': self.acquired, | |
| 'events': list(self.events)} | |
| def attention_bias(self, times, kinds): | |
| if self.focus is None: | |
| return None | |
| time, _, logits, carry = self.focus | |
| if not carry: | |
| return None | |
| mask = (kinds == 0) & torch.isclose(times, times.new_tensor(time), atol=1e-4, rtol=0) | |
| if int(mask.sum()) != len(logits): | |
| return None | |
| bias = torch.zeros_like(times) | |
| bias[mask] = (logits.log_softmax(0) + math.log(len(logits))).clamp(0, 2) | |
| return bias | |