Download lovemachine/modular_brain.py from Koppen9029/LoveMachine: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/Koppen9029/LoveMachine/resolve/main/lovemachine/modular_brain.py
- Command line
-
hf download hf://Koppen9029/LoveMachine/lovemachine/modular_brain.py
-
curl -L -o modular_brain.py https://huggingface.co/Koppen9029/LoveMachine/resolve/main/lovemachine/modular_brain.py
13.2 kB
| """Small independently parameterized agents communicating only numerically. | |
| Frozen raw-AV encoders are supplied by the experiment adapter. This module | |
| contains the learned perception bridges, recurrent cognition, memory, | |
| prediction, emission policy and optional character decoder. It has no task | |
| labels, novelty/curiosity score, speech transcription or thought-text loop. | |
| """ | |
| from dataclasses import dataclass | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class BrainState: | |
| hidden: torch.Tensor | |
| memory: torch.Tensor | |
| own_action: torch.Tensor | |
| prediction_error: torch.Tensor | |
| class SensoryBridge(nn.Module): | |
| def __init__(self,features=64,width=96): | |
| super().__init__() | |
| self.visual=nn.Sequential(nn.LayerNorm(features),nn.Linear(features,width)) | |
| self.audio=nn.Sequential(nn.LayerNorm(features),nn.Linear(features,width)) | |
| self.geometry=nn.Sequential(nn.Linear(5,width),nn.Tanh()) | |
| self.kind=nn.Embedding(2,width) | |
| def forward(self,features,geometry,kinds): | |
| a=self.visual(features[...,:64]);b=self.audio(features[...,:64]) | |
| if hasattr(self,'raw_visual') and features.shape[-1]>64:a=a+self.raw_visual(features[...,64:]) | |
| return torch.where((kinds==0)[...,None],a,b)+self.geometry(geometry)+self.kind(kinds) | |
| class AttentionAgent(nn.Module): | |
| def __init__(self,width=96): | |
| super().__init__();self.width=width | |
| self.norm=nn.LayerNorm(width);self.query=nn.Linear(width,width,bias=False) | |
| self.key=nn.Linear(width,width,bias=False);self.value=nn.Linear(width,width,bias=False) | |
| def forward(self,hidden,tokens,valid,*,uniform=False,blocked=None,attention_bias=None): | |
| logits=self.query(self.norm(hidden))@self.key(tokens).transpose(-1,-2)/self.width**.5 | |
| if attention_bias is not None: | |
| logits=logits+attention_bias[:,None] | |
| permitted=valid if blocked is None else valid&~blocked | |
| if not bool(permitted.any(-1).all()):raise ValueError('Need an arrived sensory token') | |
| logits=logits.masked_fill(~permitted[:,None],-1e4) | |
| weights=(permitted[:,None].to(logits)/permitted.sum(-1)[:,None,None]) if uniform else logits.softmax(-1) | |
| if uniform:weights=weights.expand(-1,hidden.shape[1],-1) | |
| return weights@self.value(tokens),weights | |
| class MemoryAgent(nn.Module): | |
| def __init__(self,width=96): | |
| super().__init__();self.width=width | |
| self.read=nn.Linear(width,width,bias=False) | |
| self.write=nn.Sequential(nn.LayerNorm(width),nn.Linear(width,width),nn.Tanh()) | |
| self.gate=nn.Linear(width*2,width) | |
| nn.init.constant_(self.gate.bias,-2.) | |
| def update(self,memory,hidden,dt): | |
| proposed=self.write(hidden) | |
| rate=F.softplus(self.gate(torch.cat((memory,hidden),-1))) | |
| return memory+(1-torch.exp(-rate*dt))*(proposed-memory) | |
| class CognitionAgent(nn.Module): | |
| def __init__(self,width=96,slots=4): | |
| super().__init__();self.width=width;self.slots=slots | |
| self.initial_hidden=nn.Parameter(torch.randn(slots,width)*.03) | |
| self.cell=nn.GRUCell(width,width) | |
| self.action_input=nn.Linear(width,width,bias=False) | |
| self.error_input=nn.Linear(64,width,bias=False) | |
| self.rate=nn.Parameter(torch.full((width,),1.5)) | |
| def forward(self,hidden,context,memory_read,own_action,prediction_error,dt): | |
| b,s,w=hidden.shape | |
| signal=context+memory_read+self.action_input(own_action)[:,None]+self.error_input(prediction_error)[:,None] | |
| candidate=self.cell(signal.reshape(-1,w),hidden.reshape(-1,w)).reshape(b,s,w) | |
| return hidden+(1-torch.exp(-F.softplus(self.rate)*dt))*(candidate-hidden) | |
| class PredictionAgent(nn.Module): | |
| def __init__(self,width=96,slots=4): | |
| super().__init__() | |
| self.body=nn.Sequential(nn.LayerNorm(width*slots),nn.Linear(width*slots,128),nn.SiLU()) | |
| self.location=nn.Linear(128,2);self.color=nn.Linear(128,2) | |
| self.feature=nn.Linear(128,64) | |
| def forward(self,hidden): | |
| x=self.body(hidden.flatten(1)) | |
| return {'location':self.location(x).sigmoid(),'color_logits':self.color(x),'future_feature':self.feature(x)} | |
| class EmissionAgent(nn.Module): | |
| def __init__(self,width=96,slots=4): | |
| super().__init__() | |
| self.policy=nn.Sequential(nn.LayerNorm(width*(slots+1)),nn.Linear(width*(slots+1),64),nn.SiLU(),nn.Linear(64,2)) | |
| def forward(self,state):return self.policy(torch.cat((state.hidden.flatten(1),state.own_action),-1)) | |
| class SpatialPredictionAgent(nn.Module): | |
| """Learned numerical query reads arbitrary local-token coordinates. | |
| Neither object boxes/color names nor target positions enter its inputs. | |
| The position is a differentiable soft read over native patch coordinates. | |
| """ | |
| def __init__(self,width=96,slots=4): | |
| super().__init__();self.query=nn.Sequential(nn.LayerNorm(width*slots),nn.Linear(width*slots,64)) | |
| self.key=nn.Linear(width,64);self.scale=nn.Parameter(torch.tensor(1.)) | |
| def forward(self,hidden,tokens,valid,xy,*,uniform=False): | |
| logits=(self.query(hidden.flatten(1))[:,None]@self.key(tokens).transpose(-1,-2)).squeeze(1)/8 | |
| logits=logits*F.softplus(self.scale) | |
| logits=logits.masked_fill(~valid,-1e4) | |
| weights=valid.to(logits)/valid.sum(-1).clamp_min(1)[:,None] if uniform else logits.softmax(-1) | |
| return (weights[...,None]*xy).sum(1),weights | |
| class FoveatedFeedbackAgent(nn.Module): | |
| """Learned selected-content -> numerical workspace feedback. | |
| The vector gate is an internal write mechanism, not a curiosity scalar. | |
| A zero final projection preserves the previous core at installation. | |
| """ | |
| def __init__(self,width=96): | |
| super().__init__() | |
| self.value=nn.Sequential(nn.LayerNorm(width*2),nn.Linear(width*2,64),nn.SiLU(),nn.Linear(64,width)) | |
| self.gate=nn.Sequential(nn.LayerNorm(width*3),nn.Linear(width*3,width),nn.Sigmoid()) | |
| nn.init.zeros_(self.value[-1].weight);nn.init.zeros_(self.value[-1].bias) | |
| def forward(self,hidden,selected,arrived): | |
| return self.gate(torch.cat((hidden.mean(1),selected,arrived),-1))*self.value(torch.cat((selected,arrived),-1)) | |
| class CharacterAgent(nn.Module): | |
| def __init__(self,vocabulary,width=96,slots=4): | |
| super().__init__();self.vocabulary=vocabulary;self.hidden_width=width | |
| self.condition=nn.Linear(width*slots,width) | |
| self.embedding=nn.Embedding(len(vocabulary),48) | |
| self.cell=nn.GRUCell(48,width);self.readout=nn.Linear(width,len(vocabulary)) | |
| def teacher_loss(self,hidden,targets): | |
| h=torch.tanh(self.condition(hidden.flatten(1)));previous=torch.zeros(len(h),device=h.device,dtype=torch.long) | |
| losses=[] | |
| for target in targets.unbind(1): | |
| h=self.cell(self.embedding(previous),h);logits=self.readout(h) | |
| if bool((target!=-100).any()):losses.append(F.cross_entropy(logits,target,ignore_index=-100)) | |
| previous=target.clamp_min(0) | |
| return torch.stack(losses).mean(),h | |
| def generate(self,hidden,limit=24): | |
| if len(hidden)!=1:raise ValueError('One live language stream') | |
| h=torch.tanh(self.condition(hidden.flatten(1)));previous=torch.zeros(1,device=h.device,dtype=torch.long);text=[] | |
| for _ in range(limit): | |
| h=self.cell(self.embedding(previous),h);previous=self.readout(h).argmax(-1) | |
| identity=int(previous) | |
| if identity==1:break | |
| if identity>1:text.append(self.vocabulary[identity]) | |
| return ''.join(text),h | |
| class ModularBrain(nn.Module): | |
| def __init__(self,features=64,width=96,slots=4,vocabulary=None): | |
| super().__init__();self.features=features;self.width=width;self.slots=slots | |
| self.sensory=SensoryBridge(features,width);self.attention=AttentionAgent(width) | |
| self.cognition=CognitionAgent(width,slots);self.memory=MemoryAgent(width) | |
| self.prediction=PredictionAgent(width,slots);self.emission=EmissionAgent(width,slots) | |
| self.vocabulary=vocabulary or ['<bos>','<eos>']+sorted(set('θ΅€ιγ―ε·¦ε³γ«γγγ¨ζγγγΎγ εγγγͺγε€γγ£γγ')) | |
| if len(self.vocabulary)!=len(set(self.vocabulary)):raise ValueError('Unique language vocabulary required') | |
| self.language=CharacterAgent(self.vocabulary,width,slots) | |
| def initial(self,batch,*,device=None): | |
| h=self.cognition.initial_hidden[None].expand(batch,-1,-1) | |
| if device is not None:h=h.to(device) | |
| return BrainState(h,h.new_zeros(h.shape),h.new_zeros(batch,self.width),h.new_zeros(batch,64)) | |
| def tick(self,state,tokens,valid,dt,*,uniform=False,blocked=None,visual_mask=None,coordinates=None, | |
| current_mask=None,foveated_feedback=True,attention_bias=None): | |
| context,weights=self.attention(state.hidden,tokens,valid,uniform=uniform,blocked=blocked, | |
| attention_bias=attention_bias) | |
| if hasattr(self,'foveated') and foveated_feedback and visual_mask is not None and bool(visual_mask.any(-1).all()): | |
| _,selection=self.spatial(state.hidden,tokens,visual_mask,coordinates,uniform=uniform) | |
| selected=(selection[...,None]*tokens).sum(1) | |
| arrived=valid if current_mask is None else current_mask | |
| fresh=(tokens*arrived[...,None]).sum(1)/arrived.sum(1).clamp_min(1)[:,None] | |
| context=context+self.foveated(state.hidden,selected,fresh)[:,None] | |
| hidden=self.cognition(state.hidden,context,self.memory.read(state.memory),state.own_action,state.prediction_error,dt) | |
| memory=self.memory.update(state.memory,hidden,dt) | |
| return BrainState(hidden,memory,state.own_action,state.prediction_error),weights | |
| def modules_for_optimizers(self): | |
| names=('sensory','attention','cognition','memory','prediction','emission','language') | |
| return {name:getattr(self,name) for name in names+tuple(n for n in ('foveated',) if hasattr(self,n))} | |
| def prediction_rollout(self,features,xy,kinds,times,valid,step_times,*,microticks=2,uniform=False, | |
| current_only=False,drop_audio=False,drop_visual=False,return_states=False, | |
| prediction_feedback=True,foveated_feedback=True,observed_errors=None): | |
| """Causal frozen-sensory boundaries; all new recurrent states have graph. | |
| Tokens from later steps stay masked even though a training batch holds | |
| the full captured sequence. Targets and task identities never enter. | |
| """ | |
| batch=features.shape[0];state=self.initial(batch);predictions=[];states=[];attentions=[];observations=[];spatial_attentions=[] | |
| for index,clock in enumerate(step_times): | |
| arrived=valid&(times<=clock+1e-5) | |
| if current_only: | |
| state=self.initial(batch);arrived=arrived&(times>=clock-1e-5) | |
| if drop_audio:arrived=arrived&(kinds==0) | |
| if drop_visual:arrived=arrived&(kinds==1) | |
| current_visual=valid&(kinds==0)&((times-clock).abs()<1e-5) | |
| observed=(features[...,:64]*current_visual[...,None]).sum(1)/current_visual.sum(1).clamp_min(1)[:,None] | |
| observations.append(observed) | |
| if index>=2 and not current_only and prediction_feedback: | |
| supplied=observed_errors[index] if observed_errors is not None else None | |
| if drop_visual: | |
| error=torch.zeros_like(observed) | |
| elif supplied is not None: | |
| error=supplied | |
| else: | |
| error=observed-predictions[index-2]['future_feature'] | |
| state=BrainState(state.hidden,state.memory,state.own_action,error) | |
| age=((clock-times).clamp_min(0)/10).unsqueeze(-1) | |
| geometry=torch.cat((xy,age,age.sin(),age.cos()),-1) | |
| tokens=self.sensory(features,geometry,kinds) | |
| dt=(float(clock-step_times[index-1]) if index else .5)/microticks | |
| fresh=arrived&((times-clock).abs()<1e-5) | |
| for _ in range(microticks): | |
| state,attention=self.tick(state,tokens,arrived,dt,uniform=uniform, | |
| visual_mask=arrived&(kinds==0),coordinates=xy,current_mask=fresh,foveated_feedback=foveated_feedback) | |
| prediction=self.prediction(state.hidden) | |
| if hasattr(self,'spatial') and not drop_visual: | |
| prediction['location'],spatial_attention=self.spatial(state.hidden,tokens,arrived&(kinds==0),xy,uniform=uniform) | |
| spatial_attentions.append(spatial_attention) | |
| predictions.append(prediction);states.append(state);attentions.append(attention) | |
| result={'location':torch.stack([p['location'] for p in predictions],1), | |
| 'color_logits':torch.stack([p['color_logits'] for p in predictions],1), | |
| 'future_feature':torch.stack([p['future_feature'] for p in predictions],1), | |
| 'observed_feature':torch.stack(observations,1),'last_state':state} | |
| if return_states:result.update(states=states,attention=attentions,spatial_attention=spatial_attentions) | |
| return result | |