"""Small independently parameterized agents communicating only numerically. Frozen raw-AV encoders are supplied by the experiment adapter. This module contains the learned perception bridges, recurrent cognition, memory, prediction, emission policy and optional character decoder. It has no task labels, novelty/curiosity score, speech transcription or thought-text loop. """ from dataclasses import dataclass import torch from torch import nn from torch.nn import functional as F @dataclass class BrainState: hidden: torch.Tensor memory: torch.Tensor own_action: torch.Tensor prediction_error: torch.Tensor class SensoryBridge(nn.Module): def __init__(self,features=64,width=96): super().__init__() self.visual=nn.Sequential(nn.LayerNorm(features),nn.Linear(features,width)) self.audio=nn.Sequential(nn.LayerNorm(features),nn.Linear(features,width)) self.geometry=nn.Sequential(nn.Linear(5,width),nn.Tanh()) self.kind=nn.Embedding(2,width) def forward(self,features,geometry,kinds): a=self.visual(features[...,:64]);b=self.audio(features[...,:64]) if hasattr(self,'raw_visual') and features.shape[-1]>64:a=a+self.raw_visual(features[...,64:]) return torch.where((kinds==0)[...,None],a,b)+self.geometry(geometry)+self.kind(kinds) class AttentionAgent(nn.Module): def __init__(self,width=96): super().__init__();self.width=width self.norm=nn.LayerNorm(width);self.query=nn.Linear(width,width,bias=False) self.key=nn.Linear(width,width,bias=False);self.value=nn.Linear(width,width,bias=False) def forward(self,hidden,tokens,valid,*,uniform=False,blocked=None,attention_bias=None): logits=self.query(self.norm(hidden))@self.key(tokens).transpose(-1,-2)/self.width**.5 if attention_bias is not None: logits=logits+attention_bias[:,None] permitted=valid if blocked is None else valid&~blocked if not bool(permitted.any(-1).all()):raise ValueError('Need an arrived sensory token') logits=logits.masked_fill(~permitted[:,None],-1e4) weights=(permitted[:,None].to(logits)/permitted.sum(-1)[:,None,None]) if uniform else logits.softmax(-1) if uniform:weights=weights.expand(-1,hidden.shape[1],-1) return weights@self.value(tokens),weights class MemoryAgent(nn.Module): def __init__(self,width=96): super().__init__();self.width=width self.read=nn.Linear(width,width,bias=False) self.write=nn.Sequential(nn.LayerNorm(width),nn.Linear(width,width),nn.Tanh()) self.gate=nn.Linear(width*2,width) nn.init.constant_(self.gate.bias,-2.) def update(self,memory,hidden,dt): proposed=self.write(hidden) rate=F.softplus(self.gate(torch.cat((memory,hidden),-1))) return memory+(1-torch.exp(-rate*dt))*(proposed-memory) class CognitionAgent(nn.Module): def __init__(self,width=96,slots=4): super().__init__();self.width=width;self.slots=slots self.initial_hidden=nn.Parameter(torch.randn(slots,width)*.03) self.cell=nn.GRUCell(width,width) self.action_input=nn.Linear(width,width,bias=False) self.error_input=nn.Linear(64,width,bias=False) self.rate=nn.Parameter(torch.full((width,),1.5)) def forward(self,hidden,context,memory_read,own_action,prediction_error,dt): b,s,w=hidden.shape signal=context+memory_read+self.action_input(own_action)[:,None]+self.error_input(prediction_error)[:,None] candidate=self.cell(signal.reshape(-1,w),hidden.reshape(-1,w)).reshape(b,s,w) return hidden+(1-torch.exp(-F.softplus(self.rate)*dt))*(candidate-hidden) class PredictionAgent(nn.Module): def __init__(self,width=96,slots=4): super().__init__() self.body=nn.Sequential(nn.LayerNorm(width*slots),nn.Linear(width*slots,128),nn.SiLU()) self.location=nn.Linear(128,2);self.color=nn.Linear(128,2) self.feature=nn.Linear(128,64) def forward(self,hidden): x=self.body(hidden.flatten(1)) return {'location':self.location(x).sigmoid(),'color_logits':self.color(x),'future_feature':self.feature(x)} class EmissionAgent(nn.Module): def __init__(self,width=96,slots=4): super().__init__() self.policy=nn.Sequential(nn.LayerNorm(width*(slots+1)),nn.Linear(width*(slots+1),64),nn.SiLU(),nn.Linear(64,2)) def forward(self,state):return self.policy(torch.cat((state.hidden.flatten(1),state.own_action),-1)) class SpatialPredictionAgent(nn.Module): """Learned numerical query reads arbitrary local-token coordinates. Neither object boxes/color names nor target positions enter its inputs. The position is a differentiable soft read over native patch coordinates. """ def __init__(self,width=96,slots=4): super().__init__();self.query=nn.Sequential(nn.LayerNorm(width*slots),nn.Linear(width*slots,64)) self.key=nn.Linear(width,64);self.scale=nn.Parameter(torch.tensor(1.)) def forward(self,hidden,tokens,valid,xy,*,uniform=False): logits=(self.query(hidden.flatten(1))[:,None]@self.key(tokens).transpose(-1,-2)).squeeze(1)/8 logits=logits*F.softplus(self.scale) logits=logits.masked_fill(~valid,-1e4) weights=valid.to(logits)/valid.sum(-1).clamp_min(1)[:,None] if uniform else logits.softmax(-1) return (weights[...,None]*xy).sum(1),weights class FoveatedFeedbackAgent(nn.Module): """Learned selected-content -> numerical workspace feedback. The vector gate is an internal write mechanism, not a curiosity scalar. A zero final projection preserves the previous core at installation. """ def __init__(self,width=96): super().__init__() self.value=nn.Sequential(nn.LayerNorm(width*2),nn.Linear(width*2,64),nn.SiLU(),nn.Linear(64,width)) self.gate=nn.Sequential(nn.LayerNorm(width*3),nn.Linear(width*3,width),nn.Sigmoid()) nn.init.zeros_(self.value[-1].weight);nn.init.zeros_(self.value[-1].bias) def forward(self,hidden,selected,arrived): return self.gate(torch.cat((hidden.mean(1),selected,arrived),-1))*self.value(torch.cat((selected,arrived),-1)) class CharacterAgent(nn.Module): def __init__(self,vocabulary,width=96,slots=4): super().__init__();self.vocabulary=vocabulary;self.hidden_width=width self.condition=nn.Linear(width*slots,width) self.embedding=nn.Embedding(len(vocabulary),48) self.cell=nn.GRUCell(48,width);self.readout=nn.Linear(width,len(vocabulary)) def teacher_loss(self,hidden,targets): h=torch.tanh(self.condition(hidden.flatten(1)));previous=torch.zeros(len(h),device=h.device,dtype=torch.long) losses=[] for target in targets.unbind(1): h=self.cell(self.embedding(previous),h);logits=self.readout(h) if bool((target!=-100).any()):losses.append(F.cross_entropy(logits,target,ignore_index=-100)) previous=target.clamp_min(0) return torch.stack(losses).mean(),h @torch.no_grad() def generate(self,hidden,limit=24): if len(hidden)!=1:raise ValueError('One live language stream') h=torch.tanh(self.condition(hidden.flatten(1)));previous=torch.zeros(1,device=h.device,dtype=torch.long);text=[] for _ in range(limit): h=self.cell(self.embedding(previous),h);previous=self.readout(h).argmax(-1) identity=int(previous) if identity==1:break if identity>1:text.append(self.vocabulary[identity]) return ''.join(text),h class ModularBrain(nn.Module): def __init__(self,features=64,width=96,slots=4,vocabulary=None): super().__init__();self.features=features;self.width=width;self.slots=slots self.sensory=SensoryBridge(features,width);self.attention=AttentionAgent(width) self.cognition=CognitionAgent(width,slots);self.memory=MemoryAgent(width) self.prediction=PredictionAgent(width,slots);self.emission=EmissionAgent(width,slots) self.vocabulary=vocabulary or ['','']+sorted(set('赤青は左右にあると思う。まだ分からない変わったね')) if len(self.vocabulary)!=len(set(self.vocabulary)):raise ValueError('Unique language vocabulary required') self.language=CharacterAgent(self.vocabulary,width,slots) def initial(self,batch,*,device=None): h=self.cognition.initial_hidden[None].expand(batch,-1,-1) if device is not None:h=h.to(device) return BrainState(h,h.new_zeros(h.shape),h.new_zeros(batch,self.width),h.new_zeros(batch,64)) def tick(self,state,tokens,valid,dt,*,uniform=False,blocked=None,visual_mask=None,coordinates=None, current_mask=None,foveated_feedback=True,attention_bias=None): context,weights=self.attention(state.hidden,tokens,valid,uniform=uniform,blocked=blocked, attention_bias=attention_bias) if hasattr(self,'foveated') and foveated_feedback and visual_mask is not None and bool(visual_mask.any(-1).all()): _,selection=self.spatial(state.hidden,tokens,visual_mask,coordinates,uniform=uniform) selected=(selection[...,None]*tokens).sum(1) arrived=valid if current_mask is None else current_mask fresh=(tokens*arrived[...,None]).sum(1)/arrived.sum(1).clamp_min(1)[:,None] context=context+self.foveated(state.hidden,selected,fresh)[:,None] hidden=self.cognition(state.hidden,context,self.memory.read(state.memory),state.own_action,state.prediction_error,dt) memory=self.memory.update(state.memory,hidden,dt) return BrainState(hidden,memory,state.own_action,state.prediction_error),weights def modules_for_optimizers(self): names=('sensory','attention','cognition','memory','prediction','emission','language') return {name:getattr(self,name) for name in names+tuple(n for n in ('foveated',) if hasattr(self,n))} def prediction_rollout(self,features,xy,kinds,times,valid,step_times,*,microticks=2,uniform=False, current_only=False,drop_audio=False,drop_visual=False,return_states=False, prediction_feedback=True,foveated_feedback=True,observed_errors=None): """Causal frozen-sensory boundaries; all new recurrent states have graph. Tokens from later steps stay masked even though a training batch holds the full captured sequence. Targets and task identities never enter. """ batch=features.shape[0];state=self.initial(batch);predictions=[];states=[];attentions=[];observations=[];spatial_attentions=[] for index,clock in enumerate(step_times): arrived=valid&(times<=clock+1e-5) if current_only: state=self.initial(batch);arrived=arrived&(times>=clock-1e-5) if drop_audio:arrived=arrived&(kinds==0) if drop_visual:arrived=arrived&(kinds==1) current_visual=valid&(kinds==0)&((times-clock).abs()<1e-5) observed=(features[...,:64]*current_visual[...,None]).sum(1)/current_visual.sum(1).clamp_min(1)[:,None] observations.append(observed) if index>=2 and not current_only and prediction_feedback: supplied=observed_errors[index] if observed_errors is not None else None if drop_visual: error=torch.zeros_like(observed) elif supplied is not None: error=supplied else: error=observed-predictions[index-2]['future_feature'] state=BrainState(state.hidden,state.memory,state.own_action,error) age=((clock-times).clamp_min(0)/10).unsqueeze(-1) geometry=torch.cat((xy,age,age.sin(),age.cos()),-1) tokens=self.sensory(features,geometry,kinds) dt=(float(clock-step_times[index-1]) if index else .5)/microticks fresh=arrived&((times-clock).abs()<1e-5) for _ in range(microticks): state,attention=self.tick(state,tokens,arrived,dt,uniform=uniform, visual_mask=arrived&(kinds==0),coordinates=xy,current_mask=fresh,foveated_feedback=foveated_feedback) prediction=self.prediction(state.hidden) if hasattr(self,'spatial') and not drop_visual: prediction['location'],spatial_attention=self.spatial(state.hidden,tokens,arrived&(kinds==0),xy,uniform=uniform) spatial_attentions.append(spatial_attention) predictions.append(prediction);states.append(state);attentions.append(attention) result={'location':torch.stack([p['location'] for p in predictions],1), 'color_logits':torch.stack([p['color_logits'] for p in predictions],1), 'future_feature':torch.stack([p['future_feature'] for p in predictions],1), 'observed_feature':torch.stack(observations,1),'last_state':state} if return_states:result.update(states=states,attention=attentions,spatial_attention=spatial_attentions) return result