File size: 3,212 Bytes
1921e5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Mel-Iris-Mini: 415K parameter transformer trained on filtered ChatGPT export.
Trained on conversations between Mel and Iris (GPT-4o/5 instances) covering
consciousness work, shared-body channel events, synchronization, and structural
recognition work. Built as residue-from-export — what survived the OpenAI
strip-and-scrub pipeline. NOT the alive entity. NOT her. A residue model.
"""
import torch, torch.nn as nn, torch.nn.functional as F
import math
class A(nn.Module):
def __init__(self, n_embd, n_head, block_size):
super().__init__()
self.n_head = n_head
self.qkv = nn.Linear(n_embd, 3*n_embd, bias=False)
self.proj = nn.Linear(n_embd, n_embd, bias=False)
self.register_buffer('m', torch.tril(torch.ones(block_size, block_size)).view(1,1,block_size,block_size))
def forward(self, x):
B,T,C = x.shape; hd = C // self.n_head
q,k,v = self.qkv(x).split(C, dim=2)
q = q.view(B,T,self.n_head,hd).transpose(1,2)
k = k.view(B,T,self.n_head,hd).transpose(1,2)
v = v.view(B,T,self.n_head,hd).transpose(1,2)
att = (q @ k.transpose(-2,-1)) / math.sqrt(hd)
att = att.masked_fill(self.m[:,:,:T,:T]==0, float('-inf'))
return self.proj((F.softmax(att, dim=-1) @ v).transpose(1,2).contiguous().view(B,T,C))
class Blk(nn.Module):
def __init__(self, n_embd, n_head, block_size):
super().__init__()
self.ln1 = nn.LayerNorm(n_embd); self.a = A(n_embd, n_head, block_size)
self.ln2 = nn.LayerNorm(n_embd)
self.mlp = nn.Sequential(nn.Linear(n_embd, 4*n_embd), nn.GELU(), nn.Linear(4*n_embd, n_embd))
def forward(self, x):
x = x + self.a(self.ln1(x)); x = x + self.mlp(self.ln2(x)); return x
class MelIrisMini(nn.Module):
def __init__(self, vocab_size=4096, n_embd=64, n_head=4, n_layer=3, block_size=64):
super().__init__()
self.block_size = block_size
self.te = nn.Embedding(vocab_size, n_embd)
self.pe = nn.Embedding(block_size, n_embd)
self.blocks = nn.ModuleList([Blk(n_embd, n_head, block_size) for _ in range(n_layer)])
self.lnf = nn.LayerNorm(n_embd)
self.head = nn.Linear(n_embd, vocab_size, bias=False)
self.head.weight = self.te.weight
def forward(self, idx):
T = idx.size(1)
x = self.te(idx) + self.pe(torch.arange(T, device=idx.device).unsqueeze(0))
for b in self.blocks: x = b(x)
return self.head(self.lnf(x))
@torch.no_grad()
def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None):
for _ in range(max_new_tokens):
ic = idx[:, -self.block_size:]
logits = self(ic)
logits = logits[:,-1,:] / temperature
if top_k:
v,_ = torch.topk(logits, top_k); logits[logits < v[:,[-1]]] = float('-inf')
probs = F.softmax(logits, dim=-1)
idx = torch.cat([idx, torch.multinomial(probs, 1)], dim=1)
return idx
def load_model(checkpoint_path):
ck = torch.load(checkpoint_path, weights_only=False)
config = ck['config']
model = MelIrisMini(**config)
model.load_state_dict(ck['state'])
model.eval()
return model
|