SLLM-0.5 / generate.py
Veirded's picture
Upload 4 files
931eee2 verified
Raw History Blame Contribute Delete
6.72 kB
import os
import re
import json
import torch
import torch.nn as nn
import torch.nn.functional as F
if torch.backends.mps.is_available():
device = torch.device("mps")
print("Using Apple Silicon Metal (MPS) acceleration.")
elif torch.cuda.is_available():
device = torch.device("cuda")
print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}")
else:
device = torch.device("cpu")
print("Using CPU inference.")
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
class SwiGLU(nn.Module):
def __init__(self, dim, hidden_dim):
super().__init__()
self.w1 = nn.Linear(dim, hidden_dim, bias=False)
self.w2 = nn.Linear(hidden_dim, dim, bias=False)
self.w3 = nn.Linear(dim, hidden_dim, bias=False)
def forward(self, x):
return self.w2(F.silu(self.w1(x)) * self.w3(x))
def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
t = torch.arange(end, device=freqs.device)
freqs = torch.outer(t, freqs).float()
return torch.polar(torch.ones_like(freqs), freqs)
def apply_rotary_emb(xq, xk, freqs_cis):
xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
freqs_cis = freqs_cis[:xq_.shape[1]].unsqueeze(0).unsqueeze(2)
xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
return xq_out.type_as(xq), xk_out.type_as(xk)
class RoPEAttention(nn.Module):
def __init__(self, d_model=112, n_head=4, block_size=256):
super().__init__()
self.n_head = n_head
self.head_dim = d_model // n_head
self.wq = nn.Linear(d_model, d_model, bias=False)
self.wk = nn.Linear(d_model, d_model, bias=False)
self.wv = nn.Linear(d_model, d_model, bias=False)
self.wo = nn.Linear(d_model, d_model, bias=False)
self.register_buffer('bias', torch.tril(torch.ones(block_size, block_size)).view(1, 1, block_size, block_size))
def forward(self, x, freqs_cis):
B, T, C = x.shape
q = self.wq(x).view(B, T, self.n_head, self.head_dim)
k = self.wk(x).view(B, T, self.n_head, self.head_dim)
v = self.wv(x).view(B, T, self.n_head, self.head_dim)
q, k = apply_rotary_emb(q, k, freqs_cis)
q = q.transpose(1, 2)
k = k.transpose(1, 2)
v = v.transpose(1, 2)
att = (q @ k.transpose(-2, -1)) * (1.0 / (self.head_dim ** 0.5))
att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf'))
att = F.softmax(att, dim=-1)
y = att @ v
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.wo(y)
class NovaBlock(nn.Module):
def __init__(self, d_model=112, n_head=4, hidden_dim=298, block_size=256):
super().__init__()
self.attn_norm = RMSNorm(d_model)
self.attn = RoPEAttention(d_model, n_head, block_size)
self.attn_gate = nn.Linear(d_model, d_model, bias=False)
self.ffn_norm = RMSNorm(d_model)
self.ffn = SwiGLU(d_model, hidden_dim)
self.ffn_gate = nn.Linear(d_model, d_model, bias=False)
def forward(self, x, freqs_cis):
x = x + torch.sigmoid(self.attn_gate(x)) * self.attn(self.attn_norm(x), freqs_cis)
x = x + torch.sigmoid(self.ffn_gate(x)) * self.ffn(self.ffn_norm(x))
return x
class SLLM(nn.Module):
def __init__(self, vocab_size=9385, d_model=112, n_head=4, n_layer=4, block_size=256):
super().__init__()
self.block_size = block_size
self.tok_emb = nn.Embedding(vocab_size, d_model)
self.layers = nn.ModuleList([NovaBlock(d_model, n_head, int(2 * (4 * d_model) / 3), block_size) for _ in range(n_layer)])
self.norm = RMSNorm(d_model)
self.highway_gain = nn.Parameter(torch.tensor(0.1))
self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
self.register_buffer('freqs_cis', precompute_freqs_cis(d_model // n_head, block_size))
def forward(self, idx):
B, T = idx.shape
x_emb = self.tok_emb(idx)
freqs_cis = self.freqs_cis[:T]
x = x_emb
for layer in self.layers:
x = layer(x, freqs_cis)
x_out = self.norm(x + self.highway_gain * x_emb)
return self.lm_head(x_out)
dir_path = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(dir_path, "tokenizer_sllm_t.json"), "r", encoding="utf-8") as f:
tok_data = json.load(f)
tok2id = tok_data["tok2id"]
id2tok = {int(k): v for k, v in tok_data["id2tok"].items()}
end_id = tok2id["<end_of_text>"]
pattern = r"(?:<end_of_text>|\w+|[^\w\s])"
model = SLLM(vocab_size=len(tok2id), block_size=256).to(device)
model.load_state_dict(torch.load(os.path.join(dir_path, "sllm_weights.pt"), map_location=device))
model.eval()
def detokenize(tokens):
out = ""
for t in tokens:
if t in {".", ",", "!", "?", "'", ";", ":", ")", "]"}: out = out.rstrip() + t + " "
elif t in {"(", "["}: out += t
else: out += t + " "
return out.strip()
print("\n" + "="*50)
print("🤖 Connected to Veirded/SLLM-0.5 (~2.8M Parameters)")
print("Type 'exit' to quit.")
print("="*50)
history = []
while True:
try:
user_msg = input("\nYou: ").strip()
except (KeyboardInterrupt, EOFError):
break
if not user_msg or user_msg.lower() in ("quit", "exit"):
break
user_tokens = re.findall(pattern, user_msg)
history.extend(user_tokens)
ctx = history[-256:]
idx = torch.tensor([[tok2id.get(t, 0) for t in ctx]], dtype=torch.long, device=device)
reply_tokens = []
with torch.no_grad():
for step in range(50):
logits = model(idx[:, -256:])[:, -1, :] / 0.75
if step < 8: logits[0, end_id] = -float('Inf')
for past_tok in set(reply_tokens[-6:]): logits[0, tok2id.get(past_tok, 0)] /= 1.4
v, _ = torch.topk(logits, min(30, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float('Inf')
probs = F.softmax(logits, dim=-1)
next_id = torch.multinomial(probs, num_samples=1)
token_str = id2tok[next_id.item()]
if token_str == "<end_of_text>": break
reply_tokens.append(token_str)
idx = torch.cat((idx, next_id), dim=1)
print(f"Bot: {detokenize(reply_tokens)}")
history.extend(reply_tokens)