SLLM-0.6 / generate.py
Veirded's picture
Upload 4 files
dac9a15 verified
Raw History Blame Contribute Delete
9.22 kB
import re
import json
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_model
if torch.backends.mps.is_available():
device = torch.device("mps")
print("Using Apple Silicon Metal (MPS) acceleration.")
elif torch.cuda.is_available():
device = torch.device("cuda")
print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}")
else:
device = torch.device("cpu")
print("Using CPU inference.")
SPECIAL_TOKENS = ["<pad>", "<end_of_text>", "<|user|>", "<|ai|>"]
PRETOK_RE = re.compile(
r"(?i:'(?:s|t|re|ve|m|ll|d))| ?[A-Za-z]+| ?[0-9]+| ?[^\sA-Za-z0-9]+|\s+(?!\S)|\s+"
)
SPECIAL_RE = re.compile("(" + "|".join(re.escape(tok) for tok in SPECIAL_TOKENS) + ")")
class ByteBPETokenizer:
def __init__(self, special_tokens, merges):
self.special_tokens = list(special_tokens)
self.merges = [tuple(m) for m in merges]
self.tok2id = {tok: i for i, tok in enumerate(self.special_tokens)}
self.pad_id = self.tok2id["<pad>"]
self.end_id = self.tok2id["<end_of_text>"]
self.user_id = self.tok2id["<|user|>"]
self.ai_id = self.tok2id["<|ai|>"]
self.byte_offset = len(self.special_tokens)
self.id_to_bytes = {i: b"" for i in range(len(self.special_tokens))}
for b in range(256):
self.id_to_bytes[self.byte_offset + b] = bytes([b])
self.merge_ranks = {}
for rank, (a, b, new_id) in enumerate(self.merges):
self.id_to_bytes[new_id] = self.id_to_bytes[a] + self.id_to_bytes[b]
self.merge_ranks[(a, b)] = (rank, new_id)
@classmethod
def load(cls, path):
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
return cls(data["special_tokens"], data["merges"])
def _encode_word(self, word_bytes):
ids = [self.byte_offset + b for b in word_bytes]
while len(ids) >= 2:
best_rank, best_pair, best_new_id = None, None, None
for i in range(len(ids) - 1):
info = self.merge_ranks.get((ids[i], ids[i + 1]))
if info is not None and (best_rank is None or info[0] < best_rank):
best_rank, best_new_id = info
best_pair = (ids[i], ids[i + 1])
if best_pair is None:
break
a, b = best_pair
merged, i, n = [], 0, len(ids)
while i < n:
if i + 1 < n and ids[i] == a and ids[i + 1] == b:
merged.append(best_new_id)
i += 2
else:
merged.append(ids[i])
i += 1
ids = merged
return ids
def encode(self, text):
ids = []
for part in SPECIAL_RE.split(text):
if not part:
continue
sid = self.tok2id.get(part)
if sid is not None:
ids.append(sid)
else:
for chunk in PRETOK_RE.findall(part):
ids.extend(self._encode_word(chunk.encode("utf-8")))
return ids
def decode(self, ids):
special_set = set(range(len(self.special_tokens)))
chunks = [self.id_to_bytes[int(i)] for i in ids if int(i) not in special_set and int(i) in self.id_to_bytes]
return b"".join(chunks).decode("utf-8", errors="replace")
class RMSNorm(nn.Module):
def __init__(self, dim, eps=1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
class SwiGLU(nn.Module):
def __init__(self, dim, hidden_dim):
super().__init__()
self.w1 = nn.Linear(dim, hidden_dim, bias=False)
self.w2 = nn.Linear(hidden_dim, dim, bias=False)
self.w3 = nn.Linear(dim, hidden_dim, bias=False)
def forward(self, x):
return self.w2(F.silu(self.w1(x)) * self.w3(x))
def precompute_rope_tables(dim, max_seq_len, theta=10000.0):
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim))
freqs = torch.outer(torch.arange(max_seq_len, dtype=torch.float32), inv_freq)
emb = torch.cat((freqs, freqs), dim=-1)
return emb.cos(), emb.sin()
def rotate_half(x):
half = x.shape[-1] // 2
return torch.cat((-x[..., half:], x[..., :half]), dim=-1)
class RoPEAttention(nn.Module):
def __init__(self, d_model, n_head):
super().__init__()
self.n_head = n_head
self.head_dim = d_model // n_head
self.wq = nn.Linear(d_model, d_model, bias=False)
self.wk = nn.Linear(d_model, d_model, bias=False)
self.wv = nn.Linear(d_model, d_model, bias=False)
self.wo = nn.Linear(d_model, d_model, bias=False)
self.q_norm = RMSNorm(self.head_dim)
self.k_norm = RMSNorm(self.head_dim)
def forward(self, x, cos, sin, attn_mask):
B, T, C = x.shape
q = self.q_norm(self.wq(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2))
k = self.k_norm(self.wk(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2))
v = self.wv(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
q = (q * cos) + (rotate_half(q) * sin)
k = (k * cos) + (rotate_half(k) * sin)
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(self.head_dim))
att = F.softmax(att.masked_fill(~attn_mask, float("-inf")).float(), dim=-1).type_as(v)
return self.wo((att @ v).transpose(1, 2).contiguous().view(B, T, C))
class SLLMBlock(nn.Module):
def __init__(self, d_model, n_head, hidden_dim):
super().__init__()
self.attn_norm = RMSNorm(d_model)
self.attn = RoPEAttention(d_model, n_head)
self.ffn_norm = RMSNorm(d_model)
self.ffn = SwiGLU(d_model, hidden_dim)
def forward(self, x, cos, sin, attn_mask):
x = x + self.attn(self.attn_norm(x), cos, sin, attn_mask)
return x + self.ffn(self.ffn_norm(x))
class SLLM(nn.Module):
def __init__(self, vocab_size, d_model, n_head, n_layer, hidden_dim, block_size, pad_id=0):
super().__init__()
self.block_size = block_size
self.pad_id = pad_id
self.tok_emb = nn.Embedding(vocab_size, d_model)
self.layers = nn.ModuleList([SLLMBlock(d_model, n_head, hidden_dim) for _ in range(n_layer)])
self.norm = RMSNorm(d_model)
self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
self.lm_head.weight = self.tok_emb.weight
cos, sin = precompute_rope_tables(d_model // n_head, block_size)
self.register_buffer("cos", cos, persistent=False)
self.register_buffer("sin", sin, persistent=False)
self.register_buffer("causal_mask", torch.tril(torch.ones(block_size, block_size, dtype=torch.bool)).view(1, 1, block_size, block_size), persistent=False)
self.register_buffer("eye_mask", torch.eye(block_size, dtype=torch.bool).view(1, 1, block_size, block_size), persistent=False)
def forward(self, idx):
B, T = idx.shape
non_pad_k = (idx != self.pad_id).unsqueeze(1).unsqueeze(2)
pad_q_diag = (idx == self.pad_id).unsqueeze(1).unsqueeze(3) & self.eye_mask[:, :, :T, :T]
attn_mask = self.causal_mask[:, :, :T, :T] & (non_pad_k | pad_q_diag)
x = self.tok_emb(idx)
cos, sin = self.cos[:T].unsqueeze(0).unsqueeze(0), self.sin[:T].unsqueeze(0).unsqueeze(0)
for layer in self.layers:
x = layer(x, cos, sin, attn_mask)
return self.lm_head(self.norm(x))
with open("config.json", "r") as f:
cfg = json.load(f)
tokenizer = ByteBPETokenizer.load("tokenizer.json")
model = SLLM(
vocab_size=cfg["vocab_size"],
d_model=cfg["d_model"],
n_head=cfg["n_head"],
n_layer=cfg["n_layer"],
hidden_dim=cfg["hidden_dim"],
block_size=cfg["block_size"],
pad_id=tokenizer.pad_id,
).to(device)
load_model(model, "model.safetensors")
model.eval()
print("\n" + "=" * 50)
print("Connected to Veirded/SLLM-0.6 (1.0M Parameters)")
print("Single-turn mode. Type 'quit' to exit.")
print("=" * 50)
while True:
uin = input("\nYou: ").strip()
if not uin or uin.lower() in ("quit", "exit"):
break
ids = tokenizer.encode(f"<|user|> {uin}\n<|ai|>")[-cfg["block_size"]:]
idx = torch.tensor([ids], dtype=torch.long, device=device)
gen = []
with torch.no_grad():
for _ in range(64):
logits = model(idx[:, -cfg["block_size"]:])[0, -1].float()
logits[tokenizer.pad_id] = -float("inf")
for t in set(idx[0, -64:].tolist()):
logits[t] = logits[t] * 1.15 if logits[t] < 0 else logits[t] / 1.15
logits = logits / 0.5
v, _ = torch.topk(logits, 15)
logits[logits < v[-1]] = -float("inf")
nid = int(torch.multinomial(F.softmax(logits, dim=-1), 1).item())
if nid in (tokenizer.end_id, tokenizer.user_id):
break
gen.append(nid)
idx = torch.cat((idx, torch.tensor([[nid]], dtype=torch.long, device=device)), dim=1)
print(f"SLLM: {tokenizer.decode(gen).strip()}")