import re import json import math import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_model if torch.backends.mps.is_available(): device = torch.device("mps") print("Using Apple Silicon Metal (MPS) acceleration.") elif torch.cuda.is_available(): device = torch.device("cuda") print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}") else: device = torch.device("cpu") print("Using CPU inference.") SPECIAL_TOKENS = ["", "", "<|user|>", "<|ai|>"] PRETOK_RE = re.compile( r"(?i:'(?:s|t|re|ve|m|ll|d))| ?[A-Za-z]+| ?[0-9]+| ?[^\sA-Za-z0-9]+|\s+(?!\S)|\s+" ) SPECIAL_RE = re.compile("(" + "|".join(re.escape(tok) for tok in SPECIAL_TOKENS) + ")") class ByteBPETokenizer: def __init__(self, special_tokens, merges): self.special_tokens = list(special_tokens) self.merges = [tuple(m) for m in merges] self.tok2id = {tok: i for i, tok in enumerate(self.special_tokens)} self.pad_id = self.tok2id[""] self.end_id = self.tok2id[""] self.user_id = self.tok2id["<|user|>"] self.ai_id = self.tok2id["<|ai|>"] self.byte_offset = len(self.special_tokens) self.id_to_bytes = {i: b"" for i in range(len(self.special_tokens))} for b in range(256): self.id_to_bytes[self.byte_offset + b] = bytes([b]) self.merge_ranks = {} for rank, (a, b, new_id) in enumerate(self.merges): self.id_to_bytes[new_id] = self.id_to_bytes[a] + self.id_to_bytes[b] self.merge_ranks[(a, b)] = (rank, new_id) @classmethod def load(cls, path): with open(path, "r", encoding="utf-8") as f: data = json.load(f) return cls(data["special_tokens"], data["merges"]) def _encode_word(self, word_bytes): ids = [self.byte_offset + b for b in word_bytes] while len(ids) >= 2: best_rank, best_pair, best_new_id = None, None, None for i in range(len(ids) - 1): info = self.merge_ranks.get((ids[i], ids[i + 1])) if info is not None and (best_rank is None or info[0] < best_rank): best_rank, best_new_id = info best_pair = (ids[i], ids[i + 1]) if best_pair is None: break a, b = best_pair merged, i, n = [], 0, len(ids) while i < n: if i + 1 < n and ids[i] == a and ids[i + 1] == b: merged.append(best_new_id) i += 2 else: merged.append(ids[i]) i += 1 ids = merged return ids def encode(self, text): ids = [] for part in SPECIAL_RE.split(text): if not part: continue sid = self.tok2id.get(part) if sid is not None: ids.append(sid) else: for chunk in PRETOK_RE.findall(part): ids.extend(self._encode_word(chunk.encode("utf-8"))) return ids def decode(self, ids): special_set = set(range(len(self.special_tokens))) chunks = [self.id_to_bytes[int(i)] for i in ids if int(i) not in special_set and int(i) in self.id_to_bytes] return b"".join(chunks).decode("utf-8", errors="replace") class RMSNorm(nn.Module): def __init__(self, dim, eps=1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight class SwiGLU(nn.Module): def __init__(self, dim, hidden_dim): super().__init__() self.w1 = nn.Linear(dim, hidden_dim, bias=False) self.w2 = nn.Linear(hidden_dim, dim, bias=False) self.w3 = nn.Linear(dim, hidden_dim, bias=False) def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x)) def precompute_rope_tables(dim, max_seq_len, theta=10000.0): inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim)) freqs = torch.outer(torch.arange(max_seq_len, dtype=torch.float32), inv_freq) emb = torch.cat((freqs, freqs), dim=-1) return emb.cos(), emb.sin() def rotate_half(x): half = x.shape[-1] // 2 return torch.cat((-x[..., half:], x[..., :half]), dim=-1) class RoPEAttention(nn.Module): def __init__(self, d_model, n_head): super().__init__() self.n_head = n_head self.head_dim = d_model // n_head self.wq = nn.Linear(d_model, d_model, bias=False) self.wk = nn.Linear(d_model, d_model, bias=False) self.wv = nn.Linear(d_model, d_model, bias=False) self.wo = nn.Linear(d_model, d_model, bias=False) self.q_norm = RMSNorm(self.head_dim) self.k_norm = RMSNorm(self.head_dim) def forward(self, x, cos, sin, attn_mask): B, T, C = x.shape q = self.q_norm(self.wq(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)) k = self.k_norm(self.wk(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)) v = self.wv(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2) q = (q * cos) + (rotate_half(q) * sin) k = (k * cos) + (rotate_half(k) * sin) att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(self.head_dim)) att = F.softmax(att.masked_fill(~attn_mask, float("-inf")).float(), dim=-1).type_as(v) return self.wo((att @ v).transpose(1, 2).contiguous().view(B, T, C)) class SLLMBlock(nn.Module): def __init__(self, d_model, n_head, hidden_dim): super().__init__() self.attn_norm = RMSNorm(d_model) self.attn = RoPEAttention(d_model, n_head) self.ffn_norm = RMSNorm(d_model) self.ffn = SwiGLU(d_model, hidden_dim) def forward(self, x, cos, sin, attn_mask): x = x + self.attn(self.attn_norm(x), cos, sin, attn_mask) return x + self.ffn(self.ffn_norm(x)) class SLLM(nn.Module): def __init__(self, vocab_size, d_model, n_head, n_layer, hidden_dim, block_size, pad_id=0): super().__init__() self.block_size = block_size self.pad_id = pad_id self.tok_emb = nn.Embedding(vocab_size, d_model) self.layers = nn.ModuleList([SLLMBlock(d_model, n_head, hidden_dim) for _ in range(n_layer)]) self.norm = RMSNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self.lm_head.weight = self.tok_emb.weight cos, sin = precompute_rope_tables(d_model // n_head, block_size) self.register_buffer("cos", cos, persistent=False) self.register_buffer("sin", sin, persistent=False) self.register_buffer("causal_mask", torch.tril(torch.ones(block_size, block_size, dtype=torch.bool)).view(1, 1, block_size, block_size), persistent=False) self.register_buffer("eye_mask", torch.eye(block_size, dtype=torch.bool).view(1, 1, block_size, block_size), persistent=False) def forward(self, idx): B, T = idx.shape non_pad_k = (idx != self.pad_id).unsqueeze(1).unsqueeze(2) pad_q_diag = (idx == self.pad_id).unsqueeze(1).unsqueeze(3) & self.eye_mask[:, :, :T, :T] attn_mask = self.causal_mask[:, :, :T, :T] & (non_pad_k | pad_q_diag) x = self.tok_emb(idx) cos, sin = self.cos[:T].unsqueeze(0).unsqueeze(0), self.sin[:T].unsqueeze(0).unsqueeze(0) for layer in self.layers: x = layer(x, cos, sin, attn_mask) return self.lm_head(self.norm(x)) with open("config.json", "r") as f: cfg = json.load(f) tokenizer = ByteBPETokenizer.load("tokenizer.json") model = SLLM( vocab_size=cfg["vocab_size"], d_model=cfg["d_model"], n_head=cfg["n_head"], n_layer=cfg["n_layer"], hidden_dim=cfg["hidden_dim"], block_size=cfg["block_size"], pad_id=tokenizer.pad_id, ).to(device) load_model(model, "model.safetensors") model.eval() print("\n" + "=" * 50) print("Connected to Veirded/SLLM-0.6 (1.0M Parameters)") print("Single-turn mode. Type 'quit' to exit.") print("=" * 50) while True: uin = input("\nYou: ").strip() if not uin or uin.lower() in ("quit", "exit"): break ids = tokenizer.encode(f"<|user|> {uin}\n<|ai|>")[-cfg["block_size"]:] idx = torch.tensor([ids], dtype=torch.long, device=device) gen = [] with torch.no_grad(): for _ in range(64): logits = model(idx[:, -cfg["block_size"]:])[0, -1].float() logits[tokenizer.pad_id] = -float("inf") for t in set(idx[0, -64:].tolist()): logits[t] = logits[t] * 1.15 if logits[t] < 0 else logits[t] / 1.15 logits = logits / 0.5 v, _ = torch.topk(logits, 15) logits[logits < v[-1]] = -float("inf") nid = int(torch.multinomial(F.softmax(logits, dim=-1), 1).item()) if nid in (tokenizer.end_id, tokenizer.user_id): break gen.append(nid) idx = torch.cat((idx, torch.tensor([[nid]], dtype=torch.long, device=device)), dim=1) print(f"SLLM: {tokenizer.decode(gen).strip()}")