import os import re import json import torch import torch.nn as nn import torch.nn.functional as F if torch.backends.mps.is_available(): device = torch.device("mps") print("Using Apple Silicon Metal (MPS) acceleration.") elif torch.cuda.is_available(): device = torch.device("cuda") print(f"Using NVIDIA CUDA acceleration: {torch.cuda.get_device_name(0)}") else: device = torch.device("cpu") print("Using CPU inference.") class RMSNorm(nn.Module): def __init__(self, dim, eps=1e-6): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight class SwiGLU(nn.Module): def __init__(self, dim, hidden_dim): super().__init__() self.w1 = nn.Linear(dim, hidden_dim, bias=False) self.w2 = nn.Linear(hidden_dim, dim, bias=False) self.w3 = nn.Linear(dim, hidden_dim, bias=False) def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x)) def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0): freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) t = torch.arange(end, device=freqs.device) freqs = torch.outer(t, freqs).float() return torch.polar(torch.ones_like(freqs), freqs) def apply_rotary_emb(xq, xk, freqs_cis): xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) freqs_cis = freqs_cis[:xq_.shape[1]].unsqueeze(0).unsqueeze(2) xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) return xq_out.type_as(xq), xk_out.type_as(xk) class RoPEAttention(nn.Module): def __init__(self, d_model=112, n_head=4, block_size=256): super().__init__() self.n_head = n_head self.head_dim = d_model // n_head self.wq = nn.Linear(d_model, d_model, bias=False) self.wk = nn.Linear(d_model, d_model, bias=False) self.wv = nn.Linear(d_model, d_model, bias=False) self.wo = nn.Linear(d_model, d_model, bias=False) self.register_buffer('bias', torch.tril(torch.ones(block_size, block_size)).view(1, 1, block_size, block_size)) def forward(self, x, freqs_cis): B, T, C = x.shape q = self.wq(x).view(B, T, self.n_head, self.head_dim) k = self.wk(x).view(B, T, self.n_head, self.head_dim) v = self.wv(x).view(B, T, self.n_head, self.head_dim) q, k = apply_rotary_emb(q, k, freqs_cis) q = q.transpose(1, 2) k = k.transpose(1, 2) v = v.transpose(1, 2) att = (q @ k.transpose(-2, -1)) * (1.0 / (self.head_dim ** 0.5)) att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf')) att = F.softmax(att, dim=-1) y = att @ v y = y.transpose(1, 2).contiguous().view(B, T, C) return self.wo(y) class NovaBlock(nn.Module): def __init__(self, d_model=112, n_head=4, hidden_dim=298, block_size=256): super().__init__() self.attn_norm = RMSNorm(d_model) self.attn = RoPEAttention(d_model, n_head, block_size) self.attn_gate = nn.Linear(d_model, d_model, bias=False) self.ffn_norm = RMSNorm(d_model) self.ffn = SwiGLU(d_model, hidden_dim) self.ffn_gate = nn.Linear(d_model, d_model, bias=False) def forward(self, x, freqs_cis): x = x + torch.sigmoid(self.attn_gate(x)) * self.attn(self.attn_norm(x), freqs_cis) x = x + torch.sigmoid(self.ffn_gate(x)) * self.ffn(self.ffn_norm(x)) return x class SLLM(nn.Module): def __init__(self, vocab_size=9385, d_model=112, n_head=4, n_layer=4, block_size=256): super().__init__() self.block_size = block_size self.tok_emb = nn.Embedding(vocab_size, d_model) self.layers = nn.ModuleList([NovaBlock(d_model, n_head, int(2 * (4 * d_model) / 3), block_size) for _ in range(n_layer)]) self.norm = RMSNorm(d_model) self.highway_gain = nn.Parameter(torch.tensor(0.1)) self.lm_head = nn.Linear(d_model, vocab_size, bias=False) self.register_buffer('freqs_cis', precompute_freqs_cis(d_model // n_head, block_size)) def forward(self, idx): B, T = idx.shape x_emb = self.tok_emb(idx) freqs_cis = self.freqs_cis[:T] x = x_emb for layer in self.layers: x = layer(x, freqs_cis) x_out = self.norm(x + self.highway_gain * x_emb) return self.lm_head(x_out) dir_path = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(dir_path, "tokenizer_sllm_t.json"), "r", encoding="utf-8") as f: tok_data = json.load(f) tok2id = tok_data["tok2id"] id2tok = {int(k): v for k, v in tok_data["id2tok"].items()} end_id = tok2id[""] pattern = r"(?:|\w+|[^\w\s])" model = SLLM(vocab_size=len(tok2id), block_size=256).to(device) model.load_state_dict(torch.load(os.path.join(dir_path, "sllm_weights.pt"), map_location=device)) model.eval() def detokenize(tokens): out = "" for t in tokens: if t in {".", ",", "!", "?", "'", ";", ":", ")", "]"}: out = out.rstrip() + t + " " elif t in {"(", "["}: out += t else: out += t + " " return out.strip() print("\n" + "="*50) print("🤖 Connected to Veirded/SLLM-0.5 (~2.8M Parameters)") print("Type 'exit' to quit.") print("="*50) history = [] while True: try: user_msg = input("\nYou: ").strip() except (KeyboardInterrupt, EOFError): break if not user_msg or user_msg.lower() in ("quit", "exit"): break user_tokens = re.findall(pattern, user_msg) history.extend(user_tokens) ctx = history[-256:] idx = torch.tensor([[tok2id.get(t, 0) for t in ctx]], dtype=torch.long, device=device) reply_tokens = [] with torch.no_grad(): for step in range(50): logits = model(idx[:, -256:])[:, -1, :] / 0.75 if step < 8: logits[0, end_id] = -float('Inf') for past_tok in set(reply_tokens[-6:]): logits[0, tok2id.get(past_tok, 0)] /= 1.4 v, _ = torch.topk(logits, min(30, logits.size(-1))) logits[logits < v[:, [-1]]] = -float('Inf') probs = F.softmax(logits, dim=-1) next_id = torch.multinomial(probs, num_samples=1) token_str = id2tok[next_id.item()] if token_str == "": break reply_tokens.append(token_str) idx = torch.cat((idx, next_id), dim=1) print(f"Bot: {detokenize(reply_tokens)}") history.extend(reply_tokens)