Gladios 0.3B ENG — Prototype
A from-scratch GPT-style chat model (~300M parameters), trained on English conversational data.
Model Description
Gladios is a small conversational language model, built entirely from scratch — no pretrained base, no fine-tuning of an existing LLM. Both the byte-level BPE tokenizer and the transformer weights were trained from random initialization on the datasets listed below.
This is a genuine hobbyist prototype/test project, built by a single passionate individual with the help of AI tools during development — not a team, not a company, not a production-ready assistant. At ~300M parameters trained on a modest dataset, expect coherent but simple answers — not GPT-4-level reasoning.
Full training code and documentation: GitHub — Plimb-ai/gladios-0.3b-eng-prototype
Architecture
A decoder-only transformer, implemented from scratch in PyTorch — no
transformers.GPT2Model or similar off-the-shelf model class, only nn.Linear, nn.LayerNorm,
nn.Embedding and PyTorch's scaled_dot_product_attention.
| Component | Detail |
|---|---|
| Parameters | ~300M |
| Layers | 24 |
| Attention heads | 16 |
| Embedding dim | 1024 |
| Context length | 256 tokens |
| Vocabulary | 8,000 tokens (custom-trained byte-level BPE) |
| Positional encoding | Learned absolute position embeddings |
Files in this repository
| File | Description |
|---|---|
gladios-0.3b-eng-prototype.pt |
Model weights (lightweight export — no optimizer state), ~1.25 GB |
vocab.json |
BPE tokenizer vocabulary |
merges.txt |
BPE tokenizer merge rules |
Training Data
Trained on a combination of public English conversational/instruction datasets:
- OpenAssistant/oasst1
- OpenAssistant/oasst2
- tatsu-lab/alpaca
- databricks/databricks-dolly-15k
- HuggingFaceH4/ultrachat_200k (20k-example subsample)
Each example is formatted as <|user|> question <|bot|> answer <|endoftext|>.
How to Use
1. Install dependencies
pip install torch tokenizers huggingface_hub
2. Download the weights and tokenizer
from huggingface_hub import snapshot_download
local_dir = snapshot_download(repo_id="plimb/gladios-0.3b-eng-prototype")
3. Rebuild the model class
The GPT model class isn't packaged as a pip-installable library — copy it from the snippet
below (same code used to train this model):
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class CausalSelfAttention(nn.Module):
def __init__(self, n_embd, n_head, block_size, dropout=0.1):
super().__init__()
assert n_embd % n_head == 0
self.n_head = n_head
self.n_embd = n_embd
self.head_dim = n_embd // n_head
self.qkv = nn.Linear(n_embd, 3 * n_embd, bias=False)
self.proj = nn.Linear(n_embd, n_embd, bias=False)
self.attn_dropout = nn.Dropout(dropout)
self.resid_dropout = nn.Dropout(dropout)
self.dropout = dropout
self.register_buffer(
"mask",
torch.tril(torch.ones(block_size, block_size)).view(1, 1, block_size, block_size),
)
self.flash = hasattr(F, "scaled_dot_product_attention")
def forward(self, x):
B, T, C = x.shape
qkv = self.qkv(x)
q, k, v = qkv.split(self.n_embd, dim=2)
q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
k = k.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
v = v.view(B, T, self.n_head, self.head_dim).transpose(1, 2)
if self.flash:
y = F.scaled_dot_product_attention(
q, k, v, attn_mask=None,
dropout_p=self.dropout if self.training else 0.0,
is_causal=True,
)
else:
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(self.head_dim))
att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float("-inf"))
att = F.softmax(att, dim=-1)
att = self.attn_dropout(att)
y = att @ v
y = y.transpose(1, 2).contiguous().view(B, T, C)
y = self.resid_dropout(self.proj(y))
return y
class MLP(nn.Module):
def __init__(self, n_embd, dropout=0.1):
super().__init__()
self.fc1 = nn.Linear(n_embd, 4 * n_embd, bias=False)
self.gelu = nn.GELU()
self.fc2 = nn.Linear(4 * n_embd, n_embd, bias=False)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
return self.dropout(self.fc2(self.gelu(self.fc1(x))))
class Block(nn.Module):
def __init__(self, n_embd, n_head, block_size, dropout=0.1):
super().__init__()
self.ln1 = nn.LayerNorm(n_embd)
self.attn = CausalSelfAttention(n_embd, n_head, block_size, dropout)
self.ln2 = nn.LayerNorm(n_embd)
self.mlp = MLP(n_embd, dropout)
def forward(self, x):
x = x + self.attn(self.ln1(x))
x = x + self.mlp(self.ln2(x))
return x
class GPT(nn.Module):
def __init__(self, vocab_size, block_size, n_layer=24, n_head=16, n_embd=1024, dropout=0.0):
super().__init__()
self.block_size = block_size
self.tok_emb = nn.Embedding(vocab_size, n_embd)
self.pos_emb = nn.Embedding(block_size, n_embd)
self.drop = nn.Dropout(dropout)
self.blocks = nn.ModuleList([
Block(n_embd, n_head, block_size, dropout) for _ in range(n_layer)
])
self.ln_f = nn.LayerNorm(n_embd)
self.head = nn.Linear(n_embd, vocab_size, bias=False)
self.tok_emb.weight = self.head.weight
def forward(self, idx, targets=None):
B, T = idx.shape
pos = torch.arange(0, T, dtype=torch.long, device=idx.device)
x = self.tok_emb(idx) + self.pos_emb(pos)
x = self.drop(x)
for block in self.blocks:
x = block(x)
x = self.ln_f(x)
logits = self.head(x)
loss = None
if targets is not None:
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
return logits, loss
@torch.no_grad()
def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None, stop_token=None):
self.eval()
for _ in range(max_new_tokens):
idx_cond = idx if idx.size(1) <= self.block_size else idx[:, -self.block_size:]
logits, _ = self(idx_cond)
logits = logits[:, -1, :] / max(temperature, 1e-6)
if top_k is not None:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float("inf")
probs = F.softmax(logits, dim=-1)
next_id = torch.multinomial(probs, num_samples=1)
idx = torch.cat((idx, next_id), dim=1)
if stop_token is not None and next_id.item() == stop_token:
break
self.train()
return idx
4. Load the weights and tokenizer
import os
import torch
from tokenizers import ByteLevelBPETokenizer
device = "cuda" if torch.cuda.is_available() else "cpu"
ckpt = torch.load(os.path.join(local_dir, "gladios-0.3b-eng-prototype.pt"), map_location=device)
cfg = ckpt["config"]
model = GPT(
vocab_size=cfg["vocab_size"],
block_size=cfg["block_size"],
n_layer=cfg["n_layer"],
n_head=cfg["n_head"],
n_embd=cfg["n_embd"],
dropout=0.0,
).to(device)
model.load_state_dict(ckpt["model"])
model.eval()
tokenizer = ByteLevelBPETokenizer(
os.path.join(local_dir, "vocab.json"),
os.path.join(local_dir, "merges.txt"),
)
eot_id = tokenizer.token_to_id("<|endoftext|>")
user_id = tokenizer.token_to_id("<|user|>")
bot_id = tokenizer.token_to_id("<|bot|>")
5. Chat
def chat(message, max_new_tokens=100, temperature=0.8, top_k=40):
ids = [user_id] + tokenizer.encode(message).ids + [bot_id]
x = torch.tensor([ids], dtype=torch.long, device=device)
out = model.generate(
x, max_new_tokens=max_new_tokens, temperature=temperature, top_k=top_k,
stop_token=eot_id,
)
response_ids = out[0, len(ids):].tolist()
if response_ids and response_ids[-1] == eot_id:
response_ids = response_ids[:-1]
return tokenizer.decode(response_ids).strip()
print(chat("What is the capital of France?"))
Training Details
- Optimizer: AdamW, cosine learning rate schedule with warmup
- Precision: bfloat16 autocast + gradient scaling on GPU
- Regularization: dropout 0.2, weight decay 0.15
- Hardware: single Colab GPU (T4 / A100 depending on availability)
- Checkpointing: evaluated every 200 iterations, keeping the best validation loss
Limitations
- English only — no meaningful multilingual capability; prompting in other languages produces incoherent output.
- Overfitting risk — with ~300M parameters trained on a comparatively small combined corpus, validation loss plateaus after a certain point even with regularization (dropout, weight decay) applied. Treat outputs as a learning-project result, not a benchmark-grade model.
- No safety alignment — no RLHF, no content filtering, no moderation layer. This is a from-scratch research/learning prototype.
- Short context window (256 tokens) — long conversations lose earlier context quickly.
License
MIT. Note that the training datasets each carry their own license — check each dataset's card before redistribution.
Citation
If you reference this project, please link back to the GitHub repository.