Hugging Face's logo

Snapkitty
/
sparse-router-formal

Rust
snapkitty
october-2026-drop
pascal
lean4
c
kani
work-stealing
scheduler
formal-verification
Model card Files
xet
 Community
sparse-router-formal / src /pascal /transformer_pascal_200.pas
SNAPKITTYWEST's picture
SNAPKITTYWEST
October 2026 main drop: mirror from GitHub
32b4893 verified
Raw History Blame Contribute Delete
7.15 kB
(* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ transformer_pascal_200.pas
Pascal Transformer — 200 Lines
Decoder-only transformer block: attention + MLP + layer norm + residual.
Dependency: TensorCore.pas (TTensor, TensorMatMul, TensorAdd, etc.)
License: AGPL-3.0-or-later
}
unit transformer_pascal_200;
interface
uses SysUtils, Math, TensorCore;
const
ATTN_NEG_INF = -1.0e9;
GELU_SQRT2PI = 0.7978845608028654;
GELU_COEFF = 0.044715;
LN_EPS = 1.0e-5;
type
TAttnWeights = record
WQ, WK, WV, WO : TTensor; { each D_model x D_head }
D_model, D_head, N_heads : Integer;
end;
TMLPWeights = record
W1, B1, W2, B2 : TTensor; { W1: D_ff x D_model, W2: D_model x D_ff }
D_model, D_ff : Integer;
end;
TLNWeights = record
Gamma, Beta : TTensor; { D_model }
D_model : Integer;
end;
TBlockWeights = record
Attn : TAttnWeights;
MLP : TMLPWeights;
LN1, LN2: TLNWeights;
D_model : Integer;
end;
{ Layer normalization: y = gamma * (x - mean) / sqrt(var + eps) + beta }
function LayerNorm(const X, Gamma, Beta: TTensor; N, D: Integer): TTensor;
{ GELU activation: x * 0.5 * (1 + tanh(sqrt(2/pi)*(x + 0.044715*x^3))) }
function GELU(const X: TTensor; Len: Integer): TTensor;
{ Causal softmax: numerically stable, masks upper triangle }
function CausalSoftmax(const Scores: TTensor; N: Integer): TTensor;
{ Scaled dot-product attention for one head }
function SingleHeadAttention(const Q, K, V: TTensor; N, DH: Integer): TTensor;
{ Full transformer block forward: X -> X + Attn(LN1(X)); Y + MLP(LN2(Y)) }
function BlockForward(const W: TBlockWeights; const X: TTensor; N: Integer): TTensor;
implementation
function LayerNorm(const X, Gamma, Beta: TTensor; N, D: Integer): TTensor;
var
I, Dim: Integer;
Mu, V, Inv: Double;
begin
Result := TensorCreate([N, D]);
for I := 0 to N - 1 do
begin
Mu := 0;
for Dim := 0 to D - 1 do Mu := Mu + TensorGet(X, I * D + Dim);
Mu := Mu / D;
V := 0;
for Dim := 0 to D - 1 do V := V + Sqr(TensorGet(X, I * D + Dim) - Mu);
V := V / D;
Inv := 1.0 / Sqrt(V + LN_EPS);
for Dim := 0 to D - 1 do
TensorSet(Result, I * D + Dim,
TensorGet(Gamma, Dim) * (TensorGet(X, I * D + Dim) - Mu) * Inv + TensorGet(Beta, Dim));
end;
end;
function GELU(const X: TTensor; Len: Integer): TTensor;
var
I: Integer;
XV, X3, Inner, ExpV: Double;
begin
Result := TensorCreate([Len]);
for I := 0 to Len - 1 do
begin
XV := TensorGet(X, I);
X3 := XV * XV * XV;
Inner := GELU_SQRT2PI * (XV + GELU_COEFF * X3);
ExpV := Exp(2.0 * Inner);
TensorSet(Result, I, 0.5 * XV * (1.0 + (ExpV - 1.0) / (ExpV + 1.0)));
end;
end;
function CausalSoftmax(const Scores: TTensor; N: Integer): TTensor;
var
I, J: Integer;
MxV, SumV, V: Double;
begin
Result := TensorCreate([N, N]);
for I := 0 to N - 1 do
begin
MxV := ATTN_NEG_INF;
for J := 0 to N - 1 do
begin
V := if J > I then ATTN_NEG_INF else TensorGet(Scores, I * N + J);
if V > MxV then MxV := V;
end;
SumV := 0;
for J := 0 to N - 1 do
begin
V := if J > I then ATTN_NEG_INF else TensorGet(Scores, I * N + J);
TensorSet(Result, I * N + J, Exp(V - MxV));
SumV := SumV + TensorGet(Result, I * N + J);
end;
if SumV > 1.0e-12 then
for J := 0 to N - 1 do
TensorSet(Result, I * N + J, TensorGet(Result, I * N + J) / SumV);
end;
end;
function SingleHeadAttention(const Q, K, V: TTensor; N, DH: Integer): TTensor;
var
I, J, Dim: Integer;
Scale, Dot, Acc: Double;
Scores, Weights: TTensor;
begin
Scale := 1.0 / Sqrt(DH);
Scores := TensorCreate([N, N]);
for I := 0 to N - 1 do
for J := 0 to N - 1 do
begin
Dot := 0;
for Dim := 0 to DH - 1 do
Dot := Dot + TensorGet(Q, I * DH + Dim) * TensorGet(K, J * DH + Dim);
TensorSet(Scores, I * N + J, Dot * Scale);
end;
Weights := CausalSoftmax(Scores, N);
Result := TensorCreate([N, DH]);
for I := 0 to N - 1 do
for Dim := 0 to DH - 1 do
begin
Acc := 0;
for J := 0 to N - 1 do
Acc := Acc + TensorGet(Weights, I * N + J) * TensorGet(V, J * DH + Dim);
TensorSet(Result, I * DH + Dim, Acc);
end;
end;
function BlockForward(const W: TBlockWeights; const X: TTensor; N: Integer): TTensor;
var
DM, DH: Integer;
LN1Out, Q, K, V, AttnOut, Res1, LN2Out, Pre, Act, MlpOut: TTensor;
I, Dim, K2: Integer;
Acc: Double;
begin
DM := W.D_model;
DH := W.Attn.D_head;
{ Pre-norm attention }
LN1Out := LayerNorm(X, W.LN1.Gamma, W.LN1.Beta, N, DM);
Q := TensorCreate([N, DH]); K := TensorCreate([N, DH]); V := TensorCreate([N, DH]);
for I := 0 to N - 1 do
for Dim := 0 to DH - 1 do
begin
Acc := 0; for K2 := 0 to DM - 1 do Acc := Acc + TensorGet(LN1Out, I*DM+K2) * TensorGet(W.Attn.WQ, Dim*DM+K2);
TensorSet(Q, I*DH+Dim, Acc);
Acc := 0; for K2 := 0 to DM - 1 do Acc := Acc + TensorGet(LN1Out, I*DM+K2) * TensorGet(W.Attn.WK, Dim*DM+K2);
TensorSet(K, I*DH+Dim, Acc);
Acc := 0; for K2 := 0 to DM - 1 do Acc := Acc + TensorGet(LN1Out, I*DM+K2) * TensorGet(W.Attn.WV, Dim*DM+K2);
TensorSet(V, I*DH+Dim, Acc);
end;
AttnOut := SingleHeadAttention(Q, K, V, N, DH);
{ Project back and residual }
Res1 := TensorCreate([N, DM]);
for I := 0 to N - 1 do
for Dim := 0 to DM - 1 do
begin
Acc := 0; for K2 := 0 to DH - 1 do Acc := Acc + TensorGet(AttnOut, I*DH+K2) * TensorGet(W.Attn.WO, Dim*DH+K2);
TensorSet(Res1, I*DM+Dim, TensorGet(X, I*DM+Dim) + Acc);
end;
{ Pre-norm MLP }
LN2Out := LayerNorm(Res1, W.LN2.Gamma, W.LN2.Beta, N, DM);
Pre := TensorCreate([N * W.MLP.D_ff]);
for I := 0 to N - 1 do
for Dim := 0 to W.MLP.D_ff - 1 do
begin
Acc := TensorGet(W.MLP.B1, Dim);
for K2 := 0 to DM - 1 do Acc := Acc + TensorGet(LN2Out, I*DM+K2) * TensorGet(W.MLP.W1, Dim*DM+K2);
TensorSet(Pre, I*W.MLP.D_ff+Dim, Acc);
end;
Act := GELU(Pre, N * W.MLP.D_ff);
MlpOut := TensorCreate([N, DM]);
for I := 0 to N - 1 do
for Dim := 0 to DM - 1 do
begin
Acc := TensorGet(W.MLP.B2, Dim);
for K2 := 0 to W.MLP.D_ff - 1 do Acc := Acc + TensorGet(Act, I*W.MLP.D_ff+K2) * TensorGet(W.MLP.W2, Dim*W.MLP.D_ff+K2);
TensorSet(MlpOut, I*DM+Dim, Acc);
end;
{ Final residual }
Result := TensorCreate([N, DM]);
for I := 0 to N * DM - 1 do
TensorSet(Result, I, TensorGet(Res1, I) + TensorGet(MlpOut, I));
end;
end.