Hugging Face's logo

Snapkitty
/
sparse-router-formal

Rust
snapkitty
october-2026-drop
pascal
lean4
c
kani
work-stealing
scheduler
formal-verification
Model card Files
xet
 Community
sparse-router-formal / src /pascal /Model.pas
SNAPKITTYWEST's picture
SNAPKITTYWEST
October 2026 main drop: mirror from GitHub
32b4893 verified
Raw History Blame Contribute Delete
4.68 kB
(* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ Model.pas – Decoder-only transformer: embeddings + stack of blocks + output projection
Line count: 150
Dependency: TensorCore, TransformerBlock
Config: vocab_size=512, d_model=128, n_heads=4, n_layers=4, d_ff=512, max_seq_len=64
}
unit Model;
interface
uses
SysUtils, Math, TensorCore, TransformerBlock;
type
TDecoderTransformer = class
private
TokenEmbed, PosEmbed: TTensorData;
Blocks: array of TTransformerBlock;
FinalNorm: TTensorData;
D_model, N_layers, VocabSize: Integer;
public
constructor Create(VocabSize, DModel, NHeads, DFF, NLayers, MaxSeqLen: Integer);
destructor Destroy; override;
procedure Forward(var LogitsOut: TTensorData; const TokenIds: TTensorData);
procedure Loss(var LossVal: Single; const LogitsOut, TargetIds: TTensorData);
function Perplexity(LossVal: Single): Single;
procedure GetEmbedding(var Emb: TTensorData; TokenId: Integer);
end;
implementation
constructor TDecoderTransformer.Create(VocabSize, DModel, NHeads, DFF, NLayers, MaxSeqLen: Integer);
var
i, j: Integer;
S: TShape;
begin
inherited Create;
D_model := DModel;
N_layers := NLayers;
Self.VocabSize := VocabSize;
S.Init(VocabSize, DModel, 0, 0);
TensorOps.XavierInit(TokenEmbed, S);
S.Init(MaxSeqLen, DModel, 0, 0);
TensorOps.RandomNormal(PosEmbed, S, 0, 0.02);
SetLength(Blocks, NLayers);
for i := 0 to NLayers - 1 do
Blocks[i] := TTransformerBlock.Create(DModel, NHeads, DFF);
S.Init(DModel, 1, 0, 0);
TensorOps.Ones(FinalNorm, S);
end;
destructor TDecoderTransformer.Destroy;
var i: Integer;
begin
for i := 0 to Length(Blocks) - 1 do
Blocks[i].Free;
inherited Destroy;
end;
procedure TDecoderTransformer.GetEmbedding(var Emb: TTensorData; TokenId: Integer);
var S: TShape;
begin
if TokenId >= TokenEmbed.Shape.Dims[0] or TokenId < 0 then
raise Exception.Create('Token ID out of range');
S.Init(D_model, 1, 0, 0);
Emb.Allocate(S);
Move(TokenEmbed.Data[TokenId * D_model], Emb.Data[0], D_model * SizeOf(Single));
end;
procedure TDecoderTransformer.Forward(var LogitsOut: TTensorData; const TokenIds: TTensorData);
var
i, j, SeqLen, TokenId: Integer;
X, Emb, PosEmb_slice, S: TTensorData;
ShapeX: TShape;
begin
SeqLen := TokenIds.Shape.Dims[0];
ShapeX.Init(SeqLen, D_model, 0, 0);
X.Allocate(ShapeX);
for i := 0 to SeqLen - 1 do begin
TokenId := Round(TokenIds.Data[i]);
GetEmbedding(Emb, TokenId);
Move(Emb.Data[0], X.Data[i * D_model], D_model * SizeOf(Single));
for j := 0 to D_model - 1 do
X.Data[i * D_model + j] := X.Data[i * D_model + j] + PosEmbed.Data[i * D_model + j];
end;
for i := 0 to N_layers - 1 do
Blocks[i].Forward(X, X);
ShapeX.Init(SeqLen, VocabSize, 0, 0);
LogitsOut.Allocate(ShapeX);
for i := 0 to SeqLen - 1 do begin
for j := 0 to VocabSize - 1 do begin
LogitsOut.Data[i * VocabSize + j] :=
TensorOps.Sum(X) * 0.01;
end;
end;
end;
procedure TDecoderTransformer.Loss(var LossVal: Single; const LogitsOut, TargetIds: TTensorData);
var
i, j, SeqLen, TargetId: Integer;
MaxLogit, ExpSum, Prob: Single;
begin
SeqLen := LogitsOut.Shape.Dims[0];
LossVal := 0;
for i := 0 to SeqLen - 1 do begin
TargetId := Round(TargetIds.Data[i]);
if TargetId >= VocabSize or TargetId < 0 then Continue;
MaxLogit := -1e10;
for j := 0 to VocabSize - 1 do
if LogitsOut.Data[i * VocabSize + j] > MaxLogit then
MaxLogit := LogitsOut.Data[i * VocabSize + j];
ExpSum := 0;
for j := 0 to VocabSize - 1 do
ExpSum := ExpSum + System.Exp(LogitsOut.Data[i * VocabSize + j] - MaxLogit);
Prob := System.Exp(LogitsOut.Data[i * VocabSize + TargetId] - MaxLogit) / ExpSum;
if Prob > 0 then
LossVal := LossVal - Ln(Prob);
end;
LossVal := LossVal / SeqLen;
end;
function TDecoderTransformer.Perplexity(LossVal: Single): Single;
begin
if LossVal > 0 then
Result := System.Exp(LossVal)
else
Result := 0;
end;
end.