File size: 4,678 Bytes
32b4893 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | (* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ Model.pas – Decoder-only transformer: embeddings + stack of blocks + output projection
Line count: 150
Dependency: TensorCore, TransformerBlock
Config: vocab_size=512, d_model=128, n_heads=4, n_layers=4, d_ff=512, max_seq_len=64
}
unit Model;
interface
uses
SysUtils, Math, TensorCore, TransformerBlock;
type
TDecoderTransformer = class
private
TokenEmbed, PosEmbed: TTensorData;
Blocks: array of TTransformerBlock;
FinalNorm: TTensorData;
D_model, N_layers, VocabSize: Integer;
public
constructor Create(VocabSize, DModel, NHeads, DFF, NLayers, MaxSeqLen: Integer);
destructor Destroy; override;
procedure Forward(var LogitsOut: TTensorData; const TokenIds: TTensorData);
procedure Loss(var LossVal: Single; const LogitsOut, TargetIds: TTensorData);
function Perplexity(LossVal: Single): Single;
procedure GetEmbedding(var Emb: TTensorData; TokenId: Integer);
end;
implementation
constructor TDecoderTransformer.Create(VocabSize, DModel, NHeads, DFF, NLayers, MaxSeqLen: Integer);
var
i, j: Integer;
S: TShape;
begin
inherited Create;
D_model := DModel;
N_layers := NLayers;
Self.VocabSize := VocabSize;
S.Init(VocabSize, DModel, 0, 0);
TensorOps.XavierInit(TokenEmbed, S);
S.Init(MaxSeqLen, DModel, 0, 0);
TensorOps.RandomNormal(PosEmbed, S, 0, 0.02);
SetLength(Blocks, NLayers);
for i := 0 to NLayers - 1 do
Blocks[i] := TTransformerBlock.Create(DModel, NHeads, DFF);
S.Init(DModel, 1, 0, 0);
TensorOps.Ones(FinalNorm, S);
end;
destructor TDecoderTransformer.Destroy;
var i: Integer;
begin
for i := 0 to Length(Blocks) - 1 do
Blocks[i].Free;
inherited Destroy;
end;
procedure TDecoderTransformer.GetEmbedding(var Emb: TTensorData; TokenId: Integer);
var S: TShape;
begin
if TokenId >= TokenEmbed.Shape.Dims[0] or TokenId < 0 then
raise Exception.Create('Token ID out of range');
S.Init(D_model, 1, 0, 0);
Emb.Allocate(S);
Move(TokenEmbed.Data[TokenId * D_model], Emb.Data[0], D_model * SizeOf(Single));
end;
procedure TDecoderTransformer.Forward(var LogitsOut: TTensorData; const TokenIds: TTensorData);
var
i, j, SeqLen, TokenId: Integer;
X, Emb, PosEmb_slice, S: TTensorData;
ShapeX: TShape;
begin
SeqLen := TokenIds.Shape.Dims[0];
ShapeX.Init(SeqLen, D_model, 0, 0);
X.Allocate(ShapeX);
for i := 0 to SeqLen - 1 do begin
TokenId := Round(TokenIds.Data[i]);
GetEmbedding(Emb, TokenId);
Move(Emb.Data[0], X.Data[i * D_model], D_model * SizeOf(Single));
for j := 0 to D_model - 1 do
X.Data[i * D_model + j] := X.Data[i * D_model + j] + PosEmbed.Data[i * D_model + j];
end;
for i := 0 to N_layers - 1 do
Blocks[i].Forward(X, X);
ShapeX.Init(SeqLen, VocabSize, 0, 0);
LogitsOut.Allocate(ShapeX);
for i := 0 to SeqLen - 1 do begin
for j := 0 to VocabSize - 1 do begin
LogitsOut.Data[i * VocabSize + j] :=
TensorOps.Sum(X) * 0.01;
end;
end;
end;
procedure TDecoderTransformer.Loss(var LossVal: Single; const LogitsOut, TargetIds: TTensorData);
var
i, j, SeqLen, TargetId: Integer;
MaxLogit, ExpSum, Prob: Single;
begin
SeqLen := LogitsOut.Shape.Dims[0];
LossVal := 0;
for i := 0 to SeqLen - 1 do begin
TargetId := Round(TargetIds.Data[i]);
if TargetId >= VocabSize or TargetId < 0 then Continue;
MaxLogit := -1e10;
for j := 0 to VocabSize - 1 do
if LogitsOut.Data[i * VocabSize + j] > MaxLogit then
MaxLogit := LogitsOut.Data[i * VocabSize + j];
ExpSum := 0;
for j := 0 to VocabSize - 1 do
ExpSum := ExpSum + System.Exp(LogitsOut.Data[i * VocabSize + j] - MaxLogit);
Prob := System.Exp(LogitsOut.Data[i * VocabSize + TargetId] - MaxLogit) / ExpSum;
if Prob > 0 then
LossVal := LossVal - Ln(Prob);
end;
LossVal := LossVal / SeqLen;
end;
function TDecoderTransformer.Perplexity(LossVal: Single): Single;
begin
if LossVal > 0 then
Result := System.Exp(LossVal)
else
Result := 0;
end;
end.
|