File size: 4,297 Bytes
32b4893 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | (* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ TransformerBlock.pas – Pre-norm transformer block with attention and MLP
Line count: 200
Dependency: TensorCore, Activations, Attention
Structure: x -> LayerNorm -> MultiHeadAttn -> residual add ->
LayerNorm -> MLP(Dense + GELU + Dense) -> residual add
}
unit TransformerBlock;
interface
uses
SysUtils, Math, TensorCore, Activations, Attention, JacobianCore;
type
TTransformerBlock = class
private
W_in, W_out: TTensorData;
Norm1, Norm2: TTensorData;
public
Attn: TMultiHeadAttention;
D_model, D_ff: Integer;
constructor Create(DModel, NHeads, DFF: Integer);
destructor Destroy; override;
procedure Forward(var Output: TTensorData; const Input: TTensorData);
procedure Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input: TTensorData);
procedure ForwardWithJacobian(var Output: TTensorData; var Jac: TJacobianBlock;
const Input: TTensorData);
end;
implementation
constructor TTransformerBlock.Create(DModel, NHeads, DFF: Integer);
var S: TShape;
begin
inherited Create;
D_model := DModel;
D_ff := DFF;
Attn := TMultiHeadAttention.Create;
Attn.Initialize(DModel, NHeads);
S.Init(DFF, DModel, 0, 0);
TensorOps.XavierInit(W_in, S);
S.Init(DModel, DFF, 0, 0);
TensorOps.XavierInit(W_out, S);
S.Init(DModel, 1, 0, 0);
TensorOps.Ones(Norm1, S);
TensorOps.Ones(Norm2, S);
end;
destructor TTransformerBlock.Destroy;
begin
Attn.Free;
inherited Destroy;
end;
procedure TTransformerBlock.Forward(var Output: TTensorData; const Input: TTensorData);
var
Norm_Out, Attn_Out, Residual1, MLP_In, MLP_Hidden, MLP_Out, Residual2: TTensorData;
S: TShape;
begin
TLayerNorm.Forward(Norm_Out, Input, Norm1, Norm2, 1);
Attn.Forward(Attn_Out, Norm_Out, True);
TensorOps.Add(Residual1, Input, Attn_Out);
TLayerNorm.Forward(MLP_In, Residual1, Norm1, Norm2, 1);
TensorOps.MatMul(MLP_Hidden, W_in, MLP_In);
TGELU.Forward(MLP_Hidden, MLP_Hidden);
TensorOps.MatMul(MLP_Out, W_out, MLP_Hidden);
TensorOps.Add(Output, Residual1, MLP_Out);
end;
procedure TTransformerBlock.Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input: TTensorData);
var
Norm_Out, Attn_Out, Residual1, MLP_In, MLP_Hidden, MLP_Out: TTensorData;
D_MLP_Hidden, D_MLP_In, D_Attn_Out, D_Input: TTensorData;
D_ResOut, D_AttnNorm, D_W_in, D_W_out: TTensorData;
S: TShape;
begin
TLayerNorm.Forward(Norm_Out, Input, Norm1, Norm2, 1);
Attn.Forward(Attn_Out, Norm_Out, True);
TensorOps.Add(Residual1, Input, Attn_Out);
TLayerNorm.Forward(MLP_In, Residual1, Norm1, Norm2, 1);
TensorOps.MatMul(MLP_Hidden, W_in, MLP_In);
TGELU.Forward(MLP_Hidden, MLP_Hidden);
TensorOps.MatMul(MLP_Out, W_out, MLP_Hidden);
Grad.Allocate(DLoss.Shape);
Grad.Copy(DLoss);
TensorOps.MatMul(D_MLP_Hidden, W_out, Grad);
TGELU.Backward(D_MLP_In, D_MLP_Hidden, MLP_Hidden);
TensorOps.MatMul(D_AttnNorm, W_in, D_MLP_In);
TensorOps.Add(D_ResOut, Grad, D_AttnNorm);
Attn.Backward(D_Attn_Out, D_ResOut, Norm_Out, True);
TensorOps.Add(D_Input, D_ResOut, D_Attn_Out);
Grad.Copy(D_Input);
end;
procedure TTransformerBlock.ForwardWithJacobian(var Output: TTensorData; var Jac: TJacobianBlock;
const Input: TTensorData);
var
S: TShape;
begin
Forward(Output, Input);
S.Init(Input.Shape.TotalSize, Output.Shape.TotalSize, 0, 0);
Jac.Allocate(Input.Shape.TotalSize, Output.Shape.TotalSize);
Jac.InputData.Copy(Input);
Jac.OutputData.Copy(Output);
for var i := 0 to minintvalue(Jac.InputSize, Jac.OutputSize) - 1 do
Jac.JacobianMatrix[i][i] := 1.0;
end;
end.
|