File size: 4,836 Bytes
32b4893 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | (* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ Training.pas – Adam optimizer, gradient clipping, learning rate scheduling
Line count: 200
Dependency: TensorCore
Manual Adam implementation: m_t = b1*m_{t-1} + (1-b1)*g_t
v_t = b2*v_{t-1} + (1-b2)*g_t^2
t_t = t - a * m_t / (sqrt(v_t) + eps)
}
unit Training;
interface
uses
SysUtils, Math, TensorCore;
type
TAdamState = record
M, V: TTensorData;
T: Integer;
procedure Initialize(const ParamShape: TShape);
end;
TAdamOptimizer = class
private
States: array of TAdamState;
Beta1, Beta2, Eps: Single;
T: Integer;
public
Alpha: Single;
constructor Create(LearningRate, Beta1Val, Beta2Val, Epsilon: Single);
procedure Step(var Param: TTensorData; const Grad: TTensorData);
procedure ClipGradients(var Grad: TTensorData; MaxNorm: Single);
end;
TLRScheduler = class
public
class procedure StepDecay(var LR: Single; Epoch, DecaySteps, DecayRate: Integer);
class procedure WarmupLinear(var LR: Single; Epoch, WarmupEpochs, MaxEpochs, BaseLR: Integer);
end;
TBatchHelper = class
public
class procedure CreateBatch(var Batch: TTensorData; const Data: TTensorData;
BatchStart, BatchSize: Integer);
end;
implementation
procedure TAdamState.Initialize(const ParamShape: TShape);
begin
TensorOps.Zeros(M, ParamShape);
TensorOps.Zeros(V, ParamShape);
T := 0;
end;
constructor TAdamOptimizer.Create(LearningRate, Beta1Val, Beta2Val, Epsilon: Single);
begin
inherited Create;
Alpha := LearningRate;
Beta1 := Beta1Val;
Beta2 := Beta2Val;
Eps := Epsilon;
T := 0;
end;
procedure TAdamOptimizer.Step(var Param: TTensorData; const Grad: TTensorData);
var
i, StateIdx: Integer;
State: TAdamState;
BiasCorrection1, BiasCorrection2, AlphaHat: Single;
begin
Inc(T);
StateIdx := -1;
for i := 0 to Length(States) - 1 do begin
if States[i].M.Shape.TotalSize = Param.Shape.TotalSize then begin
StateIdx := i;
Break;
end;
end;
if StateIdx = -1 then begin
StateIdx := Length(States);
SetLength(States, StateIdx + 1);
States[StateIdx].Initialize(Param.Shape);
end;
State := States[StateIdx];
Inc(State.T);
for i := 0 to Length(Grad.Data) - 1 do begin
State.M.Data[i] := Beta1 * State.M.Data[i] + (1 - Beta1) * Grad.Data[i];
State.V.Data[i] := Beta2 * State.V.Data[i] + (1 - Beta2) * Sqr(Grad.Data[i]);
end;
BiasCorrection1 := 1 - Power(Beta1, State.T);
BiasCorrection2 := 1 - Power(Beta2, State.T);
AlphaHat := Alpha * Sqrt(BiasCorrection2) / BiasCorrection1;
for i := 0 to Length(Param.Data) - 1 do
Param.Data[i] := Param.Data[i] - AlphaHat * State.M.Data[i] /
(Sqrt(State.V.Data[i]) + Eps);
States[StateIdx] := State;
end;
procedure TAdamOptimizer.ClipGradients(var Grad: TTensorData; MaxNorm: Single);
var
i: Integer;
GradNorm: Single;
begin
GradNorm := 0;
for i := 0 to Length(Grad.Data) - 1 do
GradNorm := GradNorm + Sqr(Grad.Data[i]);
GradNorm := Sqrt(GradNorm);
if GradNorm > MaxNorm then begin
for i := 0 to Length(Grad.Data) - 1 do
Grad.Data[i] := Grad.Data[i] * MaxNorm / GradNorm;
end;
end;
class procedure TLRScheduler.StepDecay(var LR: Single; Epoch, DecaySteps, DecayRate: Integer);
begin
if (Epoch > 0) and (Epoch mod DecaySteps = 0) then
LR := LR * DecayRate;
end;
class procedure TLRScheduler.WarmupLinear(var LR: Single; Epoch, WarmupEpochs, MaxEpochs, BaseLR: Integer);
var
TargetLR: Single;
begin
if Epoch < WarmupEpochs then
LR := BaseLR * Epoch / WarmupEpochs
else
LR := BaseLR * (MaxEpochs - Epoch) / (MaxEpochs - WarmupEpochs);
end;
class procedure TBatchHelper.CreateBatch(var Batch: TTensorData; const Data: TTensorData;
BatchStart, BatchSize: Integer);
var
S: TShape;
i: Integer;
begin
S.Init(BatchSize, Data.Shape.Dims[1], 0, 0);
Batch.Allocate(S);
for i := 0 to BatchSize - 1 do begin
if BatchStart + i < Data.Shape.Dims[0] then
Move(Data.Data[(BatchStart + i) * Data.Shape.Dims[1]],
Batch.Data[i * Data.Shape.Dims[1]],
Data.Shape.Dims[1] * SizeOf(Single));
end;
end;
end.
|