Hugging Face's logo

Snapkitty
/
sparse-router-formal

Rust
snapkitty
october-2026-drop
pascal
lean4
c
kani
work-stealing
scheduler
formal-verification
Model card Files
xet
 Community
sparse-router-formal / src /pascal /Activations.pas
SNAPKITTYWEST's picture
SNAPKITTYWEST
October 2026 main drop: mirror from GitHub
32b4893 verified
Raw History Blame Contribute Delete
7.58 kB
(* ========================================================================
* SOVEREIGN LEVIATHAN NODE LICENSE
* License-ID: SL-AGPL3-001 | Covenant-Version: 1.0
* Copyright (C) 2026 SnapKittyWest. Ahmad Ali Parr, Bel Esprit D'Accord Irrevocable Trust.
* ========================================================================
*
* This file is a covered work under the GNU Affero General Public License,
* version 3, together with the Sovereign Leviathan additional terms.
*
* Hark, though this node be but a spark,
* Its covenant endureth through the dark.
*
* Ignorantia juris non excusat.
* ======================================================================== *)
{ Activations.pas – Activation functions with analytic backward passes
Line count: 150
Dependency: TensorCore
}
unit Activations;
interface
uses
SysUtils, Math, TensorCore;
type
TSoftmax = class
public
class procedure Forward(var Output: TTensorData; const Input: TTensorData; Axis: Integer = 1);
class procedure Backward(var Grad: TTensorData; const DLoss: TTensorData;
const ForwardOutput: TTensorData; Axis: Integer = 1);
end;
TGELU = class
public
class procedure Forward(var Output: TTensorData; const Input: TTensorData);
class procedure Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input, ForwardOutput: TTensorData);
end;
TLayerNorm = class
public
class procedure Forward(var Output: TTensorData; const Input: TTensorData;
var Mean, Var: TTensorData; Axis: Integer; Eps: Single = 1e-6);
class procedure Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input, Mean, Var: TTensorData; Axis: Integer; Eps: Single = 1e-6);
end;
TReLU = class
public
class procedure Forward(var Output: TTensorData; const Input: TTensorData);
class procedure Backward(var Grad: TTensorData; const DLoss: TTensorData; const Input: TTensorData);
end;
implementation
class procedure TSoftmax.Forward(var Output: TTensorData; const Input: TTensorData; Axis: Integer = 1);
var
i, j, AxisSize, OutSize: Integer;
MaxVal, Sum, Exp: Single;
begin
Output.Allocate(Input.Shape);
OutSize := Input.Shape.TotalSize div Input.Shape.Dims[Axis];
AxisSize := Input.Shape.Dims[Axis];
for i := 0 to OutSize - 1 do begin
MaxVal := -1e10;
for j := 0 to AxisSize - 1 do
if Input.Data[i * AxisSize + j] > MaxVal then
MaxVal := Input.Data[i * AxisSize + j];
Sum := 0;
for j := 0 to AxisSize - 1 do begin
Exp := System.Exp(Input.Data[i * AxisSize + j] - MaxVal);
Output.Data[i * AxisSize + j] := Exp;
Sum := Sum + Exp;
end;
for j := 0 to AxisSize - 1 do
Output.Data[i * AxisSize + j] := Output.Data[i * AxisSize + j] / Sum;
end;
end;
class procedure TSoftmax.Backward(var Grad: TTensorData; const DLoss: TTensorData;
const ForwardOutput: TTensorData; Axis: Integer = 1);
var
i, j, k, AxisSize, OutSize: Integer;
JacRow, Sum: Single;
begin
Grad.Allocate(DLoss.Shape);
Grad.Fill(0);
OutSize := ForwardOutput.Shape.TotalSize div ForwardOutput.Shape.Dims[Axis];
AxisSize := ForwardOutput.Shape.Dims[Axis];
for i := 0 to OutSize - 1 do begin
for j := 0 to AxisSize - 1 do begin
Sum := 0;
for k := 0 to AxisSize - 1 do begin
if j = k then
JacRow := ForwardOutput.Data[i * AxisSize + j] *
(1 - ForwardOutput.Data[i * AxisSize + k])
else
JacRow := -ForwardOutput.Data[i * AxisSize + j] *
ForwardOutput.Data[i * AxisSize + k];
Sum := Sum + JacRow * DLoss.Data[i * AxisSize + k];
end;
Grad.Data[i * AxisSize + j] := Sum;
end;
end;
end;
class procedure TGELU.Forward(var Output: TTensorData; const Input: TTensorData);
var
i: Integer;
x, x3, TanhArg, Cdf: Single;
const
SQRT2PI = 0.7978845608;
COEFF = 0.044715;
begin
Output.Allocate(Input.Shape);
for i := 0 to Length(Input.Data) - 1 do begin
x := Input.Data[i];
x3 := x * x * x;
TanhArg := SQRT2PI * (x + COEFF * x3);
Cdf := 0.5 * (1 + System.Tanh(TanhArg));
Output.Data[i] := x * Cdf;
end;
end;
class procedure TGELU.Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input, ForwardOutput: TTensorData);
var
i: Integer;
x, x2, x3, TanhArg, Cdf, Sech2, Pdf, Deriv: Single;
const
SQRT2PI = 0.7978845608;
COEFF = 0.044715;
begin
Grad.Allocate(DLoss.Shape);
for i := 0 to Length(Input.Data) - 1 do begin
x := Input.Data[i];
x2 := x * x;
x3 := x2 * x;
TanhArg := SQRT2PI * (x + COEFF * x3);
Cdf := 0.5 * (1 + System.Tanh(TanhArg));
Sech2 := 1 - Sqr(System.Tanh(TanhArg));
Pdf := SQRT2PI * Sech2 * (1 + 3 * COEFF * x2) * 0.5;
Deriv := Cdf + x * Pdf;
Grad.Data[i] := DLoss.Data[i] * Deriv;
end;
end;
class procedure TLayerNorm.Forward(var Output: TTensorData; const Input: TTensorData;
var Mean, Var: TTensorData; Axis: Integer; Eps: Single = 1e-6);
var
i, AxisSize, OutSize, j: Integer;
Sum, M, V, StdDev: Single;
MeanShape, VarShape: TShape;
begin
if Axis >= Input.Shape.Rank then
raise Exception.Create('LayerNorm axis out of bounds');
AxisSize := Input.Shape.Dims[Axis];
OutSize := Input.Shape.TotalSize div AxisSize;
MeanShape := Input.Shape;
MeanShape.Dims[Axis] := 1;
Mean.Allocate(MeanShape);
Var.Allocate(MeanShape);
Output.Allocate(Input.Shape);
for i := 0 to OutSize - 1 do begin
Sum := 0;
for j := 0 to AxisSize - 1 do
Sum := Sum + Input.Data[i * AxisSize + j];
M := Sum / AxisSize;
Mean.Data[i] := M;
Sum := 0;
for j := 0 to AxisSize - 1 do
Sum := Sum + Sqr(Input.Data[i * AxisSize + j] - M);
V := Sum / AxisSize;
Var.Data[i] := V;
StdDev := Sqrt(V + Eps);
for j := 0 to AxisSize - 1 do
Output.Data[i * AxisSize + j] := (Input.Data[i * AxisSize + j] - M) / StdDev;
end;
end;
class procedure TLayerNorm.Backward(var Grad: TTensorData; const DLoss: TTensorData;
const Input, Mean, Var: TTensorData; Axis: Integer; Eps: Single = 1e-6);
var
i, j, AxisSize, OutSize: Integer;
StdDev, SumDLoss, SumDLossX, C: Single;
begin
AxisSize := Input.Shape.Dims[Axis];
OutSize := Input.Shape.TotalSize div AxisSize;
Grad.Allocate(Input.Shape);
for i := 0 to OutSize - 1 do begin
StdDev := Sqrt(Var.Data[i] + Eps);
SumDLoss := 0;
SumDLossX := 0;
for j := 0 to AxisSize - 1 do begin
SumDLoss := SumDLoss + DLoss.Data[i * AxisSize + j];
SumDLossX := SumDLossX + DLoss.Data[i * AxisSize + j] *
(Input.Data[i * AxisSize + j] - Mean.Data[i]);
end;
for j := 0 to AxisSize - 1 do begin
C := (Input.Data[i * AxisSize + j] - Mean.Data[i]) / (Var.Data[i] + Eps);
Grad.Data[i * AxisSize + j] :=
(DLoss.Data[i * AxisSize + j] / StdDev) -
(SumDLoss / (AxisSize * StdDev)) -
(SumDLossX * C / (AxisSize * (Var.Data[i] + Eps)));
end;
end;
end;
class procedure TReLU.Forward(var Output: TTensorData; const Input: TTensorData);
var i: Integer;
begin
Output.Allocate(Input.Shape);
for i := 0 to Length(Input.Data) - 1 do
if Input.Data[i] > 0 then
Output.Data[i] := Input.Data[i]
else
Output.Data[i] := 0;
end;
class procedure TReLU.Backward(var Grad: TTensorData; const DLoss: TTensorData; const Input: TTensorData);
var i: Integer;
begin
Grad.Allocate(DLoss.Shape);
for i := 0 to Length(Input.Data) - 1 do
if Input.Data[i] > 0 then
Grad.Data[i] := DLoss.Data[i]
else
Grad.Data[i] := 0;
end;
end.