mapdet3d/op/layer/weight_init.py

"""Model weight initialization."""

from typing import Literal

import numpy as np
from torch import nn

NonlinearityType = Literal[
    "linear",
    "conv1d",
    "conv2d",
    "conv3d",
    "conv_transpose1d",
    "conv_transpose2d",
    "conv_transpose3d",
    "sigmoid",
    "tanh",
    "relu",
    "leaky_relu",
    "selu",
]
FanMode = Literal["fan_in", "fan_out"]


def constant_init(module: nn.Module, val: float, bias: float = 0.0) -> None:
    """Initialize module with constant value."""
    if hasattr(module, "weight") and isinstance(module.weight, nn.Parameter):
        nn.init.constant_(module.weight, val)
    if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
        nn.init.constant_(module.bias, bias)


def xavier_init(
    module: nn.Module,
    gain: float = 1.0,
    bias: float = 0.0,
    distribution: str = "normal",
) -> None:
    """Initialize module with Xavier initialization."""
    assert distribution in {"uniform", "normal"}
    if hasattr(module, "weight") and isinstance(module.weight, nn.Parameter):
        if distribution == "uniform":
            nn.init.xavier_uniform_(module.weight, gain=gain)
        else:
            nn.init.xavier_normal_(module.weight, gain=gain)
    if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
        nn.init.constant_(module.bias, bias)


def kaiming_init(
    module: nn.Module,
    negative_slope: float = 0.0,
    mode: FanMode = "fan_out",
    nonlinearity: NonlinearityType = "relu",
    bias: float = 0.0,
    distribution: str = "normal",
) -> None:
    """Initialize module with Kaiming initialization.

    Args:
        module (nn.Module): Module to initialize.
        negative_slope (float, optional): The negative slope of the rectifier
            used after this layer (only used with ``'leaky_relu'``). Defaults
            to 0.0.
        mode (FanMode, optional): Either `"fan_in"` (default) or `"fan_out"``.
            Choosing `"fan_in"` preserves the magnitude of the variance of
            the weights in the forward pass. Choosing `"fan_out"` preserves
            magnitudes in the backwards pass. Defaults to "fan_out".
        nonlinearity (NonlinearityType, optional): The non-linear function
            (`nn.functional` name). Defaults to "relu".
        bias (float, optional): The bias to use. Defaults to 0.0.
        distribution (str, optional): Either ``'uniform'`` or ``'normal'``.
            Defaults to "normal".
    """
    assert distribution in {"uniform", "normal"}
    if hasattr(module, "weight") and isinstance(module.weight, nn.Parameter):
        if distribution == "uniform":
            nn.init.kaiming_uniform_(
                module.weight,
                a=negative_slope,
                mode=mode,
                nonlinearity=nonlinearity,
            )
        else:
            nn.init.kaiming_normal_(
                module.weight,
                a=negative_slope,
                mode=mode,
                nonlinearity=nonlinearity,
            )
    if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
        nn.init.constant_(module.bias, bias)


def normal_init(
    module: nn.Module, mean: float = 0.0, std: float = 1.0, bias: float = 0
) -> None:
    """Initialize module with normal distribution."""
    if hasattr(module, "weight") and isinstance(module.weight, nn.Parameter):
        nn.init.normal_(module.weight, mean, std)
    if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
        nn.init.constant_(module.bias, bias)


def bias_init_with_prob(prior_prob: float) -> float:
    """Initialize conv/fc bias value according to a given probability value."""
    return float(-np.log((1 - prior_prob) / prior_prob))


def uniform_init(
    module: nn.Module,
    lower: float = 0.0,
    upper: float = 1.0,
    bias: float = 0.0,
) -> None:
    """Initialize module with uniform distribution."""
    if hasattr(module, "weight") and isinstance(module.weight, nn.Parameter):
        nn.init.uniform_(module.weight, lower, upper)
    if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter):
        nn.init.constant_(module.bias, bias)