utils/regularizations.py

import torch
def l2_regularization(input_data: torch.Tensor,
                      weight:float=0.01):

  """
    Apply L2 regularization to the input tensor.
    Args:
        input_data (torch.Tensor): Input tensor being optimized.
        weight (float): Regularization strength.
    Returns:
        torch.Tensor: L2 regularization term.
  """
  return weight * torch.sum(input_data**2)


def total_variance_loss(input_data: torch.Tensor,
                      input_data_type:str=None,
                      weight:float=0.01):
  """
    Apply Total Variation loss to encourage smoothness.
    Args:
        input_data (torch.Tensor): Input tensor being optimized.
        weight (float): Regularization strength.
    Returns:
        torch.Tensor: Total Variation loss.
  """

  if input_data.dim()==4:
    # calculating successive differences
    x_diff = input_data[:,:,1:,:] - input_data[:,:,:-1,:]
    y_diff = input_data[:,:,:,1:] - input_data[:,:,:,:-1]
    return weight*(torch.sum(torch.abs(x_diff)) + torch.sum(torch.abs(y_diff)))

  elif input_data.dim()==3:
    x_diff = input_data[:,1:,:] - input_data[:,:-1,:]
    y_diff = input_data[:,:,1:] - input_data[:,:,:-1]
    return weight*(torch.sum(torch.abs(x_diff)) + torch.sum(torch.abs(y_diff)))

  else:
    raise ValueError(f"Unsuppported input dimensions:{input_data.dim()}. Expected 3D or 4D tensor")

def sparsity_constraint(input_data, weight:float=0.01):

  """
  Apply sparsity constraint to promote sparse activations.

  This function penalizes the magnitude of the input tensor, similar to `l1_regularization`.
  However, it is more specific to scenarios where sparsity is required in activations
  or input values during feature optimization.

  Note: While `l1_regularization` and `sparsity_constraint` are mathematically identical,
  they are kept separate to allow users to modify their implementation for different use cases.

  Args:
      input_data (torch.Tensor): Input tensor being optimized.
      weight (float): Regularization strength.

  Returns:
      torch.Tensor: Sparsity constraint term.
  """
  return weight*torch.sum(torch.abs(input_data))

#def clip_input(input_data, min_value=0, max_value=1):
#  """
#  Clip input values to a specified range.
#  Args:
#      input_data (torch.Tensor): Input tensor being optimized.
#      min_value (float): Minimum value for clipping.
#      max_value (float): Maximum value for clipping.
#  Returns:
#      torch.Tensor: Clipped input tensor.
#  """
#  return torch.clamp(input=input_data, min=min_value, max=max_value)

# adding more regularizations

def l1_regularization(input_data: torch.Tensor, weight:float = 0.01)->torch.Tensor:
  """
   Apply L1 regularization to the input tensor.

  This function penalizes the magnitude of the input tensor, encouraging sparsity
  in the input values. While it is mathematically similar to `sparsity_constraint`,
  it is more commonly associated with general regularization of inputs or model weights.

  Args:
      input_data (torch.Tensor): Input tensor being optimized.
      weight (float): Regularization strength.

  Returns:
      torch.Tensor: L1 regularization term.
  """
  return weight * torch.sum(torch.abs(input_data))

def linf_regularization(input_data:torch.Tensor, weight:float=0.01)->torch.Tensor:

  """
  Apply L∞ regularization to the input tensor.

  Args:
    input_data (torch.Tensor): Input tensor being optimized.
    weight (float): Regularization strength.

  Returns:
    torch.Tensor: L∞ regularization term.

  Mathematical Concept:
    L∞ regularization maximizes the penalty for the largest value in the tensor:
    L∞(x) = max(|x|).
  """

  return weight * torch.max(torch.abs(input_data))

def feature_map_sparsity(activation: torch.Tensor, weight:float=0.01)->torch.Tensor:

  """
  Encourage sparsity in the feature map activation.
  """
  return torch.sum(torch.abs(activation))

def entropy_loss(input_data: torch.Tensor, weight:float=0.01, epsilon:float=1e-12)->torch.Tensor:

  """
  Apply entropy loss to the input tensor.

  Args:
      input_data (torch.Tensor): Input tensor being optimized.
      weight (float): Regularization strength.
      epsilon (float): a relatively vanishingly small number to prevent logarithm from blowing up.

  Returns:
      torch.Tensor: Entropy loss term.

  Steps:
      1. Flatten the input tensor to a 1D vector using `flatten()`.
      2. Apply softmax to compute probabilities.
      3. Compute entropy as the negati ve sum of probabilities multiplied by their log values.
      4. Scale the entropy by the weight and return.
  """

  probs=torch.softmax(input_data.flatten(), dim=0)
  entropy = -torch.sum(probs*torch.log(probs+epsilon))

  return weight * entropy