Spaces:
Sleeping
Sleeping
File size: 4,747 Bytes
adcc0ff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
import torch
def l2_regularization(input_data: torch.Tensor,
weight:float=0.01):
"""
Apply L2 regularization to the input tensor.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
Returns:
torch.Tensor: L2 regularization term.
"""
return weight * torch.sum(input_data**2)
def total_variance_loss(input_data: torch.Tensor,
input_data_type:str=None,
weight:float=0.01):
"""
Apply Total Variation loss to encourage smoothness.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
Returns:
torch.Tensor: Total Variation loss.
"""
if input_data.dim()==4:
# calculating successive differences
x_diff = input_data[:,:,1:,:] - input_data[:,:,:-1,:]
y_diff = input_data[:,:,:,1:] - input_data[:,:,:,:-1]
return weight*(torch.sum(torch.abs(x_diff)) + torch.sum(torch.abs(y_diff)))
elif input_data.dim()==3:
x_diff = input_data[:,1:,:] - input_data[:,:-1,:]
y_diff = input_data[:,:,1:] - input_data[:,:,:-1]
return weight*(torch.sum(torch.abs(x_diff)) + torch.sum(torch.abs(y_diff)))
else:
raise ValueError(f"Unsuppported input dimensions:{input_data.dim()}. Expected 3D or 4D tensor")
def sparsity_constraint(input_data, weight:float=0.01):
"""
Apply sparsity constraint to promote sparse activations.
This function penalizes the magnitude of the input tensor, similar to `l1_regularization`.
However, it is more specific to scenarios where sparsity is required in activations
or input values during feature optimization.
Note: While `l1_regularization` and `sparsity_constraint` are mathematically identical,
they are kept separate to allow users to modify their implementation for different use cases.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
Returns:
torch.Tensor: Sparsity constraint term.
"""
return weight*torch.sum(torch.abs(input_data))
#def clip_input(input_data, min_value=0, max_value=1):
# """
# Clip input values to a specified range.
# Args:
# input_data (torch.Tensor): Input tensor being optimized.
# min_value (float): Minimum value for clipping.
# max_value (float): Maximum value for clipping.
# Returns:
# torch.Tensor: Clipped input tensor.
# """
# return torch.clamp(input=input_data, min=min_value, max=max_value)
# adding more regularizations
def l1_regularization(input_data: torch.Tensor, weight:float = 0.01)->torch.Tensor:
"""
Apply L1 regularization to the input tensor.
This function penalizes the magnitude of the input tensor, encouraging sparsity
in the input values. While it is mathematically similar to `sparsity_constraint`,
it is more commonly associated with general regularization of inputs or model weights.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
Returns:
torch.Tensor: L1 regularization term.
"""
return weight * torch.sum(torch.abs(input_data))
def linf_regularization(input_data:torch.Tensor, weight:float=0.01)->torch.Tensor:
"""
Apply L∞ regularization to the input tensor.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
Returns:
torch.Tensor: L∞ regularization term.
Mathematical Concept:
L∞ regularization maximizes the penalty for the largest value in the tensor:
L∞(x) = max(|x|).
"""
return weight * torch.max(torch.abs(input_data))
def feature_map_sparsity(activation: torch.Tensor, weight:float=0.01)->torch.Tensor:
"""
Encourage sparsity in the feature map activation.
"""
return torch.sum(torch.abs(activation))
def entropy_loss(input_data: torch.Tensor, weight:float=0.01, epsilon:float=1e-12)->torch.Tensor:
"""
Apply entropy loss to the input tensor.
Args:
input_data (torch.Tensor): Input tensor being optimized.
weight (float): Regularization strength.
epsilon (float): a relatively vanishingly small number to prevent logarithm from blowing up.
Returns:
torch.Tensor: Entropy loss term.
Steps:
1. Flatten the input tensor to a 1D vector using `flatten()`.
2. Apply softmax to compute probabilities.
3. Compute entropy as the negati ve sum of probabilities multiplied by their log values.
4. Scale the entropy by the weight and return.
"""
probs=torch.softmax(input_data.flatten(), dim=0)
entropy = -torch.sum(probs*torch.log(probs+epsilon))
return weight * entropy
|