Add files using upload-large-folder tool

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/__init__.py

@@ -0,0 +1,5 @@
+from keras.src.layers.activations.elu import ELU
+from keras.src.layers.activations.leaky_relu import LeakyReLU
+from keras.src.layers.activations.prelu import PReLU
+from keras.src.layers.activations.relu import ReLU
+from keras.src.layers.activations.softmax import Softmax

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/activation.py

@@ -0,0 +1,40 @@
+from keras.src import activations
+from keras.src.api_export import keras_export
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.Activation")
+class Activation(Layer):
+    """Applies an activation function to an output.
+
+    Args:
+        activation: Activation function. It could be a callable, or the name of
+            an activation from the `keras.activations` namespace.
+        **kwargs: Base layer keyword arguments, such as `name` and `dtype`.
+
+    Example:
+
+    >>> layer = keras.layers.Activation('relu')
+    >>> layer(np.array([-3.0, -1.0, 0.0, 2.0]))
+    [0.0, 0.0, 0.0, 2.0]
+    >>> layer = keras.layers.Activation(keras.activations.relu)
+    >>> layer(np.array([-3.0, -1.0, 0.0, 2.0]))
+    [0.0, 0.0, 0.0, 2.0]
+    """
+
+    def __init__(self, activation, **kwargs):
+        super().__init__(**kwargs)
+        self.supports_masking = True
+        self.activation = activations.get(activation)
+        self.built = True
+
+    def call(self, inputs):
+        return self.activation(inputs)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def get_config(self):
+        config = {"activation": activations.serialize(self.activation)}
+        base_config = super().get_config()
+        return {**base_config, **config}

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/elu.py

@@ -0,0 +1,32 @@
+from keras.src import activations
+from keras.src.api_export import keras_export
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.ELU")
+class ELU(Layer):
+    """Applies an Exponential Linear Unit function to an output.
+
+    Formula:
+
+    ```
+    f(x) = alpha * (exp(x) - 1.) for x < 0
+    f(x) = x for x >= 0
+    ```
+
+    Args:
+        alpha: float, slope of negative section. Defaults to `1.0`.
+        **kwargs: Base layer keyword arguments, such as `name` and `dtype`.
+    """
+
+    def __init__(self, alpha=1.0, **kwargs):
+        super().__init__(**kwargs)
+        self.alpha = alpha
+        self.supports_masking = True
+        self.built = True
+
+    def call(self, inputs):
+        return activations.elu(inputs, alpha=self.alpha)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/leaky_relu.py

@@ -0,0 +1,67 @@
+import warnings
+
+from keras.src import activations
+from keras.src.api_export import keras_export
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.LeakyReLU")
+class LeakyReLU(Layer):
+    """Leaky version of a Rectified Linear Unit activation layer.
+
+    This layer allows a small gradient when the unit is not active.
+
+    Formula:
+
+    ``` python
+    f(x) = alpha * x if x < 0
+    f(x) = x if x >= 0
+    ```
+
+    Example:
+
+    ``` python
+    leaky_relu_layer = LeakyReLU(negative_slope=0.5)
+    input = np.array([-10, -5, 0.0, 5, 10])
+    result = leaky_relu_layer(input)
+    # result = [-5. , -2.5,  0. ,  5. , 10.]
+    ```
+
+    Args:
+        negative_slope: Float >= 0.0. Negative slope coefficient.
+          Defaults to `0.3`.
+        **kwargs: Base layer keyword arguments, such as
+            `name` and `dtype`.
+
+    """
+
+    def __init__(self, negative_slope=0.3, **kwargs):
+        if "alpha" in kwargs:
+            negative_slope = kwargs.pop("alpha")
+            warnings.warn(
+                "Argument `alpha` is deprecated. "
+                "Use `negative_slope` instead."
+            )
+        super().__init__(**kwargs)
+        if negative_slope is None or negative_slope < 0:
+            raise ValueError(
+                "The negative_slope value of a Leaky ReLU layer "
+                "cannot be None or negative value. Expected a float."
+                f" Received: negative_slope={negative_slope}"
+            )
+        self.negative_slope = negative_slope
+        self.supports_masking = True
+        self.built = True
+
+    def call(self, inputs):
+        return activations.leaky_relu(
+            inputs, negative_slope=self.negative_slope
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"negative_slope": self.negative_slope})
+        return config
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/prelu.py

@@ -0,0 +1,99 @@
+from keras.src import activations
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import regularizers
+from keras.src.api_export import keras_export
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.PReLU")
+class PReLU(Layer):
+    """Parametric Rectified Linear Unit activation layer.
+
+    Formula:
+    ``` python
+    f(x) = alpha * x for x < 0
+    f(x) = x for x >= 0
+    ```
+    where `alpha` is a learned array with the same shape as x.
+
+    Args:
+        alpha_initializer: Initializer function for the weights.
+        alpha_regularizer: Regularizer for the weights.
+        alpha_constraint: Constraint for the weights.
+        shared_axes: The axes along which to share learnable parameters for the
+            activation function. For example, if the incoming feature maps are
+            from a 2D convolution with output shape
+            `(batch, height, width, channels)`, and you wish to share parameters
+            across space so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
+        **kwargs: Base layer keyword arguments, such as `name` and `dtype`.
+    """
+
+    def __init__(
+        self,
+        alpha_initializer="Zeros",
+        alpha_regularizer=None,
+        alpha_constraint=None,
+        shared_axes=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.supports_masking = True
+        self.alpha_initializer = initializers.get(alpha_initializer)
+        self.alpha_regularizer = regularizers.get(alpha_regularizer)
+        self.alpha_constraint = constraints.get(alpha_constraint)
+        if shared_axes is None:
+            self.shared_axes = None
+        elif not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
+
+    def build(self, input_shape):
+        param_shape = list(input_shape[1:])
+        if self.shared_axes is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+        self.alpha = self.add_weight(
+            shape=param_shape,
+            name="alpha",
+            initializer=self.alpha_initializer,
+            regularizer=self.alpha_regularizer,
+            constraint=self.alpha_constraint,
+        )
+        # Set input spec
+        axes = {}
+        if self.shared_axes:
+            for i in range(1, len(input_shape)):
+                if i not in self.shared_axes:
+                    axes[i] = input_shape[i]
+        self.input_spec = InputSpec(ndim=len(input_shape), axes=axes)
+        self.built = True
+
+    def call(self, inputs):
+        pos = activations.relu(inputs)
+        neg = -self.alpha * activations.relu(-inputs)
+        return pos + neg
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "alpha_initializer": initializers.serialize(
+                    self.alpha_initializer
+                ),
+                "alpha_regularizer": regularizers.serialize(
+                    self.alpha_regularizer
+                ),
+                "alpha_constraint": constraints.serialize(
+                    self.alpha_constraint
+                ),
+                "shared_axes": self.shared_axes,
+            }
+        )
+        return config
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/relu.py

@@ -0,0 +1,86 @@
+from keras.src import activations
+from keras.src.api_export import keras_export
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.ReLU")
+class ReLU(Layer):
+    """Rectified Linear Unit activation function layer.
+
+    Formula:
+    ``` python
+    f(x) = max(x,0)
+    f(x) = max_value if x >= max_value
+    f(x) = x if threshold <= x < max_value
+    f(x) = negative_slope * (x - threshold) otherwise
+    ```
+
+    Example:
+    ``` python
+    relu_layer = keras.layers.ReLU(
+        max_value=10,
+        negative_slope=0.5,
+        threshold=0,
+    )
+    input = np.array([-10, -5, 0.0, 5, 10])
+    result = relu_layer(input)
+    # result = [-5. , -2.5,  0. ,  5. , 10.]
+    ```
+
+    Args:
+        max_value: Float >= 0. Maximum activation value. None means unlimited.
+            Defaults to `None`.
+        negative_slope: Float >= 0. Negative slope coefficient.
+            Defaults to `0.0`.
+        threshold: Float >= 0. Threshold value for thresholded activation.
+            Defaults to `0.0`.
+        **kwargs: Base layer keyword arguments, such as `name` and `dtype`.
+    """
+
+    def __init__(
+        self, max_value=None, negative_slope=0.0, threshold=0.0, **kwargs
+    ):
+        super().__init__(**kwargs)
+        if max_value is not None and max_value < 0.0:
+            raise ValueError(
+                "max_value of a ReLU layer cannot be a negative "
+                f"value. Received: max_value={max_value}"
+            )
+        if negative_slope is None or negative_slope < 0.0:
+            raise ValueError(
+                "negative_slope of a ReLU layer cannot be a negative "
+                f"value. Received: negative_slope={negative_slope}"
+            )
+        if threshold is None or threshold < 0.0:
+            raise ValueError(
+                "threshold of a ReLU layer cannot be a negative "
+                f"value. Received: threshold={threshold}"
+            )
+
+        self.max_value = max_value
+        self.negative_slope = negative_slope
+        self.threshold = threshold
+        self.supports_masking = True
+        self.built = True
+
+    def call(self, inputs):
+        return activations.relu(
+            inputs,
+            negative_slope=self.negative_slope,
+            max_value=self.max_value,
+            threshold=self.threshold,
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "max_value": self.max_value,
+                "negative_slope": self.negative_slope,
+                "threshold": self.threshold,
+            }
+        )
+        return config
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/activations/softmax.py

@@ -0,0 +1,76 @@
+from keras.src import activations
+from keras.src import backend
+from keras.src.api_export import keras_export
+from keras.src.layers.layer import Layer
+
+
+def _large_negative_number(dtype):
+    """Return a Large negative number based on dtype."""
+    if backend.standardize_dtype(dtype) == "float16":
+        return -3e4
+    return -1e9
+
+
+@keras_export("keras.layers.Softmax")
+class Softmax(Layer):
+    """Softmax activation layer.
+
+    Formula:
+    ``` python
+    exp_x = exp(x - max(x))
+    f(x) = exp_x / sum(exp_x)
+    ```
+
+    Example:
+    >>> softmax_layer = keras.layers.Softmax()
+    >>> input = np.array([1.0, 2.0, 1.0])
+    >>> result = softmax_layer(input)
+    >>> result
+    [0.21194157, 0.5761169, 0.21194157]
+
+
+    Args:
+        axis: Integer, or list of Integers, axis along which the softmax
+            normalization is applied.
+        **kwargs: Base layer keyword arguments, such as `name` and `dtype`.
+
+    Call arguments:
+        inputs: The inputs (logits) to the softmax layer.
+        mask: A boolean mask of the same shape as `inputs`. The mask
+            specifies 1 to keep and 0 to mask. Defaults to `None`.
+
+    Returns:
+        Softmaxed output with the same shape as `inputs`.
+    """
+
+    def __init__(self, axis=-1, **kwargs):
+        super().__init__(**kwargs)
+        self.axis = axis
+        self.supports_masking = True
+        self.built = True
+
+    def call(self, inputs, mask=None):
+        if mask is not None:
+            adder = (
+                1.0 - backend.cast(mask, inputs.dtype)
+            ) * _large_negative_number(inputs.dtype)
+            inputs += adder
+        if isinstance(self.axis, (tuple, list)):
+            if len(self.axis) > 1:
+                return backend.numpy.exp(
+                    inputs
+                    - backend.math.logsumexp(
+                        inputs, axis=self.axis, keepdims=True
+                    )
+                )
+            else:
+                return activations.softmax(inputs, axis=self.axis[0])
+        return activations.softmax(inputs, axis=self.axis)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"axis": self.axis})
+        return config
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/attention/additive_attention.py

@@ -0,0 +1,103 @@
+from keras.src import ops
+from keras.src.api_export import keras_export
+from keras.src.layers.attention.attention import Attention
+
+
+@keras_export("keras.layers.AdditiveAttention")
+class AdditiveAttention(Attention):
+    """Additive attention layer, a.k.a. Bahdanau-style attention.
+
+    Inputs are a list with 2 or 3 elements:
+    1. A `query` tensor of shape `(batch_size, Tq, dim)`.
+    2. A `value` tensor of shape `(batch_size, Tv, dim)`.
+    3. A optional `key` tensor of shape `(batch_size, Tv, dim)`. If none
+        supplied, `value` will be used as `key`.
+
+    The calculation follows the steps:
+    1. Calculate attention scores using `query` and `key` with shape
+        `(batch_size, Tq, Tv)` as a non-linear sum
+        `scores = reduce_sum(tanh(query + key), axis=-1)`.
+    2. Use scores to calculate a softmax distribution with shape
+        `(batch_size, Tq, Tv)`.
+    3. Use the softmax distribution to create a linear combination of `value`
+        with shape `(batch_size, Tq, dim)`.
+
+    Args:
+        use_scale: If `True`, will create a scalar variable to scale the
+            attention scores.
+        dropout: Float between 0 and 1. Fraction of the units to drop for the
+            attention scores. Defaults to `0.0`.
+
+    Call arguments:
+        inputs: List of the following tensors:
+            - `query`: Query tensor of shape `(batch_size, Tq, dim)`.
+            - `value`: Value tensor of shape `(batch_size, Tv, dim)`.
+            - `key`: Optional key tensor of shape `(batch_size, Tv, dim)`. If
+                not given, will use `value` for both `key` and `value`, which is
+                the most common case.
+        mask: List of the following tensors:
+            - `query_mask`: A boolean mask tensor of shape `(batch_size, Tq)`.
+                If given, the output will be zero at the positions where
+                `mask==False`.
+            - `value_mask`: A boolean mask tensor of shape `(batch_size, Tv)`.
+                If given, will apply the mask such that values at positions
+                 where `mask==False` do not contribute to the result.
+        return_attention_scores: bool, it `True`, returns the attention scores
+            (after masking and softmax) as an additional output argument.
+        training: Python boolean indicating whether the layer should behave in
+            training mode (adding dropout) or in inference mode (no dropout).
+        use_causal_mask: Boolean. Set to `True` for decoder self-attention. Adds
+            a mask such that position `i` cannot attend to positions `j > i`.
+            This prevents the flow of information from the future towards the
+            past. Defaults to `False`.
+
+    Output:
+        Attention outputs of shape `(batch_size, Tq, dim)`.
+        (Optional) Attention scores after masking and softmax with shape
+            `(batch_size, Tq, Tv)`.
+    """
+
+    def __init__(
+        self,
+        use_scale=True,
+        dropout=0.0,
+        **kwargs,
+    ):
+        super().__init__(use_scale=use_scale, dropout=dropout, **kwargs)
+
+    def build(self, input_shape):
+        self._validate_inputs(input_shape)
+        dim = input_shape[0][-1]
+        self.scale = None
+        if self.use_scale:
+            self.scale = self.add_weight(
+                name="scale",
+                shape=[dim],
+                initializer="glorot_uniform",
+                dtype=self.dtype,
+                trainable=True,
+            )
+        self.built = True
+
+    def _calculate_scores(self, query, key):
+        """Calculates attention scores as a nonlinear sum of query and key.
+
+        Args:
+            query: Query tensor of shape `(batch_size, Tq, dim)`.
+            key: Key tensor of shape `(batch_size, Tv, dim)`.
+
+        Returns:
+            Tensor of shape `(batch_size, Tq, Tv)`.
+        """
+        # Reshape tensors to enable broadcasting.
+        # Reshape into [batch_size, Tq, 1, dim].
+        q_reshaped = ops.expand_dims(query, axis=-2)
+        # Reshape into [batch_size, 1, Tv, dim].
+        k_reshaped = ops.expand_dims(key, axis=-3)
+        scale = self.scale if self.use_scale else 1.0
+        return ops.sum(scale * ops.tanh(q_reshaped + k_reshaped), axis=-1)
+
+    def get_config(self):
+        base_config = super().get_config()
+        del base_config["score_mode"]
+        return base_config

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/attention/attention.py

@@ -0,0 +1,330 @@
+from keras.src import backend
+from keras.src import ops
+from keras.src.api_export import keras_export
+from keras.src.backend import KerasTensor
+from keras.src.layers.layer import Layer
+
+
+@keras_export("keras.layers.Attention")
+class Attention(Layer):
+    """Dot-product attention layer, a.k.a. Luong-style attention.
+
+    Inputs are a list with 2 or 3 elements:
+    1. A `query` tensor of shape `(batch_size, Tq, dim)`.
+    2. A `value` tensor of shape `(batch_size, Tv, dim)`.
+    3. A optional `key` tensor of shape `(batch_size, Tv, dim)`. If none
+        supplied, `value` will be used as a `key`.
+
+    The calculation follows the steps:
+    1. Calculate attention scores using `query` and `key` with shape
+        `(batch_size, Tq, Tv)`.
+    2. Use scores to calculate a softmax distribution with shape
+        `(batch_size, Tq, Tv)`.
+    3. Use the softmax distribution to create a linear combination of `value`
+        with shape `(batch_size, Tq, dim)`.
+
+    Args:
+        use_scale: If `True`, will create a scalar variable to scale the
+            attention scores.
+        dropout: Float between 0 and 1. Fraction of the units to drop for the
+            attention scores. Defaults to `0.0`.
+        seed: A Python integer to use as random seed in case of `dropout`.
+        score_mode: Function to use to compute attention scores, one of
+            `{"dot", "concat"}`. `"dot"` refers to the dot product between the
+            query and key vectors. `"concat"` refers to the hyperbolic tangent
+            of the concatenation of the `query` and `key` vectors.
+
+    Call arguments:
+        inputs: List of the following tensors:
+            - `query`: Query tensor of shape `(batch_size, Tq, dim)`.
+            - `value`: Value tensor of shape `(batch_size, Tv, dim)`.
+            - `key`: Optional key tensor of shape `(batch_size, Tv, dim)`. If
+                not given, will use `value` for both `key` and `value`, which is
+                the most common case.
+        mask: List of the following tensors:
+            - `query_mask`: A boolean mask tensor of shape `(batch_size, Tq)`.
+                If given, the output will be zero at the positions where
+                `mask==False`.
+            - `value_mask`: A boolean mask tensor of shape `(batch_size, Tv)`.
+                If given, will apply the mask such that values at positions
+                 where `mask==False` do not contribute to the result.
+        return_attention_scores: bool, it `True`, returns the attention scores
+            (after masking and softmax) as an additional output argument.
+        training: Python boolean indicating whether the layer should behave in
+            training mode (adding dropout) or in inference mode (no dropout).
+        use_causal_mask: Boolean. Set to `True` for decoder self-attention. Adds
+            a mask such that position `i` cannot attend to positions `j > i`.
+            This prevents the flow of information from the future towards the
+            past. Defaults to `False`.
+
+    Output:
+        Attention outputs of shape `(batch_size, Tq, dim)`.
+        (Optional) Attention scores after masking and softmax with shape
+            `(batch_size, Tq, Tv)`.
+    """
+
+    def __init__(
+        self,
+        use_scale=False,
+        score_mode="dot",
+        dropout=0.0,
+        seed=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.use_scale = use_scale
+        self.score_mode = score_mode
+        self.dropout = dropout
+        if self.dropout > 0:
+            self.seed_generator = backend.random.SeedGenerator(seed=seed)
+
+        if self.score_mode not in ["dot", "concat"]:
+            raise ValueError(
+                "Invalid value for argument score_mode. "
+                "Expected one of {'dot', 'concat'}. "
+                f"Received: score_mode={score_mode}"
+            )
+
+        self._return_attention_scores = False
+
+    def build(self, input_shape):
+        self._validate_inputs(input_shape)
+        self.scale = None
+        self.concat_score_weight = None
+        if self.use_scale:
+            self.scale = self.add_weight(
+                name="scale",
+                shape=(),
+                initializer="ones",
+                dtype=self.dtype,
+                trainable=True,
+            )
+        if self.score_mode == "concat":
+            self.concat_score_weight = self.add_weight(
+                name="concat_score_weight",
+                shape=(),
+                initializer="ones",
+                dtype=self.dtype,
+                trainable=True,
+            )
+        self.built = True
+
+    def _calculate_scores(self, query, key):
+        """Calculates attention scores as a query-key dot product.
+
+        Args:
+            query: Query tensor of shape `(batch_size, Tq, dim)`.
+            key: Key tensor of shape `(batch_size, Tv, dim)`.
+
+        Returns:
+            Tensor of shape `(batch_size, Tq, Tv)`.
+        """
+        if self.score_mode == "dot":
+            scores = ops.matmul(query, ops.transpose(key, axes=[0, 2, 1]))
+            if self.scale is not None:
+                scores *= self.scale
+        elif self.score_mode == "concat":
+            # Reshape tensors to enable broadcasting.
+            # Reshape into [batch_size, Tq, 1, dim].
+            q_reshaped = ops.expand_dims(query, axis=-2)
+            # Reshape into [batch_size, 1, Tv, dim].
+            k_reshaped = ops.expand_dims(key, axis=-3)
+            if self.scale is not None:
+                scores = self.concat_score_weight * ops.sum(
+                    ops.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1
+                )
+            else:
+                scores = self.concat_score_weight * ops.sum(
+                    ops.tanh(q_reshaped + k_reshaped), axis=-1
+                )
+        else:
+            raise ValueError("scores not computed")
+
+        return scores
+
+    def _apply_scores(self, scores, value, scores_mask=None, training=False):
+        """Applies attention scores to the given value tensor.
+
+        To use this method in your attention layer, follow the steps:
+
+        * Use `query` tensor of shape `(batch_size, Tq)` and `key` tensor of
+            shape `(batch_size, Tv)` to calculate the attention `scores`.
+        * Pass `scores` and `value` tensors to this method. The method applies
+            `scores_mask`, calculates
+            `attention_distribution = softmax(scores)`, then returns
+            `matmul(attention_distribution, value).
+        * Apply `query_mask` and return the result.
+
+        Args:
+            scores: Scores float tensor of shape `(batch_size, Tq, Tv)`.
+            value: Value tensor of shape `(batch_size, Tv, dim)`.
+            scores_mask: A boolean mask tensor of shape `(batch_size, 1, Tv)`
+                or `(batch_size, Tq, Tv)`. If given, scores at positions where
+                `scores_mask==False` do not contribute to the result. It must
+                contain at least one `True` value in each line along the last
+                dimension.
+            training: Python boolean indicating whether the layer should behave
+                in training mode (adding dropout) or in inference mode
+                (no dropout).
+
+        Returns:
+            Tensor of shape `(batch_size, Tq, dim)`.
+            Attention scores after masking and softmax with shape
+                `(batch_size, Tq, Tv)`.
+        """
+        if scores_mask is not None:
+            padding_mask = ops.logical_not(scores_mask)
+            # Bias so padding positions do not contribute to attention
+            # distribution.  Note 65504. is the max float16 value.
+            max_value = 65504.0 if scores.dtype == "float16" else 1.0e9
+            scores -= max_value * ops.cast(padding_mask, dtype=scores.dtype)
+
+        weights = ops.softmax(scores, axis=-1)
+        if training and self.dropout > 0:
+            weights = backend.random.dropout(
+                weights,
+                self.dropout,
+                seed=self.seed_generator,
+            )
+        return ops.matmul(weights, value), weights
+
+    def _calculate_score_mask(self, scores, v_mask, use_causal_mask):
+        if use_causal_mask:
+            # Creates a lower triangular mask, so position i cannot attend to
+            # positions j > i. This prevents the flow of information from the
+            # future into the past.
+            score_shape = ops.shape(scores)
+            # causal_mask_shape = [1, Tq, Tv].
+            mask_shape = (1, score_shape[-2], score_shape[-1])
+            ones_mask = ops.ones(shape=mask_shape, dtype="int32")
+            row_index = ops.cumsum(ones_mask, axis=-2)
+            col_index = ops.cumsum(ones_mask, axis=-1)
+            causal_mask = ops.greater_equal(row_index, col_index)
+
+            if v_mask is not None:
+                # Mask of shape [batch_size, 1, Tv].
+                v_mask = ops.expand_dims(v_mask, axis=-2)
+                return ops.logical_and(v_mask, causal_mask)
+            return causal_mask
+        else:
+            # If not using causal mask, return the value mask as is,
+            # or None if the value mask is not provided.
+            return v_mask
+
+    def call(
+        self,
+        inputs,
+        mask=None,
+        training=False,
+        return_attention_scores=False,
+        use_causal_mask=False,
+    ):
+        self._validate_inputs(inputs=inputs, mask=mask)
+        self._return_attention_scores = return_attention_scores
+        q = inputs[0]
+        v = inputs[1]
+        k = inputs[2] if len(inputs) > 2 else v
+        q_mask = mask[0] if mask else None
+        v_mask = mask[1] if mask else None
+        scores = self._calculate_scores(query=q, key=k)
+        scores_mask = self._calculate_score_mask(
+            scores, v_mask, use_causal_mask
+        )
+        attention_output, attention_scores = self._apply_scores(
+            scores=scores, value=v, scores_mask=scores_mask, training=training
+        )
+        if q_mask is not None:
+            # Mask of shape [batch_size, Tq, 1].
+            q_mask = ops.expand_dims(q_mask, axis=-1)
+            attention_output *= ops.cast(q_mask, dtype=attention_output.dtype)
+        if return_attention_scores:
+            return (attention_output, attention_scores)
+        else:
+            return attention_output
+
+    def compute_mask(self, inputs, mask=None):
+        self._validate_inputs(inputs=inputs, mask=mask)
+        if mask is None or mask[0] is None:
+            return None
+        return ops.convert_to_tensor(mask[0])
+
+    def compute_output_shape(self, input_shape):
+        query_shape, value_shape, key_shape = input_shape
+        if key_shape is None:
+            key_shape = value_shape
+
+        output_shape = (*query_shape[:-1], value_shape[-1])
+        if self._return_attention_scores:
+            scores_shape = (query_shape[0], query_shape[1], key_shape[1])
+            return output_shape, scores_shape
+        return output_shape
+
+    def compute_output_spec(
+        self,
+        inputs,
+        mask=None,
+        return_attention_scores=False,
+        training=None,
+        use_causal_mask=False,
+    ):
+        # Validate and unpack inputs
+        self._validate_inputs(inputs, mask)
+        query = inputs[0]
+        value = inputs[1]
+        key = inputs[2] if len(inputs) > 2 else value
+
+        # Compute primary output shape
+        output_shape = self.compute_output_shape(
+            [query.shape, value.shape, key.shape]
+        )
+        output_spec = KerasTensor(output_shape, dtype=self.compute_dtype)
+
+        # Handle attention scores if requested
+        if self._return_attention_scores or return_attention_scores:
+            scores_shape = (
+                query.shape[0],
+                query.shape[1],
+                key.shape[1],
+            )  # (batch_size, Tq, Tv)
+            attention_scores_spec = KerasTensor(
+                scores_shape, dtype=self.compute_dtype
+            )
+            return (output_spec, attention_scores_spec)
+
+        return output_spec
+
+    def _validate_inputs(self, inputs, mask=None):
+        """Validates arguments of the call method."""
+        class_name = self.__class__.__name__
+        if not isinstance(inputs, list):
+            raise ValueError(
+                f"{class_name} layer must be called on a list of inputs, "
+                "namely [query, value] or [query, value, key]. "
+                f"Received: inputs={inputs}."
+            )
+        if len(inputs) < 2 or len(inputs) > 3:
+            raise ValueError(
+                f"{class_name} layer accepts inputs list of length 2 or 3, "
+                "namely [query, value] or [query, value, key]. "
+                f"Received length: {len(inputs)}."
+            )
+        if mask is not None:
+            if not isinstance(mask, list):
+                raise ValueError(
+                    f"{class_name} layer mask must be a list, "
+                    f"namely [query_mask, value_mask]. Received: mask={mask}."
+                )
+            if len(mask) < 2 or len(mask) > 3:
+                raise ValueError(
+                    f"{class_name} layer accepts mask list of length 2 or 3. "
+                    f"Received: inputs={inputs}, mask={mask}."
+                )
+
+    def get_config(self):
+        base_config = super().get_config()
+        config = {
+            "use_scale": self.use_scale,
+            "score_mode": self.score_mode,
+            "dropout": self.dropout,
+        }
+        return {**base_config, **config}

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/attention/grouped_query_attention.py

@@ -0,0 +1,504 @@
+import math
+
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.api_export import keras_export
+from keras.src.backend.config import is_flash_attention_enabled
+from keras.src.layers.activations.softmax import Softmax
+from keras.src.layers.core.einsum_dense import EinsumDense
+from keras.src.layers.layer import Layer
+from keras.src.layers.regularization.dropout import Dropout
+
+
+@keras_export("keras.layers.GroupQueryAttention")
+class GroupedQueryAttention(Layer):
+    """Grouped Query Attention layer.
+
+    This is an implementation of grouped-query attention introduced by
+    [Ainslie et al., 2023](https://arxiv.org/abs/2305.13245). Here
+    `num_key_value_heads` denotes number of groups, setting
+    `num_key_value_heads` to 1 is equivalent to multi-query attention, and
+    when `num_key_value_heads` is equal to `num_query_heads` it is equivalent
+    to multi-head attention.
+
+    This layer first projects `query`, `key`, and `value` tensors. Then, `key`
+    and `value` are repeated to match the number of heads of `query`.
+
+    Then, the `query` is scaled and dot-producted with `key` tensors. These are
+    softmaxed to obtain attention probabilities. The value tensors are then
+    interpolated by these probabilities and concatenated back to a single
+    tensor.
+
+    Args:
+        head_dim: Size of each attention head.
+        num_query_heads: Number of query attention heads.
+        num_key_value_heads: Number of key and value attention heads.
+        dropout: Dropout probability.
+        use_bias: Boolean, whether the dense layers use bias vectors/matrices.
+        flash_attention: If `None`, the layer attempts to use flash
+            attention for faster and more memory-efficient attention
+            computations when possible. This behavior can be configured using
+            `keras.config.enable_flash_attention()` or
+            `keras.config.disable_flash_attention()`.
+        kernel_initializer: Initializer for dense layer kernels.
+        bias_initializer: Initializer for dense layer biases.
+        kernel_regularizer: Regularizer for dense layer kernels.
+        bias_regularizer: Regularizer for dense layer biases.
+        activity_regularizer: Regularizer for dense layer activity.
+        kernel_constraint: Constraint for dense layer kernels.
+        bias_constraint: Constraint for dense layer kernels.
+        seed: Optional integer to seed the dropout layer.
+
+    Call arguments:
+        query: Query tensor of shape `(batch_dim, target_seq_len, feature_dim)`,
+            where `batch_dim` is batch size, `target_seq_len` is the length of
+            target sequence, and `feature_dim` is dimension of feature.
+        value: Value tensor of shape `(batch_dim, source_seq_len, feature_dim)`,
+            where `batch_dim` is batch size, `source_seq_len` is the length of
+            source sequence, and `feature_dim` is dimension of feature.
+        key: Optional key tensor of shape
+            `(batch_dim, source_seq_len, feature_dim)`. If not given, will use
+            `value` for both `key` and `value`, which is most common case.
+        attention_mask: A boolean mask of shape
+            `(batch_dim, target_seq_len, source_seq_len)`, that prevents
+            attention to certain positions. The boolean mask specifies which
+            query elements can attend to which key elements, where 1 indicates
+            attention and 0 indicates no attention. Broadcasting can happen for
+            the missing batch dimensions and the head dimension.
+        return_attention_scores: A boolean to indicate whether the output
+            should be `(attention_output, attention_scores)` if `True`, or
+            `attention_output` if `False`. Defaults to `False`.
+        training: Python boolean indicating whether the layer should behave in
+            training mode (adding dropout) or in inference mode (no dropout).
+            Will go with either using the training mode of the parent
+            layer/model or `False` (inference) if there is no parent layer.
+        use_causal_mask: A boolean to indicate whether to apply a causal mask to
+            prevent tokens from attending to future tokens (e.g., used in a
+            decoder Transformer).
+
+    Returns:
+        attention_output: Result of the computation, of shape
+            `(batch_dim, target_seq_len, feature_dim)`, where `target_seq_len`
+            is for target sequence length and `feature_dim` is the query input
+            last dim.
+        attention_scores: (Optional) attention coefficients of shape
+            `(batch_dim, num_query_heads, target_seq_len, source_seq_len)`.
+    """
+
+    def __init__(
+        self,
+        head_dim,
+        num_query_heads,
+        num_key_value_heads,
+        dropout=0.0,
+        use_bias=True,
+        flash_attention=None,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        seed=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.supports_masking = True
+        self.head_dim = head_dim
+        self.num_query_heads = num_query_heads
+        self.num_key_value_heads = num_key_value_heads
+        if num_query_heads % num_key_value_heads != 0:
+            raise ValueError(
+                "`num_query_heads` must be divisible"
+                " by `num_key_value_heads`."
+            )
+        self.num_repeats = num_query_heads // num_key_value_heads
+        self.dropout = dropout
+        self.use_bias = use_bias
+        self._flash_attention = flash_attention or is_flash_attention_enabled()
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.bias_initializer = initializers.get(bias_initializer)
+        self.kernel_regularizer = regularizers.get(kernel_regularizer)
+        self.bias_regularizer = regularizers.get(bias_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+        self.kernel_constraint = constraints.get(kernel_constraint)
+        self.bias_constraint = constraints.get(bias_constraint)
+        self.seed = seed
+
+        self._inverse_sqrt_head_dim = 1.0 / math.sqrt(float(self.head_dim))
+        self._return_attention_scores = False
+
+        # Check for flash attention constraints
+        if self._flash_attention and self.dropout > 0.0:
+            raise ValueError(
+                "Dropout is not supported when flash attention is enabled. "
+                "Please set dropout to 0.0 to use flash attention."
+            )
+
+    def build(
+        self,
+        query_shape,
+        value_shape,
+        key_shape=None,
+    ):
+        # Einsum variables:
+        # b = batch size
+        # q = query length
+        # k = key/value length
+        # m = model dim
+        # u = num query heads
+        # v = num key/value heads
+        # h = head dim
+        key_shape = value_shape if key_shape is None else key_shape
+        self.feature_dim = query_shape[-1]
+        self._query_dense = EinsumDense(
+            "bqm,muh->bquh",
+            output_shape=(None, self.num_query_heads, self.head_dim),
+            bias_axes="uh" if self.use_bias else None,
+            name="query",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._query_dense.build(query_shape)
+
+        self._key_dense = EinsumDense(
+            "bkm,mvh->bkvh",
+            output_shape=(None, self.num_key_value_heads, self.head_dim),
+            bias_axes="vh" if self.use_bias else None,
+            name="key",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._key_dense.build(key_shape)
+
+        self._value_dense = EinsumDense(
+            "bkm,mvh->bkvh",
+            output_shape=(None, self.num_key_value_heads, self.head_dim),
+            bias_axes="vh" if self.use_bias else None,
+            name="value",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._value_dense.build(value_shape)
+
+        self._softmax = Softmax(axis=-1, dtype=self.dtype_policy)
+        self._dropout_layer = Dropout(
+            rate=self.dropout, dtype=self.dtype_policy, seed=self.seed
+        )
+
+        self._dot_product_equation = "bquh,bkuh->buqk"
+        self._combine_equation = "buqk,bkuh->bquh"
+
+        self._output_dense = EinsumDense(
+            "bquh,uhm->bqm",
+            output_shape=(None, self.feature_dim),
+            bias_axes="m" if self.use_bias else None,
+            name="attention_output",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._output_dense.build(
+            (None, None, self.num_query_heads, self.head_dim)
+        )
+        self.built = True
+
+    def _get_common_kwargs_for_sublayer(self):
+        common_kwargs = dict(
+            kernel_regularizer=self.kernel_regularizer,
+            bias_regularizer=self.bias_regularizer,
+            activity_regularizer=self.activity_regularizer,
+            kernel_constraint=self.kernel_constraint,
+            bias_constraint=self.bias_constraint,
+            dtype=self.dtype_policy,
+        )
+        # Create new clone of kernel/bias initializer, so that we don't reuse
+        # the initializer instance, which could lead to same init value since
+        # initializer is stateless.
+        kernel_initializer = self.kernel_initializer.__class__.from_config(
+            self.kernel_initializer.get_config()
+        )
+        bias_initializer = self.bias_initializer.__class__.from_config(
+            self.bias_initializer.get_config()
+        )
+        common_kwargs["kernel_initializer"] = kernel_initializer
+        common_kwargs["bias_initializer"] = bias_initializer
+        return common_kwargs
+
+    def call(
+        self,
+        query,
+        value,
+        key=None,
+        query_mask=None,
+        value_mask=None,
+        key_mask=None,
+        attention_mask=None,
+        return_attention_scores=False,
+        training=None,
+        use_causal_mask=False,
+    ):
+        self._return_attention_scores = return_attention_scores
+        if key is None:
+            key = value
+
+        attention_mask = self._compute_attention_mask(
+            query,
+            value,
+            query_mask=query_mask,
+            value_mask=value_mask,
+            key_mask=key_mask,
+            attention_mask=attention_mask,
+            use_causal_mask=use_causal_mask,
+        )
+
+        query = self._query_dense(query)
+        key = self._key_dense(key)
+        value = self._value_dense(value)
+
+        key = ops.repeat(
+            key, self.num_repeats, axis=2
+        )  # (batch_dim, source_seq_len, query_heads, head_dim)
+        value = ops.repeat(
+            value, self.num_repeats, axis=2
+        )  # (batch_dim, source_seq_len, query_heads, head_dim)
+
+        output, scores = self._compute_attention(
+            query,
+            key,
+            value,
+            attention_mask=attention_mask,
+            training=training,
+        )
+
+        output = self._output_dense(
+            output
+        )  # (batch_dim, target_seq_len, feature_dim)
+
+        if return_attention_scores:
+            return output, scores
+        return output
+
+    def _compute_attention_mask(
+        self,
+        query,
+        value,
+        query_mask=None,
+        value_mask=None,
+        key_mask=None,
+        attention_mask=None,
+        use_causal_mask=False,
+    ):
+        """Computes the attention mask, using the Keras masks of the inputs.
+
+        * The `query`'s mask is reshaped from [B, T] to [B, T, 1].
+        * The `value`'s mask is reshaped from [B, S] to [B, 1, S].
+        * The `key`'s mask is reshaped from [B, S] to [B, 1, S]. The `key`'s
+          mask is ignored if `key` is `None` or if `key is value`.
+        * If `use_causal_mask=True`, then the causal mask is computed. Its shape
+          is [1, T, S].
+
+        All defined masks are merged using a logical AND operation (`&`).
+
+        In general, if the `query` and `value` are masked, then there is no need
+        to define the `attention_mask`.
+
+        Args:
+            query: Projected query tensor of shape `(B, T, N, key_dim)`.
+            key: Projected key tensor of shape `(B, T, N, key_dim)`.
+            value: Projected value tensor of shape `(B, T, N, value_dim)`.
+            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+                attention to certain positions.
+            use_causal_mask: A boolean to indicate whether to apply a causal
+                mask to prevent tokens from attending to future tokens (e.g.,
+                used in a decoder Transformer).
+
+        Returns:
+            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+                attention to certain positions, based on the Keras masks of the
+                `query`, `key`, `value`, and `attention_mask` tensors, and the
+                causal mask if `use_causal_mask=True`.
+        """
+        auto_mask = None
+        if query_mask is not None:
+            query_mask = ops.cast(query_mask, "bool")  # defensive casting
+            # B = batch size, T = max query length
+            auto_mask = ops.expand_dims(query_mask, -1)  # shape is [B, T, 1]
+        if value_mask is not None:
+            value_mask = ops.cast(value_mask, "bool")  # defensive casting
+            # B = batch size, S == max value length
+            mask = ops.expand_dims(value_mask, -2)  # shape is [B, 1, S]
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+        if key_mask is not None:
+            key_mask = ops.cast(key_mask, "bool")  # defensive casting
+            # B == batch size, S == max key length == max value length
+            mask = ops.expand_dims(key_mask, -2)  # shape is [B, 1, S]
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+        if use_causal_mask:
+            # the shape of the causal mask is [1, T, S]
+            mask = self._compute_causal_mask(query, value)
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+        if auto_mask is not None:
+            # merge attention_mask & automatic mask, to shape [B, T, S]
+            attention_mask = (
+                auto_mask
+                if attention_mask is None
+                else ops.cast(attention_mask, bool) & auto_mask
+            )
+        return attention_mask
+
+    def _compute_causal_mask(self, query, value=None):
+        """Computes a causal mask (e.g., for masked self-attention layers).
+
+        For example, if query and value both contain sequences of length 4,
+        this function returns a boolean tensor equal to:
+
+        ```
+        [[[True,  False, False, False],
+          [True,  True,  False, False],
+          [True,  True,  True,  False],
+          [True,  True,  True,  True]]]
+        ```
+
+        Args:
+            query: query tensor of shape `(B, T, ...)`.
+            value: value tensor of shape `(B, S, ...)` (optional, defaults to
+                query).
+
+        Returns:
+            mask: a boolean tensor of shape `(1, T, S)` containing a lower
+                triangular matrix of shape `(T, S)`.
+        """
+        q_seq_length = ops.shape(query)[1]
+        v_seq_length = q_seq_length if value is None else ops.shape(value)[1]
+        ones_mask = ops.ones((1, q_seq_length, v_seq_length), dtype="int32")
+        row_index = ops.cumsum(ones_mask, axis=-2)
+        col_index = ops.cumsum(ones_mask, axis=-1)
+        return ops.greater_equal(row_index, col_index)
+
+    def _compute_attention(
+        self, query, key, value, attention_mask=None, training=None
+    ):
+        # Check for flash attention constraints
+        if self._flash_attention and self._return_attention_scores:
+            raise ValueError(
+                "Returning attention scores is not supported when flash "
+                "attention is enabled. Please disable flash attention to access"
+                " attention scores."
+            )
+
+        # Determine whether to use dot-product attention
+        use_dot_product_attention = not (
+            self.dropout > 0.0
+            or self._return_attention_scores
+            or (len(query.shape) != 4)
+        )
+
+        if use_dot_product_attention:
+            if attention_mask is not None:
+                # Ensure attention_mask has the correct shape for broadcasting
+                # Expected shape: [batch_size, num_heads, query_seq_len,
+                # key_seq_len].
+                mask_expansion_axis = -1 * 2 - 1
+                len_attention_scores_shape = 4  # Only accepts 4D inputs
+                for _ in range(
+                    len_attention_scores_shape - len(attention_mask.shape)
+                ):
+                    attention_mask = ops.expand_dims(
+                        attention_mask, axis=mask_expansion_axis
+                    )
+                attention_mask = ops.cast(attention_mask, dtype="bool")
+            # Directly compute the attention output using dot-product attention
+            attention_output = ops.dot_product_attention(
+                query=query,
+                key=key,
+                value=value,
+                bias=None,
+                mask=attention_mask,
+                scale=self._inverse_sqrt_head_dim,
+                is_causal=False,
+                flash_attention=self._flash_attention,
+            )
+            return attention_output, None
+
+        # Default behavior without flash attention, with explicit attention
+        # scores
+        query = ops.multiply(
+            query, ops.cast(self._inverse_sqrt_head_dim, query.dtype)
+        )
+        # Take the dot product between "query" and "key" to get the raw
+        # attention scores.
+        scores = ops.einsum(
+            self._dot_product_equation, query, key
+        )  # (batch_dim, query_heads, target_seq_len, source_seq_len)
+        scores = self._masked_softmax(scores, attention_mask=attention_mask)
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        if self.dropout > 0.0:
+            scores_dropout = self._dropout_layer(scores, training=training)
+        else:
+            scores_dropout = scores
+        output = ops.einsum(self._combine_equation, scores_dropout, value)
+        return output, scores
+
+    def _masked_softmax(self, scores, attention_mask=None):
+        # Normalize the attention scores to probabilities.
+        # scores = [B, N, T, S]
+        if attention_mask is not None:
+            # The expand dim happens starting from the `num_heads` dimension,
+            # (<batch_dims>, num_heads, <query_attention_dims,
+            # key_attention_dims>)
+            mask_expansion_axis = -1 * 2 - 1
+            for _ in range(len(scores.shape) - len(attention_mask.shape)):
+                attention_mask = ops.expand_dims(
+                    attention_mask, axis=mask_expansion_axis
+                )
+        return self._softmax(scores, mask=attention_mask)
+
+    def compute_output_shape(
+        self,
+        query_shape,
+        value_shape,
+        key_shape=None,
+    ):
+        if key_shape is None:
+            key_shape = value_shape
+
+        if query_shape[-1] != value_shape[-1]:
+            raise ValueError(
+                "The last dimension of `query_shape` and `value_shape` "
+                f"must be equal, but are {query_shape[-1]}, {value_shape[-1]}. "
+                "Received: query_shape={query_shape}, value_shape={value_shape}"
+            )
+
+        if value_shape[1:-1] != key_shape[1:-1]:
+            raise ValueError(
+                "All dimensions of `value` and `key`, except the last one, "
+                f"must be equal. Received: value_shape={value_shape} and "
+                f"key_shape={key_shape}"
+            )
+
+        return query_shape
+
+    def get_config(self):
+        config = {
+            "head_dim": self.head_dim,
+            "num_query_heads": self.num_query_heads,
+            "num_key_value_heads": self.num_key_value_heads,
+            "use_bias": self.use_bias,
+            "dropout": self.dropout,
+            "kernel_initializer": initializers.serialize(
+                self.kernel_initializer
+            ),
+            "bias_initializer": initializers.serialize(self.bias_initializer),
+            "kernel_regularizer": regularizers.serialize(
+                self.kernel_regularizer
+            ),
+            "bias_regularizer": regularizers.serialize(self.bias_regularizer),
+            "activity_regularizer": regularizers.serialize(
+                self.activity_regularizer
+            ),
+            "kernel_constraint": constraints.serialize(self.kernel_constraint),
+            "bias_constraint": constraints.serialize(self.bias_constraint),
+            "seed": self.seed,
+        }
+        base_config = super().get_config()
+        return {**base_config, **config}

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/attention/multi_head_attention.py

@@ -0,0 +1,827 @@
+import math
+import string
+
+import numpy as np
+
+from keras.src import backend
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.api_export import keras_export
+from keras.src.backend.config import is_flash_attention_enabled
+from keras.src.layers.activations.softmax import Softmax
+from keras.src.layers.core.einsum_dense import EinsumDense
+from keras.src.layers.layer import Layer
+from keras.src.layers.regularization.dropout import Dropout
+
+
+@keras_export("keras.layers.MultiHeadAttention")
+class MultiHeadAttention(Layer):
+    """MultiHeadAttention layer.
+
+    This is an implementation of multi-headed attention as described in the
+    paper "Attention is all you Need"
+    [Vaswani et al., 2017](https://arxiv.org/abs/1706.03762).
+    If `query`, `key,` `value` are the same, then
+    this is self-attention. Each timestep in `query` attends to the
+    corresponding sequence in `key`, and returns a fixed-width vector.
+
+    This layer first projects `query`, `key` and `value`. These are
+    (effectively) a list of tensors of length `num_attention_heads`, where the
+    corresponding shapes are `(batch_size, <query dimensions>, key_dim)`,
+    `(batch_size, <key/value dimensions>, key_dim)`,
+    `(batch_size, <key/value dimensions>, value_dim)`.
+
+    Then, the query and key tensors are dot-producted and scaled. These are
+    softmaxed to obtain attention probabilities. The value tensors are then
+    interpolated by these probabilities, then concatenated back to a single
+    tensor.
+
+    Finally, the result tensor with the last dimension as `value_dim` can take
+    a linear projection and return.
+
+    Args:
+        num_heads: Number of attention heads.
+        key_dim: Size of each attention head for query and key.
+        value_dim: Size of each attention head for value.
+        dropout: Dropout probability.
+        use_bias: Boolean, whether the dense layers use bias vectors/matrices.
+        output_shape: The expected shape of an output tensor, besides the batch
+            and sequence dims. If not specified, projects back to the query
+            feature dim (the query input's last dimension).
+        attention_axes: axes over which the attention is applied. `None` means
+            attention over all axes, but batch, heads, and features.
+        flash_attention: If `None`, the layer attempts to use flash
+            attention for faster and more memory-efficient attention
+            computations when possible. This behavior can be configured using
+            `keras.config.enable_flash_attention()` or
+            `keras.config.disable_flash_attention()`.
+        kernel_initializer: Initializer for dense layer kernels.
+        bias_initializer: Initializer for dense layer biases.
+        kernel_regularizer: Regularizer for dense layer kernels.
+        bias_regularizer: Regularizer for dense layer biases.
+        activity_regularizer: Regularizer for dense layer activity.
+        kernel_constraint: Constraint for dense layer kernels.
+        bias_constraint: Constraint for dense layer kernels.
+        seed: Optional integer to seed the dropout layer.
+
+    Call arguments:
+        query: Query tensor of shape `(B, T, dim)`, where `B` is the batch size,
+            `T` is the target sequence length, and dim is the feature dimension.
+        value: Value tensor of shape `(B, S, dim)`, where `B` is the batch size,
+            `S` is the source sequence length, and dim is the feature dimension.
+        key: Optional key tensor of shape `(B, S, dim)`. If not given, will
+            use `value` for both `key` and `value`, which is the most common
+            case.
+        attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+            attention to certain positions. The boolean mask specifies which
+            query elements can attend to which key elements, 1 indicates
+            attention and 0 indicates no attention. Broadcasting can happen for
+            the missing batch dimensions and the head dimension.
+        return_attention_scores: A boolean to indicate whether the output should
+            be `(attention_output, attention_scores)` if `True`, or
+            `attention_output` if `False`. Defaults to `False`.
+        training: Python boolean indicating whether the layer should behave in
+            training mode (adding dropout) or in inference mode (no dropout).
+            Will go with either using the training mode of the parent
+            layer/model, or `False` (inference) if there is no parent layer.
+        use_causal_mask: A boolean to indicate whether to apply a causal mask to
+            prevent tokens from attending to future tokens (e.g., used in a
+            decoder Transformer).
+
+    Returns:
+        attention_output: The result of the computation, of shape `(B, T, E)`,
+            where `T` is for target sequence shapes and `E` is the query input
+            last dimension if `output_shape` is `None`. Otherwise, the
+            multi-head outputs are projected to the shape specified by
+            `output_shape`.
+        attention_scores: (Optional) multi-head attention coefficients over
+            attention axes.
+    """
+
+    def __init__(
+        self,
+        num_heads,
+        key_dim,
+        value_dim=None,
+        dropout=0.0,
+        use_bias=True,
+        output_shape=None,
+        attention_axes=None,
+        flash_attention=None,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        seed=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.supports_masking = True
+        self._num_heads = num_heads
+        self._key_dim = key_dim
+        self._value_dim = value_dim if value_dim else key_dim
+        self._dropout = dropout
+        self._use_bias = use_bias
+        if output_shape:
+            if isinstance(output_shape, int):
+                output_shape = (output_shape,)
+            try:
+                output_shape = tuple(output_shape)
+            except:
+                raise ValueError(
+                    f"Invalid `output_shape`: {output_shape}. When "
+                    "specified, the `output_shape` should be of type tuple, "
+                    "list, or int."
+                )
+        self._output_shape = output_shape
+        self._flash_attention = flash_attention or is_flash_attention_enabled()
+        self._kernel_initializer = initializers.get(kernel_initializer)
+        self._bias_initializer = initializers.get(bias_initializer)
+        self._kernel_regularizer = regularizers.get(kernel_regularizer)
+        self._bias_regularizer = regularizers.get(bias_regularizer)
+        self._activity_regularizer = regularizers.get(activity_regularizer)
+        self._kernel_constraint = constraints.get(kernel_constraint)
+        self._bias_constraint = constraints.get(bias_constraint)
+        if isinstance(attention_axes, int):
+            attention_axes = (attention_axes,)
+        elif attention_axes and not isinstance(attention_axes, (list, tuple)):
+            raise ValueError(
+                "`attention_axes` must be an int, list, or tuple."
+                f"Received: attention_axes={attention_axes}"
+            )
+        self._attention_axes = attention_axes
+        self.seed = seed
+
+        self._inverse_sqrt_key_dim = 1.0 / math.sqrt(float(self._key_dim))
+        self._return_attention_scores = False
+
+        # Check for flash attention constraints
+        if self._flash_attention and self._dropout > 0.0:
+            raise ValueError(
+                "Dropout is not supported when flash attention is enabled. "
+                "Please set dropout to 0.0 to use flash attention."
+            )
+
+    @property
+    def num_heads(self):
+        return self._num_heads
+
+    @property
+    def key_dim(self):
+        return self._key_dim
+
+    @property
+    def value_dim(self):
+        return self._value_dim
+
+    @property
+    def dropout(self):
+        return self._dropout
+
+    @property
+    def use_bias(self):
+        return self._use_bias
+
+    # Avoid exposing `output_shape` as it may conflict with `Functional` and
+    # `Sequential` models when calling `summary()`.
+
+    @property
+    def attention_axes(self):
+        return self._attention_axes
+
+    def get_config(self):
+        base_config = super().get_config()
+        config = {
+            "num_heads": self._num_heads,
+            "key_dim": self._key_dim,
+            "value_dim": self._value_dim,
+            "dropout": self._dropout,
+            "use_bias": self._use_bias,
+            "output_shape": self._output_shape,
+            "attention_axes": self._attention_axes,
+            "kernel_initializer": initializers.serialize(
+                self._kernel_initializer
+            ),
+            "bias_initializer": initializers.serialize(self._bias_initializer),
+            "kernel_regularizer": regularizers.serialize(
+                self._kernel_regularizer
+            ),
+            "bias_regularizer": regularizers.serialize(self._bias_regularizer),
+            "activity_regularizer": regularizers.serialize(
+                self._activity_regularizer
+            ),
+            "kernel_constraint": constraints.serialize(self._kernel_constraint),
+            "bias_constraint": constraints.serialize(self._bias_constraint),
+            "seed": self.seed,
+        }
+        return {**base_config, **config}
+
+    def build(
+        self,
+        query_shape,
+        value_shape,
+        key_shape=None,
+    ):
+        """Builds layers and variables.
+
+        Args:
+            query_shape: Shape of the `query` tensor.
+            value_shape: Shape of the `value` tensor.
+            key: Optional shape of the `key` tensor.
+        """
+        key_shape = value_shape if key_shape is None else key_shape
+
+        if value_shape[1:-1] != key_shape[1:-1]:
+            raise ValueError(
+                "All dimensions of `value` and `key`, except the last one, "
+                f"must be equal. Received: value_shape={value_shape} and "
+                f"key_shape={key_shape}"
+            )
+
+        query_rank = len(query_shape)
+        value_rank = len(value_shape)
+        key_rank = len(key_shape)
+        einsum_equation, bias_axes, output_rank = _build_proj_equation(
+            query_rank - 1, bound_dims=1, output_dims=2
+        )
+        self._query_dense = EinsumDense(
+            einsum_equation,
+            output_shape=_get_output_shape(
+                output_rank - 1, [self._num_heads, self._key_dim]
+            ),
+            bias_axes=bias_axes if self._use_bias else None,
+            name="query",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._query_dense.build(query_shape)
+        einsum_equation, bias_axes, output_rank = _build_proj_equation(
+            key_rank - 1, bound_dims=1, output_dims=2
+        )
+        self._key_dense = EinsumDense(
+            einsum_equation,
+            output_shape=_get_output_shape(
+                output_rank - 1, [self._num_heads, self._key_dim]
+            ),
+            bias_axes=bias_axes if self._use_bias else None,
+            name="key",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._key_dense.build(key_shape)
+        einsum_equation, bias_axes, output_rank = _build_proj_equation(
+            value_rank - 1, bound_dims=1, output_dims=2
+        )
+        self._value_dense = EinsumDense(
+            einsum_equation,
+            output_shape=_get_output_shape(
+                output_rank - 1, [self._num_heads, self._value_dim]
+            ),
+            bias_axes=bias_axes if self._use_bias else None,
+            name="value",
+            **self._get_common_kwargs_for_sublayer(),
+        )
+        self._value_dense.build(value_shape)
+
+        # Builds the attention computations for multi-head dot product
+        # attention.  These computations could be wrapped into the keras
+        # attention layer once it supports multi-head einsum computations.
+        self._build_attention(output_rank)
+        self._output_dense = self._make_output_dense(
+            query_shape,
+            self._get_common_kwargs_for_sublayer(),
+            "attention_output",
+        )
+        output_dense_input_shape = list(
+            self._query_dense.compute_output_shape(query_shape)
+        )
+        output_dense_input_shape[-1] = self._value_dim
+        self._output_dense.build(tuple(output_dense_input_shape))
+        self.built = True
+
+    @property
+    def query_dense(self):
+        return self._query_dense
+
+    @property
+    def key_dense(self):
+        return self._key_dense
+
+    @property
+    def value_dense(self):
+        return self._value_dense
+
+    @property
+    def output_dense(self):
+        return self._output_dense
+
+    def _get_common_kwargs_for_sublayer(self):
+        common_kwargs = dict(
+            kernel_regularizer=self._kernel_regularizer,
+            bias_regularizer=self._bias_regularizer,
+            activity_regularizer=self._activity_regularizer,
+            kernel_constraint=self._kernel_constraint,
+            bias_constraint=self._bias_constraint,
+            dtype=self.dtype_policy,
+        )
+        # Create new clone of kernel/bias initializer, so that we don't reuse
+        # the initializer instance, which could lead to same init value since
+        # initializer is stateless.
+        kernel_initializer = self._kernel_initializer.__class__.from_config(
+            self._kernel_initializer.get_config()
+        )
+        bias_initializer = self._bias_initializer.__class__.from_config(
+            self._bias_initializer.get_config()
+        )
+        common_kwargs["kernel_initializer"] = kernel_initializer
+        common_kwargs["bias_initializer"] = bias_initializer
+        return common_kwargs
+
+    def _make_output_dense(self, query_shape, common_kwargs, name=None):
+        """Builds the output projection matrix.
+
+        Args:
+            free_dims: Number of free dimensions for einsum equation building.
+            common_kwargs: Common keyword arguments for einsum layer.
+            name: Name for the projection layer.
+
+        Returns:
+            Projection layer.
+        """
+        query_rank = len(query_shape)
+        if self._output_shape:
+            output_shape = self._output_shape
+        else:
+            output_shape = [query_shape[-1]]
+        einsum_equation, bias_axes, output_rank = _build_proj_equation(
+            query_rank - 1, bound_dims=2, output_dims=len(output_shape)
+        )
+        return EinsumDense(
+            einsum_equation,
+            output_shape=_get_output_shape(output_rank - 1, output_shape),
+            bias_axes=bias_axes if self._use_bias else None,
+            name=name,
+            **common_kwargs,
+        )
+
+    def _build_attention(self, rank):
+        """Builds multi-head dot-product attention computations.
+
+        This function builds attributes necessary for `_compute_attention` to
+        customize attention computation to replace the default dot-product
+        attention.
+
+        Args:
+            rank: the rank of query, key, value tensors.
+        """
+        if self._attention_axes is None:
+            self._attention_axes = tuple(range(1, rank - 2))
+        else:
+            self._attention_axes = tuple(self._attention_axes)
+        (
+            self._dot_product_equation,
+            self._combine_equation,
+            attn_scores_rank,
+        ) = _build_attention_equation(rank, attn_axes=self._attention_axes)
+        norm_axes = tuple(
+            range(
+                attn_scores_rank - len(self._attention_axes), attn_scores_rank
+            )
+        )
+        self._softmax = Softmax(axis=norm_axes, dtype=self.dtype_policy)
+        self._dropout_layer = Dropout(
+            rate=self._dropout, dtype=self.dtype_policy, seed=self.seed
+        )
+
+    def _masked_softmax(self, attention_scores, attention_mask=None):
+        # Normalize the attention scores to probabilities.
+        # attention_scores = [B, N, T, S]
+        if attention_mask is not None:
+            # The expand dim happens starting from the `num_heads` dimension,
+            # (<batch_dims>, num_heads, <query_attention_dims,
+            # key_attention_dims>)
+            mask_expansion_axis = -len(self._attention_axes) * 2 - 1
+            for _ in range(
+                len(attention_scores.shape) - len(attention_mask.shape)
+            ):
+                attention_mask = ops.expand_dims(
+                    attention_mask, axis=mask_expansion_axis
+                )
+        return self._softmax(attention_scores, mask=attention_mask)
+
+    def _compute_attention(
+        self,
+        query,
+        key,
+        value,
+        attention_mask=None,
+        training=None,
+    ):
+        """Applies Dot-product attention with query, key, value tensors.
+
+        This function defines the computation inside `call` with projected
+        multi-head Q, K, V inputs. Users can override this function for
+        customized attention implementation.
+
+        Args:
+            query: Projected query tensor of shape `(B, T, N, key_dim)`.
+            key: Projected key tensor of shape `(B, S, N, key_dim)`.
+            value: Projected value tensor of shape `(B, S, N, value_dim)`.
+            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+                attention to certain positions. It is generally not needed if
+                the `query` and `value` (and/or `key`) are masked.
+            training: Python boolean indicating whether the layer should behave
+                in training mode (adding dropout) or in inference mode (doing
+                nothing).
+
+        Returns:
+          attention_output: Multi-headed outputs of attention computation.
+          attention_scores: Multi-headed attention weights.
+        """
+        # Check for flash attention constraints
+        if self._flash_attention and self._return_attention_scores:
+            raise ValueError(
+                "Returning attention scores is not supported when flash "
+                "attention is enabled. Please disable flash attention to access"
+                " attention scores."
+            )
+
+        # Determine whether to use dot-product attention
+        use_dot_product_attention = not (
+            self._dropout > 0.0
+            or self._return_attention_scores
+            or (len(query.shape) != 4)
+        )
+
+        if use_dot_product_attention:
+            if attention_mask is not None:
+                # Ensure attention_mask has the correct shape for broadcasting
+                # Expected shape: [batch_size, num_heads, query_seq_len,
+                # key_seq_len].
+                mask_expansion_axis = -len(self._attention_axes) * 2 - 1
+                len_attention_scores_shape = 4  # Only accepts 4D inputs
+                for _ in range(
+                    len_attention_scores_shape - len(attention_mask.shape)
+                ):
+                    attention_mask = ops.expand_dims(
+                        attention_mask, axis=mask_expansion_axis
+                    )
+                attention_mask = ops.cast(attention_mask, dtype="bool")
+            # Directly compute the attention output using dot-product attention
+            attention_output = ops.dot_product_attention(
+                query=query,
+                key=key,
+                value=value,
+                bias=None,
+                mask=attention_mask,
+                scale=self._inverse_sqrt_key_dim,
+                is_causal=False,
+                flash_attention=self._flash_attention,
+            )
+            return attention_output, None
+
+        # Default behavior without flash attention, with explicit attention
+        # scores
+        query = ops.multiply(
+            query, ops.cast(self._inverse_sqrt_key_dim, query.dtype)
+        )
+
+        # Take the dot product between "query" and "key" to get the raw
+        # attention scores.
+        attention_scores = ops.einsum(self._dot_product_equation, key, query)
+
+        # Apply the mask using the custom masked softmax
+        attention_scores = self._masked_softmax(
+            attention_scores, attention_mask
+        )
+
+        # Apply dropout to the attention scores if needed
+        if self._dropout > 0.0:
+            final_attn_scores = self._dropout_layer(
+                attention_scores, training=training
+            )
+        else:
+            final_attn_scores = attention_scores
+
+        # `context_layer` = [B, T, N, H]
+        attention_output = ops.einsum(
+            self._combine_equation, final_attn_scores, value
+        )
+        return attention_output, attention_scores
+
+    def call(
+        self,
+        query,
+        value,
+        key=None,
+        query_mask=None,
+        value_mask=None,
+        key_mask=None,
+        attention_mask=None,
+        return_attention_scores=False,
+        training=None,
+        use_causal_mask=False,
+    ):
+        self._return_attention_scores = return_attention_scores
+        if key is None:
+            key = value
+
+        # Delete the masks because the masks are handled at the level of the
+        # layer
+        query_mask = backend.get_keras_mask(query)
+        backend.set_keras_mask(query, None)
+        backend.set_keras_mask(value, None)
+        backend.set_keras_mask(key, None)
+
+        attention_mask = self._compute_attention_mask(
+            query,
+            value,
+            query_mask=query_mask,
+            value_mask=value_mask,
+            key_mask=key_mask,
+            attention_mask=attention_mask,
+            use_causal_mask=use_causal_mask,
+        )
+        #   N = `num_attention_heads`
+        #   H = `size_per_head`
+
+        # `query` = [B, T, N, H]
+        query = self._query_dense(query)
+
+        # `key` = [B, S, N, H]
+        key = self._key_dense(key)
+
+        # `value` = [B, S, N, H]
+        value = self._value_dense(value)
+        attention_output, attention_scores = self._compute_attention(
+            query,
+            key,
+            value,
+            attention_mask,
+            training,
+        )
+        attention_output = self._output_dense(attention_output)
+
+        # Set mask on output if needed
+        if query_mask is not None:
+            backend.set_keras_mask(attention_output, query_mask)
+
+        if return_attention_scores:
+            return attention_output, attention_scores
+        return attention_output
+
+    def _compute_attention_mask(
+        self,
+        query,
+        value,
+        query_mask=None,
+        value_mask=None,
+        key_mask=None,
+        attention_mask=None,
+        use_causal_mask=False,
+    ):
+        """Computes the attention mask, using the Keras masks of the inputs.
+
+        * The `query`'s mask is reshaped from [B, T] to [B, T, 1].
+        * The `value`'s mask is reshaped from [B, S] to [B, 1, S].
+        * The `key`'s mask is reshaped from [B, S] to [B, 1, S]. The `key`'s
+          mask is ignored if `key` is `None` or if `key is value`.
+        * If `use_causal_mask=True`, then the causal mask is computed. Its shape
+          is [1, T, S].
+
+        All defined masks are merged using a logical AND operation (`&`).
+
+        In general, if the `query` and `value` are masked, then there is no need
+        to define the `attention_mask`.
+
+        Args:
+            query: Projected query tensor of shape `(B, T, N, key_dim)`.
+            key: Projected key tensor of shape `(B, T, N, key_dim)`.
+            value: Projected value tensor of shape `(B, T, N, value_dim)`.
+            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+                attention to certain positions.
+            use_causal_mask: A boolean to indicate whether to apply a causal
+                mask to prevent tokens from attending to future tokens (e.g.,
+                used in a decoder Transformer).
+
+        Returns:
+            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
+                attention to certain positions, based on the Keras masks of the
+                `query`, `key`, `value`, and `attention_mask` tensors, and the
+                causal mask if `use_causal_mask=True`.
+        """
+        auto_mask = None
+        if query_mask is not None:
+            query_mask = ops.cast(query_mask, "bool")  # defensive casting
+            # B = batch size, T = max query length
+            auto_mask = ops.expand_dims(query_mask, -1)  # shape is [B, T, 1]
+        if value_mask is not None:
+            value_mask = ops.cast(value_mask, "bool")  # defensive casting
+            # B = batch size, S == max value length
+            mask = ops.expand_dims(value_mask, -2)  # shape is [B, 1, S]
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+        if key_mask is not None:
+            key_mask = ops.cast(key_mask, "bool")  # defensive casting
+            # B == batch size, S == max key length == max value length
+            mask = ops.expand_dims(key_mask, -2)  # shape is [B, 1, S]
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+        if use_causal_mask:
+            # the shape of the causal mask is [1, T, S]
+            mask = self._compute_causal_mask(query, value)
+            auto_mask = mask if auto_mask is None else auto_mask & mask
+
+        if attention_mask is not None:
+            attention_mask = ops.cast(attention_mask, "bool")
+        if auto_mask is not None:
+            # merge attention_mask & automatic mask, to shape [B, T, S]
+            attention_mask = (
+                auto_mask
+                if attention_mask is None
+                else attention_mask & auto_mask
+            )
+        return attention_mask
+
+    def _compute_causal_mask(self, query, value=None):
+        """Computes a causal mask (e.g., for masked self-attention layers).
+
+        For example, if query and value both contain sequences of length 4,
+        this function returns a boolean tensor equal to:
+
+        ```
+        [[[True,  False, False, False],
+          [True,  True,  False, False],
+          [True,  True,  True,  False],
+          [True,  True,  True,  True]]]
+        ```
+
+        Args:
+            query: query tensor of shape `(B, T, ...)`.
+            value: value tensor of shape `(B, S, ...)` (optional, defaults to
+                query).
+
+        Returns:
+            mask: a boolean tensor of shape `(1, T, S)` containing a lower
+                triangular matrix of shape `(T, S)`.
+        """
+        q_seq_length = ops.shape(query)[1]
+        v_seq_length = q_seq_length if value is None else ops.shape(value)[1]
+        ones_mask = ops.ones((1, q_seq_length, v_seq_length), dtype="int32")
+        row_index = ops.cumsum(ones_mask, axis=-2)
+        col_index = ops.cumsum(ones_mask, axis=-1)
+        return ops.greater_equal(row_index, col_index)
+
+    def compute_output_shape(
+        self,
+        query_shape,
+        value_shape,
+        key_shape=None,
+    ):
+        query_shape = tuple(query_shape)
+        value_shape = tuple(value_shape)
+        if key_shape is None:
+            key_shape = value_shape
+        else:
+            key_shape = tuple(key_shape)
+
+        if value_shape[1:-1] != key_shape[1:-1]:
+            raise ValueError(
+                "All dimensions of `value` and `key`, except the last one, "
+                f"must be equal. Received: value_shape={value_shape} and "
+                f"key_shape={key_shape}"
+            )
+        if self._output_shape:
+            query_shape = query_shape[:-1] + self._output_shape
+        return query_shape
+
+    def compute_output_spec(
+        self,
+        query,
+        value,
+        key=None,
+        query_mask=None,
+        value_mask=None,
+        key_mask=None,
+        attention_mask=None,
+        return_attention_scores=False,
+        training=None,
+        use_causal_mask=False,
+    ):
+        if key is not None:
+            key_shape = key.shape
+        else:
+            key_shape = None
+        output_shape = self.compute_output_shape(
+            query.shape, value.shape, key_shape
+        )
+        output_spec = backend.KerasTensor(
+            output_shape, dtype=self.compute_dtype
+        )
+        if return_attention_scores:
+            length = query.shape[1]
+            attention_shape = (query.shape[0], self.num_heads, length, length)
+            return output_spec, backend.KerasTensor(
+                attention_shape, dtype=self.compute_dtype
+            )
+        return output_spec
+
+
+def _index_to_einsum_variable(i):
+    """Converts an index to a einsum variable name.
+
+    We simply map indices to lowercase characters, e.g. 0 -> 'a', 1 -> 'b'.
+    """
+    return string.ascii_lowercase[i]
+
+
+def _build_attention_equation(rank, attn_axes):
+    """Builds einsum equations for the attention computation.
+
+    Query, key, value inputs after projection are expected to have the shape as:
+    `(bs, <non-attention dims>, <attention dims>, num_heads, channels)`.
+    `bs` and `<non-attention dims>` are treated as `<batch dims>`.
+
+    The attention operations can be generalized:
+    1. Query-key dot product:
+        (<batch dims>, <query attention dims>, num_heads, channels),
+        (<batch dims>, <key attention dims>, num_heads, channels) ->
+        (<batch dims>, num_heads, <query attention dims>, <key attention dims>)
+    2. Combination:
+        (<batch dims>, num_heads, <query attention dims>, <key attention dims>),
+        (<batch dims>, <value attention dims>, num_heads, channels) -> (<batch
+        dims>, <query attention dims>, num_heads, channels)
+
+    Args:
+        rank: Rank of query, key, value tensors.
+        attn_axes: List/tuple of axes, `[-1, rank)`,
+            that attention will be applied to.
+
+    Returns:
+        Einsum equations.
+    """
+    target_notation = ""
+    for i in range(rank):
+        target_notation += _index_to_einsum_variable(i)
+    # `batch_dims` includes the head dim.
+    batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1,)))
+    letter_offset = rank
+    source_notation = ""
+    for i in range(rank):
+        if i in batch_dims or i == rank - 1:
+            source_notation += target_notation[i]
+        else:
+            source_notation += _index_to_einsum_variable(letter_offset)
+            letter_offset += 1
+
+    product_notation = "".join(
+        [target_notation[i] for i in batch_dims]
+        + [target_notation[i] for i in attn_axes]
+        + [source_notation[i] for i in attn_axes]
+    )
+    dot_product_equation = "%s,%s->%s" % (
+        source_notation,
+        target_notation,
+        product_notation,
+    )
+    attn_scores_rank = len(product_notation)
+    combine_equation = "%s,%s->%s" % (
+        product_notation,
+        source_notation,
+        target_notation,
+    )
+    return dot_product_equation, combine_equation, attn_scores_rank
+
+
+def _build_proj_equation(free_dims, bound_dims, output_dims):
+    """Builds an einsum equation for projections inside multi-head attention."""
+    input_str = ""
+    kernel_str = ""
+    output_str = ""
+    bias_axes = ""
+    letter_offset = 0
+    for i in range(free_dims):
+        char = _index_to_einsum_variable(i + letter_offset)
+        input_str += char
+        output_str += char
+
+    letter_offset += free_dims
+    for i in range(bound_dims):
+        char = _index_to_einsum_variable(i + letter_offset)
+        input_str += char
+        kernel_str += char
+
+    letter_offset += bound_dims
+    for i in range(output_dims):
+        char = _index_to_einsum_variable(i + letter_offset)
+        kernel_str += char
+        output_str += char
+        bias_axes += char
+    equation = f"{input_str},{kernel_str}->{output_str}"
+
+    return equation, bias_axes, len(output_str)
+
+
+def _get_output_shape(output_rank, known_last_dims):
+    return [None] * (output_rank - len(known_last_dims)) + list(known_last_dims)

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/base_conv.py

@@ -0,0 +1,401 @@
+"""Keras base class for convolution layers."""
+
+from keras.src import activations
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.backend import standardize_data_format
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+from keras.src.ops.operation_utils import compute_conv_output_shape
+from keras.src.utils.argument_validation import standardize_padding
+from keras.src.utils.argument_validation import standardize_tuple
+
+
+class BaseConv(Layer):
+    """Abstract N-D convolution layer (private, used as implementation base).
+
+    This layer creates a convolution kernel that is convolved (actually
+    cross-correlated) with the layer input to produce a tensor of outputs. If
+    `use_bias` is True (and a `bias_initializer` is provided), a bias vector is
+    created and added to the outputs. Finally, if `activation` is not `None`, it
+    is applied to the outputs as well.
+
+    Note: layer attributes cannot be modified after the layer has been called
+    once (except the `trainable` attribute).
+
+    Args:
+        rank: int, the rank of the convolution, e.g. 2 for 2D convolution.
+        filters: int, the dimension of the output space (the number of filters
+            in the convolution).
+        kernel_size: int or tuple/list of `rank` integers, specifying the size
+            of the convolution window.
+        strides: int or tuple/list of `rank` integers, specifying the stride
+            length of the convolution. If only one int is specified, the same
+            stride size will be used for all dimensions. `strides > 1` is
+            incompatible with `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of `rank` integers, specifying the
+            dilation rate to use for dilated convolution. If only one int is
+            specified, the same dilation rate will be used for all dimensions.
+        groups: A positive int specifying the number of groups in which the
+            input is split along the channel axis. Each group is convolved
+            separately with `filters // groups` filters. The output is the
+            concatenation of all the `groups` results along the channel axis.
+            Input channels and `filters` must both be divisible by `groups`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+        lora_rank: Optional integer. If set, the layer's forward pass
+            will implement LoRA (Low-Rank Adaptation)
+            with the provided rank. LoRA sets the layer's kernel
+            to non-trainable and replaces it with a delta over the
+            original kernel, obtained via multiplying two lower-rank
+            trainable matrices. This can be useful to reduce the
+            computation cost of fine-tuning large dense layers.
+            You can also enable LoRA on an existing layer by calling
+            `layer.enable_lora(rank)`.
+    """
+
+    def __init__(
+        self,
+        rank,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        groups=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        lora_rank=None,
+        **kwargs,
+    ):
+        super().__init__(activity_regularizer=activity_regularizer, **kwargs)
+        self.rank = rank
+        self.filters = filters
+        self.groups = groups
+        self.kernel_size = standardize_tuple(kernel_size, rank, "kernel_size")
+        self.strides = standardize_tuple(strides, rank, "strides")
+        self.dilation_rate = standardize_tuple(
+            dilation_rate, rank, "dilation_rate"
+        )
+        self.padding = standardize_padding(padding, allow_causal=rank == 1)
+        self.data_format = standardize_data_format(data_format)
+        self.activation = activations.get(activation)
+        self.use_bias = use_bias
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.bias_initializer = initializers.get(bias_initializer)
+        self.kernel_regularizer = regularizers.get(kernel_regularizer)
+        self.bias_regularizer = regularizers.get(bias_regularizer)
+        self.kernel_constraint = constraints.get(kernel_constraint)
+        self.bias_constraint = constraints.get(bias_constraint)
+        self.lora_rank = lora_rank
+        self.lora_enabled = False
+        self.input_spec = InputSpec(min_ndim=self.rank + 2)
+        self.data_format = self.data_format
+
+        if self.filters is not None and self.filters <= 0:
+            raise ValueError(
+                "Invalid value for argument `filters`. Expected a strictly "
+                f"positive value. Received filters={self.filters}."
+            )
+
+        if self.groups <= 0:
+            raise ValueError(
+                "The number of groups must be a positive integer. "
+                f"Received: groups={self.groups}."
+            )
+
+        if self.filters is not None and self.filters % self.groups != 0:
+            raise ValueError(
+                "The number of filters must be evenly divisible by the "
+                f"number of groups. Received: groups={self.groups}, "
+                f"filters={self.filters}."
+            )
+
+        if not all(self.kernel_size):
+            raise ValueError(
+                "The argument `kernel_size` cannot contain 0. Received "
+                f"kernel_size={self.kernel_size}."
+            )
+
+        if not all(self.strides):
+            raise ValueError(
+                "The argument `strides` cannot contains 0. Received "
+                f"strides={self.strides}"
+            )
+
+        if max(self.strides) > 1 and max(self.dilation_rate) > 1:
+            raise ValueError(
+                "`strides > 1` not supported in conjunction with "
+                f"`dilation_rate > 1`. Received: strides={self.strides} and "
+                f"dilation_rate={self.dilation_rate}"
+            )
+
+    def build(self, input_shape):
+        if self.data_format == "channels_last":
+            channel_axis = -1
+            input_channel = input_shape[-1]
+        else:
+            channel_axis = 1
+            input_channel = input_shape[1]
+        self.input_spec = InputSpec(
+            min_ndim=self.rank + 2, axes={channel_axis: input_channel}
+        )
+        if input_channel % self.groups != 0:
+            raise ValueError(
+                "The number of input channels must be evenly divisible by "
+                f"the number of groups. Received groups={self.groups}, but the "
+                f"input has {input_channel} channels (full input shape is "
+                f"{input_shape})."
+            )
+        kernel_shape = self.kernel_size + (
+            input_channel // self.groups,
+            self.filters,
+        )
+
+        # compute_output_shape contains some validation logic for the input
+        # shape, and make sure the output shape has all positive dimensions.
+        self.compute_output_shape(input_shape)
+
+        self._kernel = self.add_weight(
+            name="kernel",
+            shape=kernel_shape,
+            initializer=self.kernel_initializer,
+            regularizer=self.kernel_regularizer,
+            constraint=self.kernel_constraint,
+            trainable=True,
+            dtype=self.dtype,
+        )
+        if self.use_bias:
+            self.bias = self.add_weight(
+                name="bias",
+                shape=(self.filters,),
+                initializer=self.bias_initializer,
+                regularizer=self.bias_regularizer,
+                constraint=self.bias_constraint,
+                trainable=True,
+                dtype=self.dtype,
+            )
+        else:
+            self.bias = None
+        self.built = True
+        if self.lora_rank:
+            self.enable_lora(self.lora_rank)
+
+    @property
+    def kernel(self):
+        if not self.built:
+            raise AttributeError(
+                "You must build the layer before accessing `kernel`."
+            )
+        if self.lora_enabled:
+            return self._kernel + ops.matmul(
+                self.lora_kernel_a, self.lora_kernel_b
+            )
+        return self._kernel
+
+    def convolution_op(self, inputs, kernel):
+        return ops.conv(
+            inputs,
+            kernel,
+            strides=list(self.strides),
+            padding=self.padding,
+            dilation_rate=self.dilation_rate,
+            data_format=self.data_format,
+        )
+
+    def call(self, inputs):
+        outputs = self.convolution_op(
+            inputs,
+            self.kernel,
+        )
+        if self.use_bias:
+            if self.data_format == "channels_last":
+                bias_shape = (1,) * (self.rank + 1) + (self.filters,)
+            else:
+                bias_shape = (1, self.filters) + (1,) * self.rank
+            bias = ops.reshape(self.bias, bias_shape)
+            outputs = ops.add(outputs, bias)
+
+        if self.activation is not None:
+            return self.activation(outputs)
+        return outputs
+
+    def compute_output_shape(self, input_shape):
+        return compute_conv_output_shape(
+            input_shape,
+            self.filters,
+            self.kernel_size,
+            strides=self.strides,
+            padding=self.padding,
+            data_format=self.data_format,
+            dilation_rate=self.dilation_rate,
+        )
+
+    def enable_lora(
+        self, rank, a_initializer="he_uniform", b_initializer="zeros"
+    ):
+        if self.kernel_constraint:
+            raise ValueError(
+                "Lora is incompatible with kernel constraints. "
+                "In order to enable lora on this layer, remove the "
+                "`kernel_constraint` argument."
+            )
+        if not self.built:
+            raise ValueError(
+                "Cannot enable lora on a layer that isn't yet built."
+            )
+        if self.lora_enabled:
+            raise ValueError(
+                "lora is already enabled. "
+                "This can only be done once per layer."
+            )
+        self._tracker.unlock()
+        self.lora_kernel_a = self.add_weight(
+            name="lora_kernel_a",
+            shape=self._kernel.shape[:-1] + (rank,),
+            initializer=initializers.get(a_initializer),
+            regularizer=self.kernel_regularizer,
+        )
+        self.lora_kernel_b = self.add_weight(
+            name="lora_kernel_b",
+            shape=(rank, self.filters),
+            initializer=initializers.get(b_initializer),
+            regularizer=self.kernel_regularizer,
+        )
+        self._kernel.trainable = False
+        self._tracker.lock()
+        self.lora_enabled = True
+        self.lora_rank = rank
+
+    def save_own_variables(self, store):
+        # Do nothing if the layer isn't yet built
+        if not self.built:
+            return
+        target_variables = [self.kernel]
+        if self.use_bias:
+            target_variables.append(self.bias)
+        for i, variable in enumerate(target_variables):
+            store[str(i)] = variable
+
+    def load_own_variables(self, store):
+        if not self.lora_enabled:
+            self._check_load_own_variables(store)
+        # Do nothing if the layer isn't yet built
+        if not self.built:
+            return
+        target_variables = [self._kernel]
+        if self.use_bias:
+            target_variables.append(self.bias)
+        for i, variable in enumerate(target_variables):
+            variable.assign(store[str(i)])
+        if self.lora_enabled:
+            self.lora_kernel_a.assign(ops.zeros(self.lora_kernel_a.shape))
+            self.lora_kernel_b.assign(ops.zeros(self.lora_kernel_b.shape))
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "filters": self.filters,
+                "kernel_size": self.kernel_size,
+                "strides": self.strides,
+                "padding": self.padding,
+                "data_format": self.data_format,
+                "dilation_rate": self.dilation_rate,
+                "groups": self.groups,
+                "activation": activations.serialize(self.activation),
+                "use_bias": self.use_bias,
+                "kernel_initializer": initializers.serialize(
+                    self.kernel_initializer
+                ),
+                "bias_initializer": initializers.serialize(
+                    self.bias_initializer
+                ),
+                "kernel_regularizer": regularizers.serialize(
+                    self.kernel_regularizer
+                ),
+                "bias_regularizer": regularizers.serialize(
+                    self.bias_regularizer
+                ),
+                "activity_regularizer": regularizers.serialize(
+                    self.activity_regularizer
+                ),
+                "kernel_constraint": constraints.serialize(
+                    self.kernel_constraint
+                ),
+                "bias_constraint": constraints.serialize(self.bias_constraint),
+            }
+        )
+        if self.lora_rank:
+            config["lora_rank"] = self.lora_rank
+        return config
+
+    def _check_load_own_variables(self, store):
+        all_vars = self._trainable_variables + self._non_trainable_variables
+        if len(store.keys()) != len(all_vars):
+            if len(all_vars) == 0 and not self.built:
+                raise ValueError(
+                    f"Layer '{self.name}' was never built "
+                    "and thus it doesn't have any variables. "
+                    f"However the weights file lists {len(store.keys())} "
+                    "variables for this layer.\n"
+                    "In most cases, this error indicates that either:\n\n"
+                    "1. The layer is owned by a parent layer that "
+                    "implements a `build()` method, but calling the "
+                    "parent's `build()` method did NOT create the state of "
+                    f"the child layer '{self.name}'. A `build()` method "
+                    "must create ALL state for the layer, including "
+                    "the state of any children layers.\n\n"
+                    "2. You need to implement "
+                    "the `def build_from_config(self, config)` method "
+                    f"on layer '{self.name}', to specify how to rebuild "
+                    "it during loading. "
+                    "In this case, you might also want to implement the "
+                    "method that generates the build config at saving time, "
+                    "`def get_build_config(self)`. "
+                    "The method `build_from_config()` is meant "
+                    "to create the state "
+                    "of the layer (i.e. its variables) upon deserialization.",
+                )
+            raise ValueError(
+                f"Layer '{self.name}' expected {len(all_vars)} variables, "
+                "but received "
+                f"{len(store.keys())} variables during loading. "
+                f"Expected: {[v.name for v in all_vars]}"
+            )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/base_conv_transpose.py

@@ -0,0 +1,259 @@
+"""Keras base class for transpose convolution layers."""
+
+from keras.src import activations
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.backend import standardize_data_format
+from keras.src.backend.common.backend_utils import (
+    compute_conv_transpose_output_shape,
+)
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+from keras.src.utils.argument_validation import standardize_padding
+from keras.src.utils.argument_validation import standardize_tuple
+
+
+class BaseConvTranspose(Layer):
+    """Abstract N-D transposed convolution layer.
+
+    The need for transposed convolutions generally arises from the desire to use
+    a transformation going in the opposite direction of a normal convolution,
+    i.e., from something that has the shape of the output of some convolution to
+    something that has the shape of its input while maintaining a connectivity
+    pattern that is compatible with said convolution.
+
+    Args:
+        rank: int, the rank of the transposed convolution, e.g. 2 for 2D
+            transposed convolution.
+        filters: int, the dimension of the output space (the number of filters
+            in the transposed convolution).
+        kernel_size: int or tuple/list of `rank` integers, specifying the size
+            of the transposed convolution window.
+        strides: int or tuple/list of `rank` integers, specifying the stride
+            length of the transposed convolution. If only one int is specified,
+            the same stride size will be used for all dimensions.
+            `strides > 1` is incompatible with `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input such that output has the same
+            height/width dimension as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of `rank` integers, specifying the
+            dilation rate to use for dilated convolution. If only one int is
+            specified, the same dilation rate will be used for all dimensions.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+    """
+
+    def __init__(
+        self,
+        rank,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        output_padding=None,
+        data_format=None,
+        dilation_rate=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        trainable=True,
+        name=None,
+        **kwargs,
+    ):
+        super().__init__(
+            trainable=trainable,
+            name=name,
+            activity_regularizer=activity_regularizer,
+            **kwargs,
+        )
+        self.rank = rank
+        self.filters = filters
+        self.kernel_size = standardize_tuple(kernel_size, rank, "kernel_size")
+        self.strides = standardize_tuple(strides, rank, "strides")
+        self.dilation_rate = standardize_tuple(
+            dilation_rate, rank, "dilation_rate"
+        )
+        self.padding = standardize_padding(padding)
+        if output_padding is None:
+            self.output_padding = None
+        else:
+            self.output_padding = standardize_tuple(
+                output_padding,
+                rank,
+                "output_padding",
+            )
+        self.data_format = standardize_data_format(data_format)
+        self.activation = activations.get(activation)
+        self.use_bias = use_bias
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.bias_initializer = initializers.get(bias_initializer)
+        self.kernel_regularizer = regularizers.get(kernel_regularizer)
+        self.bias_regularizer = regularizers.get(bias_regularizer)
+        self.kernel_constraint = constraints.get(kernel_constraint)
+        self.bias_constraint = constraints.get(bias_constraint)
+        self.input_spec = InputSpec(min_ndim=self.rank + 2)
+        self.data_format = self.data_format
+
+        if self.filters is not None and self.filters <= 0:
+            raise ValueError(
+                "Invalid value for argument `filters`. Expected a strictly "
+                f"positive value. Received filters={self.filters}."
+            )
+
+        if not all(self.kernel_size):
+            raise ValueError(
+                "The argument `kernel_size` cannot contain 0. Received "
+                f"kernel_size={self.kernel_size}."
+            )
+
+        if not all(self.strides):
+            raise ValueError(
+                "The argument `strides` cannot contains 0. Received "
+                f"strides={self.strides}."
+            )
+
+        if max(self.strides) > 1 and max(self.dilation_rate) > 1:
+            raise ValueError(
+                "`strides > 1` not supported in conjunction with "
+                f"`dilation_rate > 1`. Received: strides={self.strides} and "
+                f"dilation_rate={self.dilation_rate}"
+            )
+
+    def build(self, input_shape):
+        if self.data_format == "channels_last":
+            channel_axis = -1
+            input_channel = input_shape[-1]
+        else:
+            channel_axis = 1
+            input_channel = input_shape[1]
+        self.input_spec = InputSpec(
+            min_ndim=self.rank + 2, axes={channel_axis: input_channel}
+        )
+        kernel_shape = self.kernel_size + (
+            self.filters,
+            input_channel,
+        )
+
+        self.kernel = self.add_weight(
+            name="kernel",
+            shape=kernel_shape,
+            initializer=self.kernel_initializer,
+            regularizer=self.kernel_regularizer,
+            constraint=self.kernel_constraint,
+            trainable=True,
+            dtype=self.dtype,
+        )
+        if self.use_bias:
+            self.bias = self.add_weight(
+                name="bias",
+                shape=(self.filters,),
+                initializer=self.bias_initializer,
+                regularizer=self.bias_regularizer,
+                constraint=self.bias_constraint,
+                trainable=True,
+                dtype=self.dtype,
+            )
+        else:
+            self.bias = None
+        self.built = True
+
+    def call(self, inputs):
+        outputs = ops.conv_transpose(
+            inputs,
+            self.kernel,
+            strides=list(self.strides),
+            padding=self.padding,
+            output_padding=self.output_padding,
+            dilation_rate=self.dilation_rate,
+            data_format=self.data_format,
+        )
+
+        if self.use_bias:
+            if self.data_format == "channels_last":
+                bias_shape = (1,) * (self.rank + 1) + (self.filters,)
+            else:
+                bias_shape = (1, self.filters) + (1,) * self.rank
+            bias = ops.reshape(self.bias, bias_shape)
+            outputs = ops.add(outputs, bias)
+
+        if self.activation is not None:
+            return self.activation(outputs)
+        return outputs
+
+    def compute_output_shape(self, input_shape):
+        return compute_conv_transpose_output_shape(
+            input_shape,
+            self.kernel_size,
+            self.filters,
+            strides=self.strides,
+            padding=self.padding,
+            output_padding=self.output_padding,
+            data_format=self.data_format,
+            dilation_rate=self.dilation_rate,
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "filters": self.filters,
+                "kernel_size": self.kernel_size,
+                "strides": self.strides,
+                "padding": self.padding,
+                "data_format": self.data_format,
+                "dilation_rate": self.dilation_rate,
+                "activation": activations.serialize(self.activation),
+                "use_bias": self.use_bias,
+                "kernel_initializer": initializers.serialize(
+                    self.kernel_initializer
+                ),
+                "bias_initializer": initializers.serialize(
+                    self.bias_initializer
+                ),
+                "kernel_regularizer": regularizers.serialize(
+                    self.kernel_regularizer
+                ),
+                "bias_regularizer": regularizers.serialize(
+                    self.bias_regularizer
+                ),
+                "activity_regularizer": regularizers.serialize(
+                    self.activity_regularizer
+                ),
+                "kernel_constraint": constraints.serialize(
+                    self.kernel_constraint
+                ),
+                "bias_constraint": constraints.serialize(self.bias_constraint),
+            }
+        )
+        return config

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/base_depthwise_conv.py

@@ -0,0 +1,274 @@
+"""Keras base class for depthwise convolution layers."""
+
+from keras.src import activations
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.backend import standardize_data_format
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+from keras.src.ops.operation_utils import compute_conv_output_shape
+from keras.src.utils.argument_validation import standardize_padding
+from keras.src.utils.argument_validation import standardize_tuple
+
+
+class BaseDepthwiseConv(Layer):
+    """Abstract N-D depthwise convolution layer.
+
+    Depthwise convolution is a type of convolution in which each input channel
+    is convolved with a different kernel (called a depthwise kernel). You can
+    understand depthwise convolution as the first step in a depthwise separable
+    convolution.
+
+    It is implemented via the following steps:
+
+    - Split the input into individual channels.
+    - Convolve each channel with an individual depthwise kernel with
+      `depth_multiplier` output channels.
+    - Concatenate the convolved outputs along the channels axis.
+
+    Unlike a regular convolution, depthwise convolution does not mix information
+    across different input channels.
+
+    The `depth_multiplier` argument determines how many filter are applied to
+    one input channel. As such, it controls the amount of output channels that
+    are generated per input channel in the depthwise step.
+
+
+    Args:
+        rank: int, the rank of the convolution, e.g. 2 for 2D convolution.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        kernel_size: int or tuple/list of `rank` integers, specifying the size
+            of the depthwise convolution window.
+        strides: int or tuple/list of `rank` integers, specifying the stride
+            length of the depthwise convolution. If only one int is specified,
+            the same stride size will be used for all dimensions.
+            `strides > 1` is incompatible with `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of `rank` integers, specifying the
+            dilation rate to use for dilated convolution. If only one int is
+            specified, the same dilation rate will be used for all dimensions.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: Initializer for the depthwsie convolution
+            kernel. If `None`, the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        depthwise_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+    """
+
+    def __init__(
+        self,
+        rank,
+        depth_multiplier,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        bias_constraint=None,
+        trainable=True,
+        name=None,
+        **kwargs,
+    ):
+        super().__init__(
+            trainable=trainable,
+            name=name,
+            activity_regularizer=regularizers.get(activity_regularizer),
+            **kwargs,
+        )
+        self.rank = rank
+        self.depth_multiplier = depth_multiplier
+        self.kernel_size = standardize_tuple(kernel_size, rank, "kernel_size")
+        self.strides = standardize_tuple(strides, rank, "strides")
+        self.dilation_rate = standardize_tuple(
+            dilation_rate, rank, "dilation_rate"
+        )
+        self.padding = standardize_padding(padding)
+        self.data_format = standardize_data_format(data_format)
+        self.activation = activations.get(activation)
+        self.use_bias = use_bias
+        self.depthwise_initializer = initializers.get(depthwise_initializer)
+        self.bias_initializer = initializers.get(bias_initializer)
+        self.depthwise_regularizer = regularizers.get(depthwise_regularizer)
+        self.bias_regularizer = regularizers.get(bias_regularizer)
+        self.depthwise_constraint = constraints.get(depthwise_constraint)
+        self.bias_constraint = constraints.get(bias_constraint)
+        self.input_spec = InputSpec(min_ndim=self.rank + 2)
+        self.data_format = self.data_format
+
+        if self.depth_multiplier is not None and self.depth_multiplier <= 0:
+            raise ValueError(
+                "Invalid value for argument `depth_multiplier`. Expected a "
+                "strictly positive value. Received "
+                f"depth_multiplier={self.depth_multiplier}."
+            )
+
+        if not all(self.kernel_size):
+            raise ValueError(
+                "The argument `kernel_size` cannot contain 0. Received "
+                f"kernel_size={self.kernel_size}."
+            )
+
+        if not all(self.strides):
+            raise ValueError(
+                "The argument `strides` cannot contains 0. Received "
+                f"strides={self.strides}"
+            )
+
+        if max(self.strides) > 1 and max(self.dilation_rate) > 1:
+            raise ValueError(
+                "`strides > 1` not supported in conjunction with "
+                f"`dilation_rate > 1`. Received: strides={self.strides} and "
+                f"dilation_rate={self.dilation_rate}"
+            )
+
+    def build(self, input_shape):
+        if self.data_format == "channels_last":
+            channel_axis = -1
+            input_channel = input_shape[-1]
+        else:
+            channel_axis = 1
+            input_channel = input_shape[1]
+        self.input_spec = InputSpec(
+            min_ndim=self.rank + 2, axes={channel_axis: input_channel}
+        )
+        depthwise_shape = self.kernel_size + (
+            input_channel,
+            self.depth_multiplier,
+        )
+        self.kernel = self.add_weight(
+            name="kernel",
+            shape=depthwise_shape,
+            initializer=self.depthwise_initializer,
+            regularizer=self.depthwise_regularizer,
+            constraint=self.depthwise_constraint,
+            trainable=True,
+            dtype=self.dtype,
+        )
+        if self.use_bias:
+            self.bias = self.add_weight(
+                name="bias",
+                shape=(self.depth_multiplier * input_channel,),
+                initializer=self.bias_initializer,
+                regularizer=self.bias_regularizer,
+                constraint=self.bias_constraint,
+                trainable=True,
+                dtype=self.dtype,
+            )
+        else:
+            self.bias = None
+        self.built = True
+
+    def _get_input_channel(self, input_shape):
+        if self.data_format == "channels_last":
+            input_channel = input_shape[-1]
+        else:
+            input_channel = input_shape[1]
+        return input_channel
+
+    def call(self, inputs):
+        input_channel = self._get_input_channel(inputs.shape)
+        outputs = ops.depthwise_conv(
+            inputs,
+            self.kernel,
+            strides=self.strides,
+            padding=self.padding,
+            dilation_rate=self.dilation_rate,
+            data_format=self.data_format,
+        )
+
+        if self.use_bias:
+            if self.data_format == "channels_last":
+                bias_shape = (1,) * (self.rank + 1) + (
+                    self.depth_multiplier * input_channel,
+                )
+            else:
+                bias_shape = (1, self.depth_multiplier * input_channel) + (
+                    1,
+                ) * self.rank
+            bias = ops.reshape(self.bias, bias_shape)
+            outputs = ops.add(outputs, bias)
+
+        if self.activation is not None:
+            return self.activation(outputs)
+        return outputs
+
+    def compute_output_shape(self, input_shape):
+        input_channel = self._get_input_channel(input_shape)
+        return compute_conv_output_shape(
+            input_shape,
+            self.depth_multiplier * input_channel,
+            self.kernel_size,
+            strides=self.strides,
+            padding=self.padding,
+            data_format=self.data_format,
+            dilation_rate=self.dilation_rate,
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "depth_multiplier": self.depth_multiplier,
+                "kernel_size": self.kernel_size,
+                "strides": self.strides,
+                "padding": self.padding,
+                "data_format": self.data_format,
+                "dilation_rate": self.dilation_rate,
+                "activation": activations.serialize(self.activation),
+                "use_bias": self.use_bias,
+                "depthwise_initializer": initializers.serialize(
+                    self.depthwise_initializer
+                ),
+                "bias_initializer": initializers.serialize(
+                    self.bias_initializer
+                ),
+                "depthwise_regularizer": regularizers.serialize(
+                    self.depthwise_regularizer
+                ),
+                "bias_regularizer": regularizers.serialize(
+                    self.bias_regularizer
+                ),
+                "activity_regularizer": regularizers.serialize(
+                    self.activity_regularizer
+                ),
+                "depthwise_constraint": constraints.serialize(
+                    self.depthwise_constraint
+                ),
+                "bias_constraint": constraints.serialize(self.bias_constraint),
+            }
+        )
+        return config

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/base_separable_conv.py

@@ -0,0 +1,295 @@
+"""Keras abstract base layer for separable convolution."""
+
+from keras.src import activations
+from keras.src import constraints
+from keras.src import initializers
+from keras.src import ops
+from keras.src import regularizers
+from keras.src.backend import standardize_data_format
+from keras.src.layers.input_spec import InputSpec
+from keras.src.layers.layer import Layer
+from keras.src.ops.operation_utils import compute_conv_output_shape
+from keras.src.utils.argument_validation import standardize_padding
+from keras.src.utils.argument_validation import standardize_tuple
+
+
+class BaseSeparableConv(Layer):
+    """Abstract base layer for separable convolution.
+
+    This layer performs a depthwise convolution that acts separately on
+    channels, followed by a pointwise convolution that mixes channels. If
+    `use_bias` is True and a bias initializer is provided, it adds a bias vector
+    to the output.
+
+    Args:
+        rank: int, the rank of the convolution, e.g. 2 for 2D convolution.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        filters: int, the dimensionality of the output space (i.e. the number
+            of filters in the pointwise convolution).
+        kernel_size: int or tuple/list of `rank` integers, specifying the size
+            of the depthwise convolution window.
+        strides: int or tuple/list of `rank` integers, specifying the stride
+            length of the depthwise convolution. If only one int is specified,
+            the same stride size will be used for all dimensions.
+            `stride value != 1` is incompatible with `dilation_rate != 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of `rank` integers, specifying the
+            dilation rate to use for dilated convolution. If only one int is
+            specified, the same dilation rate will be used for all dimensions.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: An initializer for the depthwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        pointwise_initializer: An initializer for the pointwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: An initializer for the bias vector. If None, the
+            default initializer ('"zeros"') will be used.
+        depthwise_regularizer: Optional regularizer for the depthwise
+            convolution kernel.
+        pointwise_regularizer: Optional regularizer for the pointwise
+            convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            depthwise kernel after being updated by an `Optimizer` (e.g. used
+            for norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape).
+        pointwise_constraint: Optional projection function to be applied to the
+            pointwise kernel after being updated by an `Optimizer`.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+    """
+
+    def __init__(
+        self,
+        rank,
+        depth_multiplier,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        pointwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        pointwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        pointwise_constraint=None,
+        bias_constraint=None,
+        trainable=True,
+        name=None,
+        **kwargs,
+    ):
+        super().__init__(
+            trainable=trainable,
+            name=name,
+            activity_regularizer=regularizers.get(activity_regularizer),
+            **kwargs,
+        )
+        self.rank = rank
+        self.depth_multiplier = depth_multiplier
+        self.filters = filters
+        self.kernel_size = standardize_tuple(kernel_size, rank, "kernel_size")
+        self.strides = standardize_tuple(strides, rank, "strides")
+        self.dilation_rate = standardize_tuple(
+            dilation_rate, rank, "dilation_rate"
+        )
+        self.padding = standardize_padding(padding)
+        self.data_format = standardize_data_format(data_format)
+        self.activation = activations.get(activation)
+        self.use_bias = use_bias
+        self.depthwise_initializer = initializers.get(depthwise_initializer)
+        self.pointwise_initializer = initializers.get(pointwise_initializer)
+        self.bias_initializer = initializers.get(bias_initializer)
+        self.depthwise_regularizer = regularizers.get(depthwise_regularizer)
+        self.pointwise_regularizer = regularizers.get(pointwise_regularizer)
+        self.bias_regularizer = regularizers.get(bias_regularizer)
+        self.depthwise_constraint = constraints.get(depthwise_constraint)
+        self.pointwise_constraint = constraints.get(pointwise_constraint)
+        self.bias_constraint = constraints.get(bias_constraint)
+        self.data_format = self.data_format
+
+        self.input_spec = InputSpec(min_ndim=self.rank + 2)
+
+        if self.depth_multiplier is not None and self.depth_multiplier <= 0:
+            raise ValueError(
+                "Invalid value for argument `depth_multiplier`. Expected a "
+                "strictly positive value. Received "
+                f"depth_multiplier={self.depth_multiplier}."
+            )
+
+        if self.filters is not None and self.filters <= 0:
+            raise ValueError(
+                "Invalid value for argument `filters`. Expected a strictly "
+                f"positive value. Received filters={self.filters}."
+            )
+
+        if not all(self.kernel_size):
+            raise ValueError(
+                "The argument `kernel_size` cannot contain 0. Received: "
+                f"kernel_size={self.kernel_size}."
+            )
+
+        if not all(self.strides):
+            raise ValueError(
+                "The argument `strides` cannot contains 0(s). Received: "
+                f"strides={self.strides}"
+            )
+
+        if max(self.strides) > 1 and max(self.dilation_rate) > 1:
+            raise ValueError(
+                "`strides > 1` not supported in conjunction with "
+                f"`dilation_rate > 1`. Received: strides={self.strides} and "
+                f"dilation_rate={self.dilation_rate}"
+            )
+
+    def build(self, input_shape):
+        if self.data_format == "channels_last":
+            channel_axis = -1
+            input_channel = input_shape[-1]
+        else:
+            channel_axis = 1
+            input_channel = input_shape[1]
+        self.input_spec = InputSpec(
+            min_ndim=self.rank + 2, axes={channel_axis: input_channel}
+        )
+        depthwise_kernel_shape = self.kernel_size + (
+            input_channel,
+            self.depth_multiplier,
+        )
+        pointwise_kernel_shape = (1,) * self.rank + (
+            self.depth_multiplier * input_channel,
+            self.filters,
+        )
+
+        self.depthwise_kernel = self.add_weight(
+            name="depthwise_kernel",
+            shape=depthwise_kernel_shape,
+            initializer=self.depthwise_initializer,
+            regularizer=self.depthwise_regularizer,
+            constraint=self.depthwise_constraint,
+            trainable=True,
+            dtype=self.dtype,
+        )
+        self.pointwise_kernel = self.add_weight(
+            name="pointwise_kernel",
+            shape=pointwise_kernel_shape,
+            initializer=self.pointwise_initializer,
+            regularizer=self.pointwise_regularizer,
+            constraint=self.pointwise_constraint,
+            trainable=True,
+            dtype=self.dtype,
+        )
+        if self.use_bias:
+            self.bias = self.add_weight(
+                name="bias",
+                shape=(self.filters,),
+                initializer=self.bias_initializer,
+                regularizer=self.bias_regularizer,
+                constraint=self.bias_constraint,
+                trainable=True,
+                dtype=self.dtype,
+            )
+        else:
+            self.bias = None
+        self.built = True
+
+    def call(self, inputs):
+        outputs = ops.separable_conv(
+            inputs,
+            self.depthwise_kernel,
+            self.pointwise_kernel,
+            strides=self.strides,
+            padding=self.padding,
+            dilation_rate=self.dilation_rate,
+            data_format=self.data_format,
+        )
+
+        if self.use_bias:
+            if self.data_format == "channels_last":
+                bias_shape = (1,) * (self.rank + 1) + (self.filters,)
+            else:
+                bias_shape = (1, self.filters) + (1,) * self.rank
+            bias = ops.reshape(self.bias, bias_shape)
+            outputs = ops.add(outputs, bias)
+
+        if self.activation is not None:
+            return self.activation(outputs)
+        return outputs
+
+    def compute_output_shape(self, input_shape):
+        return compute_conv_output_shape(
+            input_shape,
+            self.filters,
+            self.kernel_size,
+            strides=self.strides,
+            padding=self.padding,
+            data_format=self.data_format,
+            dilation_rate=self.dilation_rate,
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "depth_multiplier": self.depth_multiplier,
+                "filters": self.filters,
+                "kernel_size": self.kernel_size,
+                "strides": self.strides,
+                "padding": self.padding,
+                "data_format": self.data_format,
+                "dilation_rate": self.dilation_rate,
+                "activation": activations.serialize(self.activation),
+                "use_bias": self.use_bias,
+                "depthwise_initializer": initializers.serialize(
+                    self.depthwise_initializer
+                ),
+                "pointwise_initializer": initializers.serialize(
+                    self.pointwise_initializer
+                ),
+                "bias_initializer": initializers.serialize(
+                    self.bias_initializer
+                ),
+                "depthwise_regularizer": regularizers.serialize(
+                    self.depthwise_regularizer
+                ),
+                "pointwise_regularizer": regularizers.serialize(
+                    self.pointwise_regularizer
+                ),
+                "bias_regularizer": regularizers.serialize(
+                    self.bias_regularizer
+                ),
+                "activity_regularizer": regularizers.serialize(
+                    self.activity_regularizer
+                ),
+                "depthwise_constraint": constraints.serialize(
+                    self.depthwise_constraint
+                ),
+                "pointwise_constraint": constraints.serialize(
+                    self.pointwise_constraint
+                ),
+                "bias_constraint": constraints.serialize(self.bias_constraint),
+            }
+        )
+        return config

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv1d.py

@@ -0,0 +1,170 @@
+from keras.src import ops
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv import BaseConv
+
+
+@keras_export(["keras.layers.Conv1D", "keras.layers.Convolution1D"])
+class Conv1D(BaseConv):
+    """1D convolution layer (e.g. temporal convolution).
+
+    This layer creates a convolution kernel that is convolved with the layer
+    input over a single spatial (or temporal) dimension to produce a tensor of
+    outputs. If `use_bias` is True, a bias vector is created and added to the
+    outputs. Finally, if `activation` is not `None`, it is applied to the
+    outputs as well.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the convolution).
+        kernel_size: int or tuple/list of 1 integer, specifying the size of the
+            convolution window.
+        strides: int or tuple/list of 1 integer, specifying the stride length
+            of the convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, `"valid"`, `"same"` or `"causal"`(case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+            `"causal"` results in causal(dilated) convolutions, e.g. `output[t]`
+            does not depend on`input[t+1:]`. Useful when modeling temporal data
+            where the model should not violate the temporal order.
+            See [WaveNet: A Generative Model for Raw Audio, section2.1](
+            https://arxiv.org/abs/1609.03499).
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated convolution.
+        groups: A positive int specifying the number of groups in which the
+            input is split along the channel axis. Each group is convolved
+            separately with `filters // groups` filters. The output is the
+            concatenation of all the `groups` results along the channel axis.
+            Input channels and `filters` must both be divisible by `groups`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, steps, channels)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, channels, steps)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, new_steps, filters)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, filters, new_steps)`
+
+    Returns:
+        A 3D tensor representing `activation(conv1d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    Example:
+
+    >>> # The inputs are 128-length vectors with 10 timesteps, and the
+    >>> # batch size is 4.
+    >>> x = np.random.rand(4, 10, 128)
+    >>> y = keras.layers.Conv1D(32, 3, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 8, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        groups=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=1,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            groups=groups,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )
+
+    def _compute_causal_padding(self):
+        left_pad = self.dilation_rate[0] * (self.kernel_size[0] - 1)
+        if self.data_format == "channels_last":
+            causal_padding = [[0, 0], [left_pad, 0], [0, 0]]
+        else:
+            causal_padding = [[0, 0], [0, 0], [left_pad, 0]]
+        return causal_padding
+
+    def call(self, inputs):
+        padding = self.padding
+        if self.padding == "causal":
+            # Apply causal padding to inputs.
+            inputs = ops.pad(inputs, self._compute_causal_padding())
+            padding = "valid"
+
+        outputs = ops.conv(
+            inputs,
+            self.kernel,
+            strides=list(self.strides),
+            padding=padding,
+            dilation_rate=self.dilation_rate,
+            data_format=self.data_format,
+        )
+
+        if self.use_bias:
+            if self.data_format == "channels_last":
+                bias_shape = (1,) * (self.rank + 1) + (self.filters,)
+            else:
+                bias_shape = (1, self.filters) + (1,) * self.rank
+            bias = ops.reshape(self.bias, bias_shape)
+            outputs = ops.add(outputs, bias)
+
+        if self.activation is not None:
+            return self.activation(outputs)
+        return outputs

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv1d_transpose.py

@@ -0,0 +1,131 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv_transpose import BaseConvTranspose
+
+
+@keras_export(
+    [
+        "keras.layers.Conv1DTranspose",
+        "keras.layers.Convolution1DTranspose",
+    ]
+)
+class Conv1DTranspose(BaseConvTranspose):
+    """1D transposed convolution layer.
+
+    The need for transposed convolutions generally arise from the desire to use
+    a transformation going in the opposite direction of a normal convolution,
+    i.e., from something that has the shape of the output of some convolution
+    to something that has the shape of its input while maintaining a
+    connectivity pattern that is compatible with said convolution.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the transpose convolution).
+        kernel_size: int or tuple/list of 1 integer, specifying the size of the
+            transposed convolution window.
+        strides: int or tuple/list of 1 integer, specifying the stride length
+            of the transposed convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input such that output has the same
+            height/width dimension as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated transposed convolution.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, steps, channels)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, channels, steps)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, new_steps, filters)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, filters, new_steps)`
+
+    Returns:
+        A 3D tensor representing
+        `activation(conv1d_transpose(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    References:
+    - [A guide to convolution arithmetic for deep learning](
+        https://arxiv.org/abs/1603.07285v1)
+    - [Deconvolutional Networks](
+        https://www.matthewzeiler.com/mattzeiler/deconvolutionalnetworks.pdf)
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 128)
+    >>> y = keras.layers.Conv1DTranspose(32, 3, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 21, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=1,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv2d.py

@@ -0,0 +1,128 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv import BaseConv
+
+
+@keras_export(["keras.layers.Conv2D", "keras.layers.Convolution2D"])
+class Conv2D(BaseConv):
+    """2D convolution layer.
+
+    This layer creates a convolution kernel that is convolved with the layer
+    input over a 2D spatial (or temporal) dimension (height and width) to
+    produce a tensor of outputs. If `use_bias` is True, a bias vector is created
+    and added to the outputs. Finally, if `activation` is not `None`, it is
+    applied to the outputs as well.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the convolution).
+        kernel_size: int or tuple/list of 2 integer, specifying the size of the
+            convolution window.
+        strides: int or tuple/list of 2 integer, specifying the stride length
+            of the convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape
+            `(batch_size, height, width, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch_size, channels, height, width)`. It defaults to the
+            `image_data_format` value found in your Keras config file at
+            `~/.keras/keras.json`. If you never set it, then it will be
+            `"channels_last"`.
+        dilation_rate: int or tuple/list of 2 integers, specifying the dilation
+            rate to use for dilated convolution.
+        groups: A positive int specifying the number of groups in which the
+            input is split along the channel axis. Each group is convolved
+            separately with `filters // groups` filters. The output is the
+            concatenation of all the `groups` results along the channel axis.
+            Input channels and `filters` must both be divisible by `groups`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, height, width, channels)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, channels, height, width)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, new_height, new_width, filters)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, filters, new_height, new_width)`
+
+    Returns:
+        A 4D tensor representing `activation(conv2d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 10, 128)
+    >>> y = keras.layers.Conv2D(32, 3, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 8, 8, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=(1, 1),
+        padding="valid",
+        data_format=None,
+        dilation_rate=(1, 1),
+        groups=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=2,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            groups=groups,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv2d_transpose.py

@@ -0,0 +1,133 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv_transpose import BaseConvTranspose
+
+
+@keras_export(
+    [
+        "keras.layers.Conv2DTranspose",
+        "keras.layers.Convolution2DTranspose",
+    ]
+)
+class Conv2DTranspose(BaseConvTranspose):
+    """2D transposed convolution layer.
+
+    The need for transposed convolutions generally arise from the desire to use
+    a transformation going in the opposite direction of a normal convolution,
+    i.e., from something that has the shape of the output of some convolution
+    to something that has the shape of its input while maintaining a
+    connectivity pattern that is compatible with said convolution.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the transposed convolution).
+        kernel_size: int or tuple/list of 1 integer, specifying the size of the
+            transposed convolution window.
+        strides: int or tuple/list of 1 integer, specifying the stride length
+            of the transposed convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape
+            `(batch_size, height, width, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch_size, channels, height, width)`. It defaults to the
+            `image_data_format` value found in your Keras config file at
+            `~/.keras/keras.json`. If you never set it, then it will be
+            `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated transposed convolution.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, height, width, channels)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, channels, height, width)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, new_height, new_width, filters)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, filters, new_height, new_width)`
+
+    Returns:
+        A 4D tensor representing
+        `activation(conv2d_transpose(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    References:
+    - [A guide to convolution arithmetic for deep learning](
+        https://arxiv.org/abs/1603.07285v1)
+    - [Deconvolutional Networks](
+        https://www.matthewzeiler.com/mattzeiler/deconvolutionalnetworks.pdf)
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 8, 128)
+    >>> y = keras.layers.Conv2DTranspose(32, 2, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 20, 16, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=(1, 1),
+        padding="valid",
+        data_format=None,
+        dilation_rate=(1, 1),
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=2,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv3d.py

@@ -0,0 +1,134 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv import BaseConv
+
+
+@keras_export(["keras.layers.Conv3D", "keras.layers.Convolution3D"])
+class Conv3D(BaseConv):
+    """3D convolution layer.
+
+    This layer creates a convolution kernel that is convolved with the layer
+    input over a 3D spatial (or temporal) dimension (width,height and depth) to
+    produce a tensor of outputs. If `use_bias` is True, a bias vector is created
+    and added to the outputs. Finally, if `activation` is not `None`, it is
+    applied to the outputs as well.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the convolution).
+        kernel_size: int or tuple/list of 3 integer, specifying the size of the
+            convolution window.
+        strides: int or tuple/list of 3 integer, specifying the stride length
+            of the convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape
+            `(batch_size, spatial_dim1, spatial_dim2, spatial_dim3, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch_size, channels, spatial_dim1, spatial_dim2, spatial_dim3)`.
+            It defaults to the `image_data_format` value found in your Keras
+            config file at `~/.keras/keras.json`. If you never set it, then it
+            will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 3 integers, specifying the dilation
+            rate to use for dilated convolution.
+        groups: A positive int specifying the number of groups in which the
+            input is split along the channel axis. Each group is convolved
+            separately with `filters // groups` filters. The output is the
+            concatenation of all the `groups` results along the channel axis.
+            Input channels and `filters` must both be divisible by `groups`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        5D tensor with shape:
+        `(batch_size, spatial_dim1, spatial_dim2, spatial_dim3, channels)`
+    - If `data_format="channels_first"`:
+        5D tensor with shape:
+        `(batch_size, channels, spatial_dim1, spatial_dim2, spatial_dim3)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        5D tensor with shape:
+        `(batch_size, new_spatial_dim1, new_spatial_dim2, new_spatial_dim3,
+        filters)`
+    - If `data_format="channels_first"`:
+        5D tensor with shape:
+        `(batch_size, filters, new_spatial_dim1, new_spatial_dim2,
+        new_spatial_dim3)`
+
+    Returns:
+        A 5D tensor representing `activation(conv3d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 10, 10, 128)
+    >>> y = keras.layers.Conv3D(32, 3, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 8, 8, 8, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=(1, 1, 1),
+        padding="valid",
+        data_format=None,
+        dilation_rate=(1, 1, 1),
+        groups=1,
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=3,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            groups=groups,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/conv3d_transpose.py

@@ -0,0 +1,138 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_conv_transpose import BaseConvTranspose
+
+
+@keras_export(
+    [
+        "keras.layers.Conv3DTranspose",
+        "keras.layers.Convolution3DTranspose",
+    ]
+)
+class Conv3DTranspose(BaseConvTranspose):
+    """3D transposed convolution layer.
+
+    The need for transposed convolutions generally arise from the desire to use
+    a transformation going in the opposite direction of a normal convolution,
+    i.e., from something that has the shape of the output of some convolution
+    to something that has the shape of its input while maintaining a
+    connectivity pattern that is compatible with said convolution.
+
+    Args:
+        filters: int, the dimension of the output space (the number of filters
+            in the transposed convolution).
+        kernel_size: int or tuple/list of 1 integer, specifying the size of the
+            transposed convolution window.
+        strides: int or tuple/list of 1 integer, specifying the stride length
+            of the transposed convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape
+            `(batch_size, spatial_dim1, spatial_dim2, spatial_dim3, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch_size, channels, spatial_dim1, spatial_dim2, spatial_dim3)`.
+            It defaults to the `image_data_format` value found in your Keras
+            config file at `~/.keras/keras.json`. If you never set it, then it
+            will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated transposed convolution.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        kernel_initializer: Initializer for the convolution kernel. If `None`,
+            the default initializer (`"glorot_uniform"`) will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        kernel_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        kernel_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        5D tensor with shape:
+        `(batch_size, spatial_dim1, spatial_dim2, spatial_dim3, channels)`
+    - If `data_format="channels_first"`:
+        5D tensor with shape:
+        `(batch_size, channels, spatial_dim1, spatial_dim2, spatial_dim3)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        5D tensor with shape:
+        `(batch_size, new_spatial_dim1, new_spatial_dim2, new_spatial_dim3,
+        filters)`
+    - If `data_format="channels_first"`:
+        5D tensor with shape:
+        `(batch_size, filters, new_spatial_dim1, new_spatial_dim2,
+        new_spatial_dim3)`
+
+    Returns:
+        A 5D tensor representing `activation(conv3d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    References:
+    - [A guide to convolution arithmetic for deep learning](
+        https://arxiv.org/abs/1603.07285v1)
+    - [Deconvolutional Networks](
+        https://www.matthewzeiler.com/mattzeiler/deconvolutionalnetworks.pdf)
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 8, 12, 128)
+    >>> y = keras.layers.Conv3DTranspose(32, 2, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 20, 16, 24, 32)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=(1, 1, 1),
+        padding="valid",
+        data_format=None,
+        dilation_rate=(1, 1, 1),
+        activation=None,
+        use_bias=True,
+        kernel_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        kernel_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        kernel_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=3,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            kernel_initializer=kernel_initializer,
+            bias_initializer=bias_initializer,
+            kernel_regularizer=kernel_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            kernel_constraint=kernel_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/depthwise_conv1d.py

@@ -0,0 +1,137 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_depthwise_conv import BaseDepthwiseConv
+
+
+@keras_export("keras.layers.DepthwiseConv1D")
+class DepthwiseConv1D(BaseDepthwiseConv):
+    """1D depthwise convolution layer.
+
+    Depthwise convolution is a type of convolution in which each input channel
+    is convolved with a different kernel (called a depthwise kernel). You can
+    understand depthwise convolution as the first step in a depthwise separable
+    convolution.
+
+    It is implemented via the following steps:
+
+    - Split the input into individual channels.
+    - Convolve each channel with an individual depthwise kernel with
+      `depth_multiplier` output channels.
+    - Concatenate the convolved outputs along the channels axis.
+
+    Unlike a regular 1D convolution, depthwise convolution does not mix
+    information across different input channels.
+
+    The `depth_multiplier` argument determines how many filters are applied to
+    one input channel. As such, it controls the amount of output channels that
+    are generated per input channel in the depthwise step.
+
+    Args:
+        kernel_size: int or tuple/list of 1 integer, specifying the size of the
+            depthwise convolution window.
+        strides: int or tuple/list of 1 integer, specifying the stride length
+            of the convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated convolution.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: Initializer for the convolution kernel.
+            If `None`, the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        depthwise_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, steps, channels)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, channels, steps)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape:
+        `(batch_shape, new_steps, channels * depth_multiplier)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape:
+        `(batch_shape, channels * depth_multiplier, new_steps)`
+
+    Returns:
+        A 3D tensor representing
+        `activation(depthwise_conv1d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 12)
+    >>> y = keras.layers.DepthwiseConv1D(3, 3, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 4, 36)
+    """
+
+    def __init__(
+        self,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        depth_multiplier=1,
+        data_format=None,
+        dilation_rate=1,
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=1,
+            depth_multiplier=depth_multiplier,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            depthwise_initializer=depthwise_initializer,
+            bias_initializer=bias_initializer,
+            depthwise_regularizer=depthwise_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            depthwise_constraint=depthwise_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/depthwise_conv2d.py

@@ -0,0 +1,138 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_depthwise_conv import BaseDepthwiseConv
+
+
+@keras_export("keras.layers.DepthwiseConv2D")
+class DepthwiseConv2D(BaseDepthwiseConv):
+    """2D depthwise convolution layer.
+
+    Depthwise convolution is a type of convolution in which each input channel
+    is convolved with a different kernel (called a depthwise kernel). You can
+    understand depthwise convolution as the first step in a depthwise separable
+    convolution.
+
+    It is implemented via the following steps:
+
+    - Split the input into individual channels.
+    - Convolve each channel with an individual depthwise kernel with
+      `depth_multiplier` output channels.
+    - Concatenate the convolved outputs along the channels axis.
+
+    Unlike a regular 2D convolution, depthwise convolution does not mix
+    information across different input channels.
+
+    The `depth_multiplier` argument determines how many filters are applied to
+    one input channel. As such, it controls the amount of output channels that
+    are generated per input channel in the depthwise step.
+
+    Args:
+        kernel_size: int or tuple/list of 2 integer, specifying the size of the
+            depthwise convolution window.
+        strides: int or tuple/list of 2 integer, specifying the stride length
+            of the depthwise convolution. `strides > 1` is incompatible with
+            `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, height, width, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, channels, height, width)`. It defaults to the
+            `image_data_format` value found in your Keras config file
+            at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 2 integers, specifying the dilation
+            rate to use for dilated convolution.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: Initializer for the convolution kernel.
+            If `None`, the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: Initializer for the bias vector. If `None`, the
+            default initializer (`"zeros"`) will be used.
+        depthwise_regularizer: Optional regularizer for the convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            kernel after being updated by an `Optimizer` (e.g. used to implement
+            norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape). Constraints
+            are not safe to use when doing asynchronous distributed training.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, height, width, channels)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, channels, height, width)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape:
+        `(batch_size, new_height, new_width, channels * depth_multiplier)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape:
+        `(batch_size, channels * depth_multiplier, new_height, new_width)`
+
+    Returns:
+        A 4D tensor representing
+        `activation(depthwise_conv2d(inputs, kernel) + bias)`.
+
+    Raises:
+        ValueError: when both `strides > 1` and `dilation_rate > 1`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 10, 12)
+    >>> y = keras.layers.DepthwiseConv2D(kernel_size=3, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 8, 8, 12)
+    """
+
+    def __init__(
+        self,
+        kernel_size,
+        strides=(1, 1),
+        padding="valid",
+        depth_multiplier=1,
+        data_format=None,
+        dilation_rate=(1, 1),
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=2,
+            depth_multiplier=depth_multiplier,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            depthwise_initializer=depthwise_initializer,
+            bias_initializer=bias_initializer,
+            depthwise_regularizer=depthwise_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            depthwise_constraint=depthwise_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/separable_conv1d.py

@@ -0,0 +1,143 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_separable_conv import BaseSeparableConv
+
+
+@keras_export(
+    [
+        "keras.layers.SeparableConv1D",
+        "keras.layers.SeparableConvolution1D",
+    ]
+)
+class SeparableConv1D(BaseSeparableConv):
+    """1D separable convolution layer.
+
+    This layer performs a depthwise convolution that acts separately on
+    channels, followed by a pointwise convolution that mixes channels.
+    If `use_bias` is True and a bias initializer is provided,
+    it adds a bias vector to the output. It then optionally applies an
+    activation function to produce the final output.
+
+    Args:
+        filters: int, the dimensionality of the output space (i.e. the number
+            of filters in the pointwise convolution).
+        kernel_size: int or tuple/list of 1 integers, specifying the size of the
+            depthwise convolution window.
+        strides: int or tuple/list of 1 integers, specifying the stride length
+            of the depthwise convolution. If only one int is specified, the same
+            stride size will be used for all dimensions. `strides > 1` is
+            incompatible with `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, steps, features)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, features, steps)`. It defaults to the `image_data_format`
+            value found in your Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 1 integers, specifying the dilation
+            rate to use for dilated convolution. If only one int is specified,
+            the same dilation rate will be used for all dimensions.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: An initializer for the depthwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        pointwise_initializer: An initializer for the pointwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: An initializer for the bias vector. If None, the
+            default initializer ('"zeros"') will be used.
+        depthwise_regularizer: Optional regularizer for the depthwise
+            convolution kernel.
+        pointwise_regularizer: Optional regularizer for the pointwise
+            convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            depthwise kernel after being updated by an `Optimizer` (e.g. used
+            for norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape).
+        pointwise_constraint: Optional projection function to be applied to the
+            pointwise kernel after being updated by an `Optimizer`.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, steps, channels)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, channels, steps)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 3D tensor with shape: `(batch_shape, new_steps, filters)`
+    - If `data_format="channels_first"`:
+        A 3D tensor with shape: `(batch_shape, filters, new_steps)`
+
+    Returns:
+        A 3D tensor representing
+        `activation(separable_conv1d(inputs, kernel) + bias)`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 12)
+    >>> y = keras.layers.SeparableConv1D(3, 4, 3, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 4, 4)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=1,
+        padding="valid",
+        data_format=None,
+        dilation_rate=1,
+        depth_multiplier=1,
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        pointwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        pointwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        pointwise_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=1,
+            depth_multiplier=depth_multiplier,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            depthwise_initializer=depthwise_initializer,
+            pointwise_initializer=pointwise_initializer,
+            bias_initializer=bias_initializer,
+            depthwise_regularizer=depthwise_regularizer,
+            pointwise_regularizer=pointwise_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            depthwise_constraint=depthwise_constraint,
+            pointwise_constraint=pointwise_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )

SwarmUI/dlbackend/ComfyUI/venv/lib/python3.10/site-packages/keras/src/layers/convolutional/separable_conv2d.py

@@ -0,0 +1,144 @@
+from keras.src.api_export import keras_export
+from keras.src.layers.convolutional.base_separable_conv import BaseSeparableConv
+
+
+@keras_export(
+    [
+        "keras.layers.SeparableConv2D",
+        "keras.layers.SeparableConvolution2D",
+    ]
+)
+class SeparableConv2D(BaseSeparableConv):
+    """2D separable convolution layer.
+
+    This layer performs a depthwise convolution that acts separately on
+    channels, followed by a pointwise convolution that mixes channels.
+    If `use_bias` is True and a bias initializer is provided,
+    it adds a bias vector to the output. It then optionally applies an
+    activation function to produce the final output.
+
+    Args:
+        filters: int, the dimensionality of the output space (i.e. the number
+            of filters in the pointwise convolution).
+        kernel_size: int or tuple/list of 2 integers, specifying the size of the
+            depthwise convolution window.
+        strides: int or tuple/list of 2 integers, specifying the stride length
+            of the depthwise convolution. If only one int is specified, the same
+            stride size will be used for all dimensions. `strides > 1` is
+            incompatible with `dilation_rate > 1`.
+        padding: string, either `"valid"` or `"same"` (case-insensitive).
+            `"valid"` means no padding. `"same"` results in padding evenly to
+            the left/right or up/down of the input. When `padding="same"` and
+            `strides=1`, the output has the same size as the input.
+        data_format: string, either `"channels_last"` or `"channels_first"`.
+            The ordering of the dimensions in the inputs. `"channels_last"`
+            corresponds to inputs with shape `(batch, height, width, channels)`
+            while `"channels_first"` corresponds to inputs with shape
+            `(batch, channels, height, width)`. It defaults to the
+            `image_data_format` value found in your Keras config file
+            at `~/.keras/keras.json`.
+            If you never set it, then it will be `"channels_last"`.
+        dilation_rate: int or tuple/list of 2 integers, specifying the dilation
+            rate to use for dilated convolution. If only one int is specified,
+            the same dilation rate will be used for all dimensions.
+        depth_multiplier: The number of depthwise convolution output channels
+            for each input channel. The total number of depthwise convolution
+            output channels will be equal to `input_channel * depth_multiplier`.
+        activation: Activation function. If `None`, no activation is applied.
+        use_bias: bool, if `True`, bias will be added to the output.
+        depthwise_initializer: An initializer for the depthwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        pointwise_initializer: An initializer for the pointwise convolution
+            kernel. If None, then the default initializer (`"glorot_uniform"`)
+            will be used.
+        bias_initializer: An initializer for the bias vector. If None, the
+            default initializer ('"zeros"') will be used.
+        depthwise_regularizer: Optional regularizer for the depthwise
+            convolution kernel.
+        pointwise_regularizer: Optional regularizer for the pointwise
+            convolution kernel.
+        bias_regularizer: Optional regularizer for the bias vector.
+        activity_regularizer: Optional regularizer function for the output.
+        depthwise_constraint: Optional projection function to be applied to the
+            depthwise kernel after being updated by an `Optimizer` (e.g. used
+            for norm constraints or value constraints for layer weights). The
+            function must take as input the unprojected variable and must return
+            the projected variable (which must have the same shape).
+        pointwise_constraint: Optional projection function to be applied to the
+            pointwise kernel after being updated by an `Optimizer`.
+        bias_constraint: Optional projection function to be applied to the
+            bias after being updated by an `Optimizer`.
+
+    Input shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, height, width, channels)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, channels, height, width)`
+
+    Output shape:
+
+    - If `data_format="channels_last"`:
+        A 4D tensor with shape: `(batch_size, new_height, new_width, filters)`
+    - If `data_format="channels_first"`:
+        A 4D tensor with shape: `(batch_size, filters, new_height, new_width)`
+
+    Returns:
+        A 4D tensor representing
+        `activation(separable_conv2d(inputs, kernel) + bias)`.
+
+    Example:
+
+    >>> x = np.random.rand(4, 10, 10, 12)
+    >>> y = keras.layers.SeparableConv2D(3, 4, 3, 2, activation='relu')(x)
+    >>> print(y.shape)
+    (4, 4, 4, 4)
+    """
+
+    def __init__(
+        self,
+        filters,
+        kernel_size,
+        strides=(1, 1),
+        padding="valid",
+        data_format=None,
+        dilation_rate=(1, 1),
+        depth_multiplier=1,
+        activation=None,
+        use_bias=True,
+        depthwise_initializer="glorot_uniform",
+        pointwise_initializer="glorot_uniform",
+        bias_initializer="zeros",
+        depthwise_regularizer=None,
+        pointwise_regularizer=None,
+        bias_regularizer=None,
+        activity_regularizer=None,
+        depthwise_constraint=None,
+        pointwise_constraint=None,
+        bias_constraint=None,
+        **kwargs,
+    ):
+        super().__init__(
+            rank=2,
+            depth_multiplier=depth_multiplier,
+            filters=filters,
+            kernel_size=kernel_size,
+            strides=strides,
+            padding=padding,
+            data_format=data_format,
+            dilation_rate=dilation_rate,
+            activation=activation,
+            use_bias=use_bias,
+            depthwise_initializer=depthwise_initializer,
+            pointwise_initializer=pointwise_initializer,
+            bias_initializer=bias_initializer,
+            depthwise_regularizer=depthwise_regularizer,
+            pointwise_regularizer=pointwise_regularizer,
+            bias_regularizer=bias_regularizer,
+            activity_regularizer=activity_regularizer,
+            depthwise_constraint=depthwise_constraint,
+            pointwise_constraint=pointwise_constraint,
+            bias_constraint=bias_constraint,
+            **kwargs,
+        )