def milliseconds(value: Union[int, float]) -> Duration:
    return float(value / 1000)

Converts input value from milliseconds to a `Duration` in seconds.

Example:
    ```python
    >>> duration = tp.duration.milliseconds(250)
    >>> duration
    0.25

    >>> # Usage in a window operation
    >>> a = tp.event_set(
    ...     timestamps=[0.5, 1.0, 1.2],
    ...     features={"f1": [1, 5, -5]}
    ... )
    >>> a.moving_sum(window_length=duration)
    indexes: ...
            timestamps: [0.5 1.  1.2]
            'f1': [1 5 0]
    ...

    ```

Args:
    value: Number of milliseconds.

Returns:
    Equivalent number of seconds.


def _flip(image, flip_index, scope_name):
    with ops.name_scope(None, scope_name, [image]):
        image = ops.convert_to_tensor(image, name='image')
        image = _AssertAtLeast3DImage(image)
        shape = image.get_shape()

        def f_rank3():
            return fix_image_flip_shape(image, array_ops.reverse(image, [flip_index]))

        def f_rank4():
            return array_ops.reverse(image, [flip_index + 1])
        if shape.ndims is None:
            rank = array_ops.rank(image)
            return tf_cond.cond(math_ops.equal(rank, 3), f_rank3, f_rank4)
        elif shape.ndims == 3:
            return f_rank3()
        elif shape.ndims == 4:
            return f_rank4()
        else:
            raise ValueError("'image' (shape %s)must have either 3 or 4 dimensions." % shape)

Flip an image either horizontally or vertically.

Outputs the contents of `image` flipped along the dimension `flip_index`.

See also `reverse()`.

Args:
  image: 4-D Tensor of shape `[batch, height, width, channels]` or 3-D Tensor
    of shape `[height, width, channels]`.
  flip_index: 0 For vertical, 1 for horizontal.
  scope_name: string, scope name.

Returns:
  A `Tensor` of the same type and shape as `image`.

Raises:
  ValueError: if the shape of `image` not supported.


def ZerosLikeForExit(self, val):
    val_shape = val.get_shape()
    forward_ctxt = val.op._get_control_flow_context()
    outer_forward_ctxt = forward_ctxt.outer_context
    if outer_forward_ctxt:
        outer_forward_ctxt = outer_forward_ctxt.GetWhileContext()
    outer_grad_state = None
    if outer_forward_ctxt:
        outer_grad_state = self._map.get(outer_forward_ctxt)
    if outer_grad_state:
        if val_shape.is_fully_defined():
            outer_grad_state.grad_context.Enter()
            result = array_ops.zeros(val_shape.dims, val.dtype)
            outer_grad_state.grad_context.Exit()
        else:
            forward_ctxt.outer_context.Enter()
            shape = array_ops.shape_internal(val, optimize=False)
            forward_ctxt.outer_context.Exit()
            history_shape = outer_grad_state.AddForwardAccumulator(shape)
            outer_grad_ctxt = outer_grad_state.grad_context
            outer_grad_ctxt.Enter()
            real_shape = outer_grad_state.AddBackpropAccumulatedValue(history_shape, shape)
            result = array_ops.zeros(real_shape, val.dtype)
            outer_grad_ctxt.Exit()
    elif val_shape.is_fully_defined():
        result = array_ops.zeros(val_shape.dims, val.dtype)
    else:
        result = array_ops.zeros_like(val, optimize=False)
    return result

Create zeros_like gradient for a loop exit.

If the result of a loop variable is not used but is involved in
computing the result of some needed loop variable, we create a
zero-valued tensor that is fed as gradient for the Exit node of that
loop variable. Note that val.op is an Exit, and this method must be
called in the control flow context where gradients() is called.

Args:
  val: The output tensor of an Exit op.

Returns:
  A zero tensor of the same shape of val.


def pad_image(self, image: np.ndarray, size_divisibility: int=32, data_format: Optional[Union[str, ChannelDimension]]=None, input_data_format: Optional[Union[str, ChannelDimension]]=None) -> np.ndarray:
    if input_data_format is None:
        input_data_format = infer_channel_dimension_format(image)
    height, width = get_image_size(image, input_data_format)
    pad_height = 0 if height % size_divisibility == 0 else size_divisibility - height % size_divisibility
    pad_width = 0 if width % size_divisibility == 0 else size_divisibility - width % size_divisibility
    if pad_width + pad_height > 0:
        padding = ((0, pad_height), (0, pad_width))
        image = pad(image, padding=padding, data_format=data_format, input_data_format=input_data_format)
    if data_format is not None:
        image = to_channel_dimension_format(image, data_format, input_data_format)
    return image

Args:
    image (`np.ndarray`):
        Image to pad.
    size_divisibility (`int`, *optional*, defaults to 32):
        The width and height of the image will be padded to be divisible by this number.
    data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
        The channel dimension format for the output image. Can be one of:
        - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
        - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
        - Unset: Use the channel dimension format of the input image.
    input_data_format (`ChannelDimension` or `str`, *optional*):
        The channel dimension format for the input image. If unset, the channel dimension format is inferred
        from the input image. Can be one of:
        - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
        - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
        - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.


def deprecated_internal_set_learning_phase(value):
    global _GRAPH_LEARNING_PHASES
    if value not in {0, 1}:
        raise ValueError('Expected learning phase to be 0 or 1.')
    with ops.init_scope():
        if context.executing_eagerly():
            _DUMMY_EAGER_GRAPH.learning_phase_is_set = True
            _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH.key] = value
        _GRAPH_LEARNING_PHASES[get_graph()] = value

A deprecated internal implementation of set_learning_phase.

This method is an internal-only version of `set_learning_phase` that
does not raise a deprecation error. It is required because
saved_model needs to keep working with user code that uses the deprecated
learning phase methods until those APIs are fully removed from the public API.

Specifically SavedModel saving needs to make sure the learning phase is 0
during tracing even if users overwrote it to a different value.

But, we don't want to raise deprecation warnings for users when savedmodel
sets learning phase just for compatibility with code that relied on
explicitly setting the learning phase for other values.

Args:
    value: Learning phase value, either 0 or 1 (integers). 0 = test, 1 = train

Raises:
    ValueError: if `value` is neither `0` nor `1`.


def _calculate_acceptance_probs_with_mixing(initial_probs, target_probs):
    ratio_l = _get_target_to_initial_ratio(initial_probs, target_probs)
    max_ratio = math_ops.reduce_max(ratio_l)
    min_ratio = math_ops.reduce_min(ratio_l)
    m = min_ratio
    a_i = (ratio_l - m) / (max_ratio - m)
    return (a_i, m)

Calculates the acceptance probabilities and mixing ratio.

In this case, we assume that we can *either* sample from the original data
distribution with probability `m`, or sample from a reshaped distribution
that comes from rejection sampling on the original distribution. This
rejection sampling is done on a per-class basis, with `a_i` representing the
probability of accepting data from class `i`.

This method is based on solving the following analysis for the reshaped
distribution:

Let F be the probability of a rejection (on any example).
Let p_i be the proportion of examples in the data in class i (init_probs)
Let a_i is the rate the rejection sampler should *accept* class i
Let t_i is the target proportion in the minibatches for class i (target_probs)

```
F = sum_i(p_i * (1-a_i))
  = 1 - sum_i(p_i * a_i)     using sum_i(p_i) = 1
```

An example with class `i` will be accepted if `k` rejections occur, then an
example with class `i` is seen by the rejector, and it is accepted. This can
be written as follows:

```
t_i = sum_k=0^inf(F^k * p_i * a_i)
    = p_i * a_j / (1 - F)    using geometric series identity, since 0 <= F < 1
    = p_i * a_i / sum_j(p_j * a_j)        using F from above
```

Note that the following constraints hold:
```
0 <= p_i <= 1, sum_i(p_i) = 1
0 <= a_i <= 1
0 <= t_i <= 1, sum_i(t_i) = 1
```

A solution for a_i in terms of the other variables is the following:
  ```a_i = (t_i / p_i) / max_i[t_i / p_i]```

If we try to minimize the amount of data rejected, we get the following:

M_max = max_i [ t_i / p_i ]
M_min = min_i [ t_i / p_i ]

The desired probability of accepting data if it comes from class `i`:

a_i = (t_i/p_i - m) / (M_max - m)

The desired probability of pulling a data element from the original dataset,
rather than the filtered one:

m = M_min

Args:
  initial_probs: A Tensor of the initial probability distribution, given or
    estimated.
  target_probs: A Tensor of the corresponding classes.

Returns:
  (A 1D Tensor with the per-class acceptance probabilities, the desired
  probability of pull from the original distribution.)


def remat(f):
    return backend.core.remat(f)

Applies rematerialization to a function or layer for memory optimization.

Rematerialization is a memory optimization technique that trades off
computation for memory. Instead of storing intermediate results
(e.g. activations) for backpropagation, they are recomputed during the
backward pass. This reduces peak memory usage at the cost of increased
computation time, allowing the training of larger models or using larger
batch sizes within the same memory constraints.

Args:
    f: A callable function, to which rematerialization is
       applied. This is typically a computationally expensive operation
       where intermediate states can be recomputed instead of stored.

Returns:
    A wrapped function that applies rematerialization. The returned
    function defines a custom gradient, ensuring that during the backward
    pass, the forward computation is recomputed as needed.

Example:

```python
from keras import Model
class CustomRematLayer(layers.Layer):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.remat_function = remat(self.intermediate_function)

    def intermediate_function(self, x):
        for _ in range(2):
            x = x + x * 0.1  # Simple scaled transformation
        return x

    def call(self, inputs):
        return self.remat_function(inputs)

# Define a simple model using the custom layer
inputs = layers.Input(shape=(4,))
x = layers.Dense(4, activation="relu")(inputs)
x = CustomRematLayer()(x)  # Custom layer with rematerialization
outputs = layers.Dense(1)(x)

# Create and compile the model
model = Model(inputs=inputs, outputs=outputs)
model.compile(optimizer="sgd", loss="mse")
```


def __init__(self, reflection_axis, is_non_singular=None, is_self_adjoint=None, is_positive_definite=None, is_square=None, name='LinearOperatorHouseholder'):
    parameters = dict(reflection_axis=reflection_axis, is_non_singular=is_non_singular, is_self_adjoint=is_self_adjoint, is_positive_definite=is_positive_definite, is_square=is_square, name=name)
    with ops.name_scope(name, values=[reflection_axis]):
        self._reflection_axis = linear_operator_util.convert_nonref_to_tensor(reflection_axis, name='reflection_axis')
        self._check_reflection_axis(self._reflection_axis)
        if is_self_adjoint is False:
            raise ValueError('A Householder operator is always self adjoint.')
        else:
            is_self_adjoint = True
        if is_positive_definite is True:
            raise ValueError('A Householder operator is always non-positive definite.')
        else:
            is_positive_definite = False
        if is_square is False:
            raise ValueError('A Householder operator is always square.')
        is_square = True
        super(LinearOperatorHouseholder, self).__init__(dtype=self._reflection_axis.dtype, is_non_singular=is_non_singular, is_self_adjoint=is_self_adjoint, is_positive_definite=is_positive_definite, is_square=is_square, parameters=parameters, name=name)

Initialize a `LinearOperatorHouseholder`.

Args:
  reflection_axis:  Shape `[B1,...,Bb, N]` `Tensor` with `b >= 0` `N >= 0`.
    The vector defining the hyperplane to reflect about.
    Allowed dtypes: `float16`, `float32`, `float64`, `complex64`,
    `complex128`.
  is_non_singular:  Expect that this operator is non-singular.
  is_self_adjoint:  Expect that this operator is equal to its hermitian
    transpose.  This is autoset to true
  is_positive_definite:  Expect that this operator is positive definite,
    meaning the quadratic form `x^H A x` has positive real part for all
    nonzero `x`.  Note that we do not require the operator to be
    self-adjoint to be positive-definite.  See:
    https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
    This is autoset to false.
  is_square:  Expect that this operator acts like square [batch] matrices.
    This is autoset to true.
  name: A name for this `LinearOperator`.

Raises:
  ValueError:  `is_self_adjoint` is not `True`, `is_positive_definite` is
    not `False` or `is_square` is not `True`.


def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    if token_ids_1 is None:
        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
    cls = [self.cls_token_id]
    sep = [self.sep_token_id]
    return cls + token_ids_0 + sep + sep + token_ids_1 + sep

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A LUKE sequence has the following format:

- single sequence: `<s> X </s>`
- pair of sequences: `<s> A </s></s> B </s>`

Args:
    token_ids_0 (`List[int]`):
        List of IDs to which the special tokens will be added.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.

Returns:
    `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.


class GenerateBeamEncoderDecoderOutput(ModelOutput):
    sequences: torch.LongTensor
    sequences_scores: Optional[torch.FloatTensor] = None
    scores: Optional[Tuple[torch.FloatTensor]] = None
    logits: Optional[Tuple[torch.FloatTensor]] = None
    beam_indices: Optional[torch.LongTensor] = None
    encoder_attentions: Optional[Tuple[torch.FloatTensor]] = None
    encoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    decoder_attentions: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
    cross_attentions: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
    decoder_hidden_states: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
    past_key_values: Optional[Tuple[Tuple[Tuple[torch.FloatTensor]]]] = None

Outputs of encoder-decoder generation models, when using beam methods.

Args:
    sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
        The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
        if all batches finished early due to the `eos_token_id`.
    sequences_scores (`torch.FloatTensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True`):
        Final beam scores of the generated `sequences`.
    scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True`):
        Beam transition scores for each vocabulary token at each generation step. Beam transition scores consisting
        of log probabilities of tokens conditioned on log softmax of previously generated tokens in this beam.
        Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for each generated token),
        with each tensor of shape `(batch_size*num_beams, config.vocab_size)`.
    logits (`tuple(torch.FloatTensor)` *optional*, returned when `output_logits=True`):
        Unprocessed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
        at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for
        each generated token), with each tensor of shape `(batch_size*num_beams, config.vocab_size)`.
    beam_indices (`torch.LongTensor`, *optional*, returned when `output_scores=True`):
        Beam indices of generated token id at each generation step. `torch.LongTensor` of shape
        `(batch_size*num_return_sequences, sequence_length)`.
    encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads,
        sequence_length, sequence_length)`.
    encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size*num_beams*num_return_sequences, sequence_length, hidden_size)`.
    decoder_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
        Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
        `torch.FloatTensor` of shape `(batch_size*num_beams*num_return_sequences, num_heads, generated_length,
        sequence_length)`.
    cross_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True`):
        Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
        `torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
    decoder_hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True`):
        Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
        `torch.FloatTensor` of shape `(batch_size*num_beams*num_return_sequences, generated_length, hidden_size)`.
    past_key_values (`tuple(tuple(torch.FloatTensor)))`, *optional*, returned when `use_cache=True`):
        Returns the model cache, used to speed up decoding. Different models have a different cache format, check
        the model's documentation. Usually, a [`~cache_utils.Cache`] instance.


def __rmod__(self, other):
    other = as_dimension(other)
    return other % self

Returns `other` modulo `self`.

Args:
  other: Another Dimension, or a value accepted by `as_dimension`.

Returns:
  A Dimension whose value is `other` modulo `self`.


def materialize_value_set_expansion(self, urls: Iterable[str], expander: Union[terminology_service_client.TerminologyServiceClient, value_sets.ValueSetResolver], terminology_service_url: Optional[str]=None, batch_size: int=500) -> None:
    if terminology_service_url is not None and (not isinstance(expander, terminology_service_client.TerminologyServiceClient)):
        raise TypeError('`terminology_service_url` can only be given if `expander` is a TerminologyServiceClient')
    if terminology_service_url is not None and isinstance(expander, terminology_service_client.TerminologyServiceClient):
        expanded_value_sets = (expander.expand_value_set_url_using_service(url, terminology_service_url) for url in urls)
    else:
        expanded_value_sets = (expander.expand_value_set_url(url) for url in urls)
    self.materialize_value_sets(expanded_value_sets, batch_size=batch_size)

Expands a sequence of value set and materializes their expanded codes.

Expands the given value set URLs to obtain the set of codes they describe.
Then writes these expanded codes into the database
named after the `value_set_codes_table` provided at class initialization.
Builds a valueset_codes table as described by
https://github.com/FHIR/sql-on-fhir/blob/master/sql-on-fhir.md#valueset-support

The table will be created if it does not already exist.

The function will avoid inserting duplicate rows if some of the codes are
already present in the given table. It will not attempt to perform an
'upsert' or modify any existing rows.

Provided as a utility function for user convenience. If `urls` is a large
set of URLs, callers may prefer to use multi-processing and/or
multi-threading to perform expansion and table insertion of the URLs
concurrently. This function performs all expansions and table insertions
serially.

Args:
  urls: The urls for value sets to expand and materialize.
  expander: The ValueSetResolver or TerminologyServiceClient to perform
    value set expansion. A ValueSetResolver may be used to attempt to avoid
    some network requests by expanding value sets locally. A
    TerminologyServiceClient will use external terminology services to
    perform all value set expansions.
  terminology_service_url: If `expander` is a TerminologyServiceClient, the
    URL of the terminology service to use when expanding value set URLs. If
    not given, the client will attempt to infer the correct terminology
    service to use for each value set URL based on its domain.
  batch_size: The maximum number of rows to insert in a single query.

Raises:
  TypeError: If a `terminology_service_url` is given but `expander` is not a
  TerminologyServiceClient.


def take_while(self, predicate, name=None) -> 'DatasetV2':
    from tensorflow.python.data.ops import take_while_op
    return take_while_op._take_while(self, predicate, name=name)

A transformation that stops dataset iteration based on a `predicate`.

>>> dataset = tf.data.Dataset.range(10)
>>> dataset = dataset.take_while(lambda x: x < 5)
>>> [a.item() for a in dataset.as_numpy_iterator()]
[0, 1, 2, 3, 4]

Args:
  predicate: A function that maps a nested structure of tensors (having
    shapes and types defined by `self.output_shapes` and
    `self.output_types`) to a scalar `tf.bool` tensor.
  name: (Optional.) A name for the tf.data operation.

Returns:
  A new `Dataset` with the transformation applied as described above.


def less_equal(x, y):
    return math_ops.less_equal(x, y)

Element-wise truth value of (x <= y).

Args:
    x: Tensor or variable.
    y: Tensor or variable.

Returns:
    A bool tensor.


def __init__(self, pqc: cirq.Circuit, initializer: tf.keras.initializers.Initializer=tf.keras.initializers.RandomUniform(0, 2), name: Union[None, str]=None):
    raw_symbol_names = list(sorted(tfq.util.get_circuit_symbols(pqc)))
    symbol_names = tf.constant([str(x) for x in raw_symbol_names], dtype=tf.string)
    values = [tf.Variable(initializer(shape=[len(raw_symbol_names)]))]
    value_layers = [[]]
    super().__init__(tfq.convert_to_tensor([pqc]), pqc.all_qubits(), symbol_names, values, value_layers)

Initializes a DirectQuantumCircuit.

Args:
  pqc: Representation of a parameterized quantum circuit.
  initializer: A `tf.keras.initializers.Initializer` which specifies how to
    initialize the values of the parameters in `circuit`.  The default
    initializer assumes parameters of gates are exponents, so that one full
    period is covered by the parameter range 0 to 2.
  name: Optional name for the model.


def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    if token_ids_1 is None:
        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
    cls = [self.cls_token_id]
    sep = [self.sep_token_id]
    return cls + token_ids_0 + sep + token_ids_1 + sep

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A ConvBERT sequence has the following format:

- single sequence: `[CLS] X [SEP]`
- pair of sequences: `[CLS] A [SEP] B [SEP]`

Args:
    token_ids_0 (`List[int]`):
        List of IDs to which the special tokens will be added.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.

Returns:
    `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.


def reduce(self, initial_state, reduce_func):

Reduces this iterable object to a single element.

The transformation calls `reduce_func` successively on each element.
The `initial_state` argument is used for the initial state and the final
state is returned as the result.

Args:
  initial_state: An element representing the initial state of the
    reduction.
  reduce_func: A function that maps `(old_state, input_element)` to
    `new_state`. The structure of `new_state` must match the structure of
    `old_state`. For the first element, `old_state` is `initial_state`.

Returns:
  The final state of the transformation.


class LlamaTokenizerFast(PreTrainedTokenizerFast):
    vocab_files_names = VOCAB_FILES_NAMES
    slow_tokenizer_class = LlamaTokenizer
    padding_side = 'left'
    model_input_names = ['input_ids', 'attention_mask']

    def __init__(self, vocab_file=None, tokenizer_file=None, clean_up_tokenization_spaces=False, unk_token='<unk>', bos_token='<s>', eos_token='</s>', add_bos_token=True, add_eos_token=False, use_default_system_prompt=False, legacy=None, add_prefix_space=None, **kwargs):
        if legacy is None:
            logger.warning_once(f'You are using the default legacy behaviour of the {self.__class__}. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message.')
            legacy = True
        self.legacy = legacy
        if add_prefix_space is not None:
            kwargs['from_slow'] = True
        super().__init__(vocab_file=vocab_file, tokenizer_file=tokenizer_file, clean_up_tokenization_spaces=clean_up_tokenization_spaces, unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, add_bos_token=add_bos_token, add_eos_token=add_eos_token, use_default_system_prompt=use_default_system_prompt, add_prefix_space=add_prefix_space, legacy=legacy, **kwargs)
        self._add_bos_token = add_bos_token
        self._add_eos_token = add_eos_token
        self.update_post_processor()
        self.use_default_system_prompt = use_default_system_prompt
        self.vocab_file = vocab_file

    def update_post_processor(self):
        """
        Updates the underlying post processor with the current `bos_token` and `eos_token`.
        """
        bos = self.bos_token
        bos_token_id = self.bos_token_id
        if bos is None and self.add_bos_token:
            raise ValueError('add_bos_token = True but bos_token = None')
        eos = self.eos_token
        eos_token_id = self.eos_token_id
        if eos is None and self.add_eos_token:
            raise ValueError('add_eos_token = True but eos_token = None')
        single = f'{(bos + ':0 ' if self.add_bos_token else '')}$A:0{(' ' + eos + ':0' if self.add_eos_token else '')}'
        pair = f'{single}{(' ' + bos + ':1' if self.add_bos_token else '')} $B:1{(' ' + eos + ':1' if self.add_eos_token else '')}'
        special_tokens = []
        if self.add_bos_token:
            special_tokens.append((bos, bos_token_id))
        if self.add_eos_token:
            special_tokens.append((eos, eos_token_id))
        self._tokenizer.post_processor = processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)

    @property
    def add_eos_token(self):
        return self._add_eos_token

    @property
    def add_bos_token(self):
        return self._add_bos_token

    @add_eos_token.setter
    def add_eos_token(self, value):
        self._add_eos_token = value
        self.update_post_processor()

    @add_bos_token.setter
    def add_bos_token(self, value):
        self._add_bos_token = value
        self.update_post_processor()

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str]=None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError('Your fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.')
        if not os.path.isdir(save_directory):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
            return
        out_vocab_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        return (out_vocab_file,)

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
        output = bos_token_id + token_ids_0 + eos_token_id
        if token_ids_1 is not None:
            output = output + bos_token_id + token_ids_1 + eos_token_id
        return output

Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.

This uses notably ByteFallback and no normalization.

```python
>>> from transformers import LlamaTokenizerFast

>>> tokenizer = LlamaTokenizerFast.from_pretrained("hf-internal-testing/llama-tokenizer")
>>> tokenizer.encode("Hello this is a test")
[1, 15043, 445, 338, 263, 1243]
```

If you want to change the `bos_token` or the `eos_token`, make sure to specify them when initializing the model, or
call `tokenizer.update_post_processor()` to make sure that the post-processing is correctly done (otherwise the
values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
[post-processors] (https://huggingface.co/docs/tokenizers/api/post-processors) documentation.

This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods.

Args:
    vocab_file (`str`, *optional*):
        [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .model extension) that
        contains the vocabulary necessary to instantiate a tokenizer.
    tokenizer_file (`str`, *optional*):
        [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
        contains everything needed to load the tokenizer.
    clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
        Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like
        extra spaces.
    unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<s>"`):
        The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
    eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"</s>"`):
        The end of sequence token.
    add_bos_token (`bool`, *optional*, defaults to `True`):
        Whether or not to add an `bos_token` at the start of sequences.
    add_eos_token (`bool`, *optional*, defaults to `False`):
        Whether or not to add an `eos_token` at the end of sequences.
    use_default_system_prompt (`bool`, *optional*, defaults to `False`):
        Whether or not the default system prompt for Llama should be used
    legacy (`bool`, *optional*):
        Whether or not the `legacy` behavior of the tokenizer should be used. Legacy is before the merge of #24622
        and #25224 which includes fixes to properly handle tokens that appear after special tokens.
        Make sure to also set `from_slow` to `True`.
        A simple example:

        - `legacy=True`:
        ```python
        >>> from transformers import LlamaTokenizerFast

        >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=True, from_slow=True)
        >>> tokenizer.encode("Hello <s>.") # 869 is '▁.'
        [1, 15043, 29871, 1, 869]
        ```
        - `legacy=False`:
        ```python
        >>> from transformers import LlamaTokenizerFast

        >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=False, from_slow=True)
        >>> tokenizer.encode("Hello <s>.")  # 29889 is '.'
        [1, 15043, 29871, 1, 29889]
        ```
        Checkout the [pull request](https://github.com/huggingface/transformers/pull/24565) for more details.
    add_prefix_space (`bool`, *optional*):
        Whether or not the tokenizer should automatically add a prefix space


def _verify_pipeline_uuid(self, pipeline_uuid):
    try:
        uuid.UUID(pipeline_uuid)
    except ValueError as ve:
        raise ValueError(f"Incorrect pipeline uuid: '{pipeline_uuid}'") from ve

Verify the received pipeline_uuid format

Args:
    pipeline_uuid: uuid of the pipeline

Returns:
    If pipeline ID is not verified, will raise an exception


def _in_multi_worker_mode(self):
    strategy = self._distribution_strategy
    if not strategy and distribute_lib.has_strategy():
        strategy = distribute_lib.get_strategy()
    return strategy and strategy.extended._in_multi_worker_mode()

Method to infer if this `Model` is working in multi-worker settings.

Multi-worker training refers to the setup where the training is
distributed across multiple workers, as opposed to the case where
only a local process performs the training. This function is
used to infer for example whether or not a distribute coordinator
should be run, and thus TensorFlow servers should be started for
communication with other servers in the cluster, or whether or not
saving/restoring checkpoints is relevant for preemption fault tolerance.

Experimental. Signature and implementation are subject to change.

Returns:
  Whether this model indicates it's working in multi-worker settings.


def get_conversion_metadata(model_buffer):
    model_object = flatbuffer_utils.convert_bytearray_to_object(model_buffer)
    if not model_object or not model_object.metadata:
        return None
    for meta in model_object.metadata:
        if meta.name.decode('utf-8') == CONVERSION_METADATA_FIELD_NAME:
            metadata_buf = model_object.buffers[meta.buffer].data.tobytes()
            return conversion_metadata_fb.ConversionMetadataT.InitFromObj(conversion_metadata_fb.ConversionMetadata.GetRootAsConversionMetadata(metadata_buf, 0))
    return None

Read conversion metadata from a tflite model.

Args:
  model_buffer: A tflite model.

Returns:
  The conversion metadata or None if it is not populated.


def call(self, input_ids: TFModelInputType=None, attention_mask: tf.Tensor | None=None, decoder_input_ids: tf.Tensor | None=None, decoder_attention_mask: tf.Tensor | None=None, decoder_position_ids: tf.Tensor | None=None, head_mask: tf.Tensor | None=None, decoder_head_mask: tf.Tensor | None=None, cross_attn_head_mask: tf.Tensor | None=None, encoder_outputs: Optional[TFBaseModelOutput]=None, past_key_values: Optional[Tuple[Tuple[tf.Tensor]]]=None, inputs_embeds: tf.Tensor | None=None, decoder_inputs_embeds: tf.Tensor | None=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None, labels: tf.Tensor | None=None, training: Optional[bool]=False) -> Union[TFSeq2SeqLMOutput, Tuple[tf.Tensor]]:
    if labels is not None:
        labels = tf.where(labels == self.config.pad_token_id, tf.cast(tf.fill(shape_list(labels), -100), labels.dtype), labels)
        use_cache = False
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            decoder_input_ids = shift_tokens_right(labels, self.config.pad_token_id)
    outputs = self.model(input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, encoder_outputs=encoder_outputs, decoder_attention_mask=decoder_attention_mask, decoder_position_ids=decoder_position_ids, head_mask=head_mask, decoder_head_mask=decoder_head_mask, cross_attn_head_mask=cross_attn_head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training)
    lm_logits = tf.matmul(outputs[0], self.model.shared.weights, transpose_b=True)
    lm_logits = self.bias_layer(lm_logits)
    masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits)
    if not return_dict:
        output = (lm_logits,) + outputs[1:]
        return (masked_lm_loss,) + output if masked_lm_loss is not None else output
    return TFSeq2SeqLMOutput(loss=masked_lm_loss, logits=lm_logits, past_key_values=outputs.past_key_values, decoder_hidden_states=outputs.decoder_hidden_states, decoder_attentions=outputs.decoder_attentions, cross_attentions=outputs.cross_attentions, encoder_last_hidden_state=outputs.encoder_last_hidden_state, encoder_hidden_states=outputs.encoder_hidden_states, encoder_attentions=outputs.encoder_attentions)

labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Returns:


def constant_value_as_shape(tensor):
    if isinstance(tensor, core.Value):
        return tensor_shape.TensorShape([dim if dim != -1 else None for dim in tensor.numpy()])
    if tensor.get_shape().ndims == 0:
        value = constant_value(tensor)
        if value is None:
            raise ValueError("Received a scalar with unknown value as shape; require a statically known scalar with value '-1' to describe an unknown shape.")
        if value != -1:
            raise ValueError(f"Received a scalar value '{value}' as shape; require a statically known scalar with value '-1' to describe an unknown shape.")
        return tensor_shape.unknown_shape()
    shape = tensor.get_shape().with_rank(1)
    if shape == [0]:
        return tensor_shape.TensorShape([])
    elif tensor.op.type == 'Cast':
        pre_cast = constant_value_as_shape(tensor.op.inputs[0])
        if pre_cast.dims is None:
            return pre_cast
        cast_dtype = dtypes.as_dtype(tensor.op.get_attr('DstT'))
        if cast_dtype not in (dtypes.int32, dtypes.int64):
            return tensor_shape.unknown_shape(shape.dims[0].value)
        dest_dtype_shape_array = np.array([x if x is not None else -1 for x in pre_cast.as_list()]).astype(cast_dtype.as_numpy_dtype)
        return tensor_shape.TensorShape([x if x >= 0 else None for x in dest_dtype_shape_array])
    elif tensor.op.type == 'Shape':
        return tensor.op.inputs[0].get_shape()
    elif tensor.op.type == 'Pack':
        ret = tensor_shape.TensorShape([])
        assert tensor.op.get_attr('axis') == 0
        for pack_input in tensor.op.inputs:
            pack_input_val = constant_value(pack_input)
            if pack_input_val is None or pack_input_val < 0:
                new_dim = tensor_shape.Dimension(None)
            else:
                new_dim = tensor_shape.Dimension(pack_input_val)
            ret = ret.concatenate([new_dim])
        return ret
    elif tensor.op.type == 'Concat':
        ret = tensor_shape.TensorShape([])
        for concat_input in tensor.op.inputs[1:]:
            ret = ret.concatenate(constant_value_as_shape(concat_input))
        return ret
    elif tensor.op.type == 'ConcatV2':
        ret = tensor_shape.TensorShape([])
        for concat_input in tensor.op.inputs[:-1]:
            ret = ret.concatenate(constant_value_as_shape(concat_input))
        return ret
    elif tensor.op.type == 'StridedSlice':
        try:
            begin = constant_value(tensor.op.inputs[1])
            end = constant_value(tensor.op.inputs[2])
            strides = constant_value(tensor.op.inputs[3])
            if begin is not None and end is not None and (strides is not None):
                begin = begin[0]
                end = end[0]
                strides = strides[0]
                begin_mask = tensor.op.get_attr('begin_mask')
                if begin_mask == 1:
                    begin = None
                end_mask = tensor.op.get_attr('end_mask')
                if end_mask == 1:
                    end = None
                ellipsis_mask = tensor.op.get_attr('ellipsis_mask')
                new_axis_mask = tensor.op.get_attr('new_axis_mask')
                shrink_axis_mask = tensor.op.get_attr('shrink_axis_mask')
                valid_attributes = not ellipsis_mask and (not new_axis_mask) and (not shrink_axis_mask) and (not begin_mask or begin_mask == 1) and (not end_mask or end_mask == 1)
                if valid_attributes:
                    prev = constant_value_as_shape(tensor.op.inputs[0])
                    prev = prev[begin:end:strides]
                    ret = tensor_shape.TensorShape(prev)
                    return ret
        except ValueError:
            pass
        except TypeError:
            pass
    elif tensor.op.type == 'Placeholder' and tensor.op.graph.building_function and hasattr(tensor.op.graph, 'internal_captures'):
        for i, capture in enumerate(tensor.op.graph.internal_captures):
            if capture is tensor:
                external_capture = tensor.op.graph.external_captures[i]
                return constant_value_as_shape(external_capture)
    ret = tensor_shape.unknown_shape(shape.dims[0].value)
    value = constant_value(tensor)
    if value is not None:
        ret = ret.merge_with(tensor_shape.TensorShape([d if d >= 0 else None for d in value]))
    return ret

A version of `constant_value()` that returns a `TensorShape`.

This version should be used when a constant tensor value is
interpreted as a (possibly partial) shape, e.g. in the shape
function for `tf.reshape()`. By explicitly requesting a
`TensorShape` as the return value, it is possible to represent
unknown dimensions; by contrast, `constant_value()` is
all-or-nothing.

Args:
  tensor: The rank-0 or rank-1 Tensor to be evaluated.

Returns:
  A `TensorShape` based on the constant value of the given `tensor`.

Raises:
  ValueError: If the shape is rank-0 and is not statically known to be -1.


class TimmBackboneConfig(PretrainedConfig):
    model_type = 'timm_backbone'

    def __init__(self, backbone=None, num_channels=3, features_only=True, use_pretrained_backbone=True, out_indices=None, freeze_batch_norm_2d=False, **kwargs):
        super().__init__(**kwargs)
        self.backbone = backbone
        self.num_channels = num_channels
        self.features_only = features_only
        self.use_pretrained_backbone = use_pretrained_backbone
        self.use_timm_backbone = True
        self.out_indices = out_indices if out_indices is not None else [-1]
        self.freeze_batch_norm_2d = freeze_batch_norm_2d

This is the configuration class to store the configuration for a timm backbone [`TimmBackbone`].

It is used to instantiate a timm backbone model according to the specified arguments, defining the model.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    backbone (`str`, *optional*):
        The timm checkpoint to load.
    num_channels (`int`, *optional*, defaults to 3):
        The number of input channels.
    features_only (`bool`, *optional*, defaults to `True`):
        Whether to output only the features or also the logits.
    use_pretrained_backbone (`bool`, *optional*, defaults to `True`):
        Whether to use a pretrained backbone.
    out_indices (`List[int]`, *optional*):
        If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how
        many stages the model has). Will default to the last stage if unset.
    freeze_batch_norm_2d (`bool`, *optional*, defaults to `False`):
        Converts all `BatchNorm2d` and `SyncBatchNorm` layers of provided module into `FrozenBatchNorm2d`.

Example:
```python
>>> from transformers import TimmBackboneConfig, TimmBackbone

>>> # Initializing a timm backbone
>>> configuration = TimmBackboneConfig("resnet50")

>>> # Initializing a model from the configuration
>>> model = TimmBackbone(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```


def print_layer_summary(layer):
    try:
        output_shape = layer.output_shape
    except AttributeError:
        output_shape = 'multiple'
    except RuntimeError:
        output_shape = '?'
    name = layer.name
    cls_name = layer.__class__.__name__
    if not layer.built and (not getattr(layer, '_is_graph_network', False)):
        params = '0 (unused)'
    else:
        params = layer.count_params()
    fields = [name + ' (' + cls_name + ')', output_shape, params]
    print_row(fields, positions)

Prints a summary for a single layer.

Args:
    layer: target layer.


def __init__(self, logits=None, probs=None, dtype=dtypes.int32, validate_args=False, allow_nan_stats=True, name='Categorical'):
    parameters = dict(locals())
    with ops.name_scope(name, values=[logits, probs]) as name:
        self._logits, self._probs = distribution_util.get_logits_and_probs(logits=logits, probs=probs, validate_args=validate_args, multidimensional=True, name=name)
        if validate_args:
            self._logits = distribution_util.embed_check_categorical_event_shape(self._logits)
        logits_shape_static = self._logits.get_shape().with_rank_at_least(1)
        if logits_shape_static.ndims is not None:
            self._batch_rank = ops.convert_to_tensor(logits_shape_static.ndims - 1, dtype=dtypes.int32, name='batch_rank')
        else:
            with ops.name_scope(name='batch_rank'):
                self._batch_rank = array_ops.rank(self._logits) - 1
        logits_shape = array_ops.shape(self._logits, name='logits_shape')
        if tensor_shape.dimension_value(logits_shape_static[-1]) is not None:
            self._event_size = ops.convert_to_tensor(logits_shape_static.dims[-1].value, dtype=dtypes.int32, name='event_size')
        else:
            with ops.name_scope(name='event_size'):
                self._event_size = logits_shape[self._batch_rank]
        if logits_shape_static[:-1].is_fully_defined():
            self._batch_shape_val = constant_op.constant(logits_shape_static[:-1].as_list(), dtype=dtypes.int32, name='batch_shape')
        else:
            with ops.name_scope(name='batch_shape'):
                self._batch_shape_val = logits_shape[:-1]
    super(Categorical, self).__init__(dtype=dtype, reparameterization_type=distribution.NOT_REPARAMETERIZED, validate_args=validate_args, allow_nan_stats=allow_nan_stats, parameters=parameters, graph_parents=[self._logits, self._probs], name=name)

Initialize Categorical distributions using class log-probabilities.

Args:
  logits: An N-D `Tensor`, `N >= 1`, representing the log probabilities
    of a set of Categorical distributions. The first `N - 1` dimensions
    index into a batch of independent distributions and the last dimension
    represents a vector of logits for each class. Only one of `logits` or
    `probs` should be passed in.
  probs: An N-D `Tensor`, `N >= 1`, representing the probabilities
    of a set of Categorical distributions. The first `N - 1` dimensions
    index into a batch of independent distributions and the last dimension
    represents a vector of probabilities for each class. Only one of
    `logits` or `probs` should be passed in.
  dtype: The type of the event samples (default: int32).
  validate_args: Python `bool`, default `False`. When `True` distribution
    parameters are checked for validity despite possibly degrading runtime
    performance. When `False` invalid inputs may silently render incorrect
    outputs.
  allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
    (e.g., mean, mode, variance) use the value "`NaN`" to indicate the
    result is undefined. When `False`, an exception is raised if one or
    more of the statistic's batch members are undefined.
  name: Python `str` name prefixed to Ops created by this class.


def traverse(func, structure, top_down=True):
    return tree_impl.traverse(func, structure, top_down=top_down)

Traverses the given nested structure, applying the given function.

The traversal is depth-first. If `top_down` is True (default), parents
are returned before their children (giving the option to avoid traversing
into a sub-tree).

Examples:

>>> v = []
>>> keras.tree.traverse(v.append, [(1, 2), [3], {"a": 4}], top_down=True)
[(1, 2), [3], {'a': 4}]
>>> v
[[(1, 2), [3], {'a': 4}], (1, 2), 1, 2, [3], 3, {'a': 4}, 4]

>>> v = []
>>> keras.tree.traverse(v.append, [(1, 2), [3], {"a": 4}], top_down=False)
[(1, 2), [3], {'a': 4}]
>>> v
[1, 2, (1, 2), 3, [3], 4, {'a': 4}, [(1, 2), [3], {'a': 4}]]

Args:
    func: The function to be applied to each sub-nest of the structure.

    When traversing top-down:
        If `func(subtree) is None` the traversal continues into the
        sub-tree.
        If `func(subtree) is not None` the traversal does not continue
        into the sub-tree. The sub-tree will be replaced by `func(subtree)`
        in the returned structure (to replace the sub-tree with `None`, use
        the special value `MAP_TO_NONE`).

    When traversing bottom-up:
        If `func(subtree) is None` the traversed sub-tree is returned
        unaltered.
        If `func(subtree) is not None` the sub-tree will be replaced by
        `func(subtree)` in the returned structure (to replace the sub-tree
        with None, use the special value `MAP_TO_NONE`).

    structure: The structure to traverse.
    top_down: If True, parent structures will be visited before their
        children.

Returns:
    The structured output from the traversal.

Raises:
    TypeError: If `func` is not callable.


def cosine_similarity(y_true, y_pred, axis=-1):
    y_true = nn.l2_normalize(y_true, axis=axis)
    y_pred = nn.l2_normalize(y_pred, axis=axis)
    return -math_ops.reduce_sum(y_true * y_pred, axis=axis)

Computes the cosine similarity between labels and predictions.

Note that it is a number between -1 and 1. When it is a negative number
between -1 and 0, 0 indicates orthogonality and values closer to -1
indicate greater similarity. The values closer to 1 indicate greater
dissimilarity. This makes it usable as a loss function in a setting
where you try to maximize the proximity between predictions and
targets. If either `y_true` or `y_pred` is a zero vector, cosine
similarity will be 0 regardless of the proximity between predictions
and targets.

`loss = -sum(l2_norm(y_true) * l2_norm(y_pred))`

Standalone usage:

>>> y_true = [[0., 1.], [1., 1.], [1., 1.]]
>>> y_pred = [[1., 0.], [1., 1.], [-1., -1.]]
>>> loss = tf.keras.losses.cosine_similarity(y_true, y_pred, axis=1)
>>> loss.numpy()
array([-0., -0.999, 0.999], dtype=float32)

Args:
  y_true: Tensor of true targets.
  y_pred: Tensor of predicted targets.
  axis: Axis along which to determine similarity.

Returns:
  Cosine similarity tensor.


def __init__(self, runner: Optional[Union[str, PipelineRunner]]=None, options: Optional[PipelineOptions]=None, argv: Optional[List[str]]=None, display_data: Optional[Dict[str, Any]]=None):
    logging.basicConfig()
    if options is not None:
        if isinstance(options, PipelineOptions):
            saved_runner = options.view_as(StandardOptions).runner
            options.view_as(StandardOptions).runner = None
            self._options = copy.deepcopy(options)
            self._options.view_as(StandardOptions).runner = saved_runner
            options.view_as(StandardOptions).runner = saved_runner
        else:
            raise ValueError('Parameter options, if specified, must be of type PipelineOptions. Received : %r' % options)
    elif argv is not None:
        if isinstance(argv, list):
            self._options = PipelineOptions(argv)
        else:
            raise ValueError('Parameter argv, if specified, must be a list. Received : %r' % argv)
    else:
        self._options = PipelineOptions([])
    FileSystems.set_options(self._options)
    if runner is None:
        runner = self._options.view_as(StandardOptions).runner
        if runner is None:
            runner = StandardOptions.DEFAULT_RUNNER
            logging.info('Missing pipeline option (runner). Executing pipeline using the default runner: %s.', runner)
    if isinstance(runner, str):
        runner = create_runner(runner)
    elif not isinstance(runner, PipelineRunner):
        raise TypeError('Runner %s is not a PipelineRunner object or the name of a registered runner.' % runner)
    if self._options.view_as(SetupOptions).pickle_library == 'default' and runner.default_pickle_library_override():
        logging.info('Runner defaulting to pickling library: %s.', runner.default_pickle_library_override())
        self._options.view_as(SetupOptions).pickle_library = runner.default_pickle_library_override()
    pickler.set_library(self._options.view_as(SetupOptions).pickle_library)
    errors = PipelineOptionsValidator(self._options, runner).validate()
    if errors:
        raise ValueError('Pipeline has validations errors: \n' + '\n'.join(errors))
    if runner.is_fnapi_compatible():
        experiments = self._options.view_as(DebugOptions).experiments or []
        if not 'beam_fn_api' in experiments:
            experiments.append('beam_fn_api')
            self._options.view_as(DebugOptions).experiments = experiments
    self.local_tempdir = tempfile.mkdtemp(prefix='beam-pipeline-temp')
    self.runner = runner
    self.transforms_stack = [AppliedPTransform(None, None, '', None, None, None)]
    self.applied_labels = set()
    self._root_transform().resource_hints = resource_hints_from_options(options)
    self.component_id_map = ComponentIdMap()
    self.contains_external_transforms = False
    self._display_data = display_data or {}
    self._error_handlers = []
    self._annotations_stack = [{}]

Initialize a pipeline object.

Args:
  runner (~apache_beam.runners.runner.PipelineRunner): An object of
    type :class:`~apache_beam.runners.runner.PipelineRunner` that will be
    used to execute the pipeline. For registered runners, the runner name
    can be specified, otherwise a runner object must be supplied.
  options (~apache_beam.options.pipeline_options.PipelineOptions):
    A configured
    :class:`~apache_beam.options.pipeline_options.PipelineOptions` object
    containing arguments that should be used for running the Beam job.
  argv (List[str]): a list of arguments (such as :data:`sys.argv`)
    to be used for building a
    :class:`~apache_beam.options.pipeline_options.PipelineOptions` object.
    This will only be used if argument **options** is :data:`None`.
  display_data (Dict[str, Any]): a dictionary of static data associated
    with this pipeline that can be displayed when it runs.

Raises:
  ValueError: if either the runner or options argument is not
    of the expected type.


def __init__(self, namespace: str):
    self.requests = Metrics.counter(namespace, 'requests')
    self.responses = Metrics.counter(namespace, 'responses')
    self.failures = Metrics.counter(namespace, 'failures')
    self.throttled_requests = Metrics.counter(namespace, 'throttled_requests')
    self.throttled_secs = Metrics.counter(namespace, 'cumulativeThrottlingSeconds')
    self.timeout_requests = Metrics.counter(namespace, 'requests_timed_out')
    self.call_counter = Metrics.counter(namespace, 'call_invocations')
    self.setup_counter = Metrics.counter(namespace, 'setup_counter')
    self.teardown_counter = Metrics.counter(namespace, 'teardown_counter')
    self.backoff_counter = Metrics.counter(namespace, 'backoff_counter')
    self.sleeper_counter = Metrics.counter(namespace, 'sleeper_counter')
    self.should_backoff_counter = Metrics.counter(namespace, 'should_backoff_counter')

Args:
  namespace: Namespace for the metrics.


def find_all_dependencies(dependency_mapping: Dict[str, set], start_entity: Optional[str]=None, initial_dependencies: Optional[set]=None, initial_checked_dependencies: Optional[set]=None, return_parent: bool=False) -> Union[list, set]:
    if initial_dependencies is None and start_entity is not None:
        initial_dependencies = dependency_mapping[start_entity]
    if initial_checked_dependencies is None:
        initial_checked_dependencies = set()
    dependency_queue = deque(initial_dependencies)
    all_dependencies = set()
    all_dependencies_with_parent = []
    checked_dependencies = set(initial_checked_dependencies)
    parents = dict.fromkeys(initial_dependencies, start_entity)
    while len(dependency_queue) > 0:
        current = dependency_queue.popleft()
        if current not in checked_dependencies:
            all_dependencies.add(current)
            all_dependencies_with_parent += [(current, parents[current])]
            if current in dependency_mapping.keys():
                dependency_queue.extend(dependency_mapping[current])
                parents.update(dict.fromkeys(dependency_mapping[current], current))
            checked_dependencies.add(current)
    if not return_parent:
        return all_dependencies
    return all_dependencies_with_parent

Return all the dependencies of the given `start_entity` or `initial_dependencies`. This is basically some kind of
BFS traversal algorithm. It can either start from `start_entity`, or `initial_dependencies`.

Args:
    dependency_mapping (`Dict[str, set]`):
        A mapping from entities (usually function/assignment names), to immediate dependencies. That is, for function names,
        a mapping {"foo": {"bar", "test"}} would indicate that functions `bar` and `test` are immediately called
        in `foo`'s definition.
    start_entity (str | None, *optional*):
        A key of `dependency_mapping`, indicating from which entity to start the search.
    initial_dependencies (set | None, *optional*):
        If `start_entity` is not provided, this can be used as an alternative. In this case, the search will continue
        from all the entities in `initial_dependencies`, if they are in `dependency_mapping`.
    initial_checked_dependencies (set | None, *optional*):
        If provided, entities already present in `initial_checked_dependencies` will not be part of the returned dependencies.
    return_parent (bool, *optional*):
        If `True`, will return a list consisting of tuples (dependency, parent) instead of a simple set of dependencies. Note
        that the order of the items in the list reflects the traversal order. Thus, no parent can ever appear before childs.
Returns:
    A set of all the dependencies, or a list of tuples `(dependency, parent)` if `return_parent=True`.

Example:
Given the following structure in the `modular_xxx.py` file:
```
def foo1():
    pass

def foo2():
    pass

def bar():
    foo1()

def foobar():
    bar()
    foo2()

class MyLayer(SomeOtherModelLayer):
    def forward(...):
        foobar()
```
and the `dependency_mapping` created when visiting the `modular_xxx.py` file, we get:
```
dependency_mapping = {'bar': {'foo1'}, 'foobar': {'bar', 'foo2'}}
find_all_dependencies(dependency_mapping, start_entity='foobar', return_parent=True)
>>> [('bar', 'foobar'), ('foo2', 'foobar'), ('foo1', 'bar')]
```
That is, all the functions needed (and potentially their immediate parent) so that the function to be added
in MyLayer (`foobar`) can work correctly.


def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None, already_has_special_tokens: bool=False) -> List[int]:
    if already_has_special_tokens:
        return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True)
    if not self.add_bos_token:
        return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=False)
    if token_ids_1 is None:
        return [1] + [0] * len(token_ids_0)
    return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1)

Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.

Args:
    token_ids_0 (`List[int]`):
        List of IDs.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.
    already_has_special_tokens (`bool`, *optional*, defaults to `False`):
        Whether or not the token list is already formatted with special tokens for the model.

Returns:
    `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.


def _is_user_included_op(self, op):
    for opname_re in self._parameters.included_opname_re_list:
        if opname_re.match(op.name):
            return True
    for optype_re in self._parameters.included_optype_re_list:
        if optype_re.match(op.type):
            return True
    return False

Checks whether the op is included in the tensor tracer flags.

Args:
  op: tf Operation
Returns:
  True, if the op is included.
  An op is included if:
  - Its op name is given in included_opnames
  - Its op type is given in included_optypes
  - The op is at most _trace_ops_before_included hops before an included op
  - The op is at most _trace_ops_after_included hops after an included op


class SvenssonParameters:
    beta_0: types.RealTensor
    beta_1: types.RealTensor
    beta_2: types.RealTensor
    beta_3: types.RealTensor
    tau_1: types.RealTensor
    tau_2: types.RealTensor

Nelson Seigel Svensson interpolation parameters.

Attributes:
  beta_0: A real `Tensor` of arbitrary shape `batch_shape`.
  beta_1: A real `Tensor` of arbitrary shape `batch_shape`.
  beta_2: A real `Tensor` of arbitrary shape `batch_shape`.
  beta_3: A real `Tensor` of arbitrary shape `batch_shape`.
  tau_1: A real `Tensor` of arbitrary shape `batch_shape`.
  tau_2: A real `Tensor` of arbitrary shape `batch_shape`.


def add_collection_def(meta_graph_def, key, graph=None, export_scope=None, exclude_nodes=None, override_contents=None):
    if graph and (not isinstance(graph, ops.Graph)):
        raise TypeError(f'graph must be of type Graph. Received type: {type(graph)}.')
    if not isinstance(key, str) and (not isinstance(key, bytes)):
        logging.warning('Only collections with string type keys will be serialized. This key has %s', type(key))
        return
    graph = graph or ops.get_default_graph()
    if override_contents:
        collection_list = override_contents
    else:
        collection_list = graph.get_collection(key)
    collection_list = [x for x in collection_list if _should_include_node(x, export_scope, exclude_nodes)]
    if not collection_list:
        return
    try:
        col_def = meta_graph_def.collection_def[key]
        to_proto = ops.get_to_proto_function(key)
        proto_type = ops.get_collection_proto_type(key)
        if to_proto:
            kind = 'bytes_list'
            for x in collection_list:
                proto = to_proto(x, export_scope=export_scope)
                if proto:
                    assert isinstance(proto, proto_type)
                    getattr(col_def, kind).value.append(proto.SerializeToString())
        else:
            kind = _get_kind_name(collection_list[0])
            if kind == 'node_list':
                for x in collection_list:
                    if not export_scope or x.name.startswith(export_scope):
                        getattr(col_def, kind).value.append(ops.strip_name_scope(x.name, export_scope))
            elif kind == 'bytes_list':
                getattr(col_def, kind).value.extend([compat.as_bytes(x) for x in collection_list])
            else:
                getattr(col_def, kind).value.extend([x for x in collection_list])
    except Exception as e:
        logging.warning("Issue encountered when serializing %s.\nType is unsupported, or the types of the items don't match field type in CollectionDef. Note this is a warning and probably safe to ignore.\n%s", key, str(e))
        if key in meta_graph_def.collection_def:
            del meta_graph_def.collection_def[key]
        return

Adds a collection to MetaGraphDef protocol buffer.

Args:
  meta_graph_def: MetaGraphDef protocol buffer.
  key: One of the GraphKeys or user-defined string.
  graph: The `Graph` from which to get collections.
  export_scope: Optional `string`. Name scope to remove.
  exclude_nodes: An iterable of nodes or `string` node names to omit from the
    collection, or None.
  override_contents: An iterable of values to place in the collection,
    ignoring the current values (if set).


def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, cache_kwargs: Optional[Dict[str, Any]]=None) -> Tuple[torch.Tensor, torch.Tensor]:
    if cache_kwargs is None:
        cache_kwargs = {}
    key_states = key_states.to(self.key_cache[layer_idx].dtype)
    value_states = value_states.to(self.value_cache[layer_idx].dtype)
    return _static_cache_update(self.key_cache[layer_idx], self.value_cache[layer_idx], key_states, value_states, cache_kwargs.get('cache_position'))

Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
It is VERY important to index using a tensor, otherwise you introduce a copy to the device.

Parameters:
    key_states (`torch.Tensor`):
        The new key states to cache.
    value_states (`torch.Tensor`):
        The new value states to cache.
    layer_idx (`int`):
        The index of the layer to cache the states for.
    cache_kwargs (`Dict[str, Any]`, `optional`):
        Additional arguments for the cache subclass. The `StaticCache` needs the `cache_position` input
        to know how where to write in the cache.

Return:
    A tuple containing the updated key and value states.


def forward(self, hidden_states: torch.Tensor, original_hidden_states: torch.Tensor, layer_idx: int, attention_mask: Optional[torch.Tensor]=None, past_key_value: Optional[Zamba2HybridDynamicCache]=None, output_attentions: Optional[bool]=False, position_embeddings: Optional[torch.LongTensor]=None, **kwargs: Unpack[FlashAttentionKwargs]) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
    hidden_states = torch.concatenate([hidden_states, original_hidden_states], dim=-1)
    hidden_states = self.input_layernorm(hidden_states)
    hidden_states, self_attn_weights = self.self_attn(hidden_states=hidden_states, layer_idx=layer_idx, attention_mask=attention_mask, past_key_value=past_key_value, output_attentions=output_attentions, position_embeddings=position_embeddings, **kwargs)
    hidden_states = self.pre_ff_layernorm(hidden_states)
    hidden_states = self.feed_forward(hidden_states, layer_idx)
    outputs = (hidden_states,)
    if output_attentions:
        outputs += (self_attn_weights,)
    return outputs

Args:
    hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
    original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
        This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
        concatenated tensor is then used as input of the pre-attention RMSNorm
        (see fig. 2 in https://huggingface.co/papers/2405.16712).
    attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
        `(batch, sequence_length)` where padding elements are indicated by 0.
    past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.


def _AvailableString(variables, verbose=False):
    modules = []
    other = []
    for name, value in variables.items():
        if not verbose and name.startswith('_'):
            continue
        if '-' in name or '/' in name:
            continue
        if inspect.ismodule(value):
            modules.append(name)
        else:
            other.append(name)
    lists = [('Modules', modules), ('Objects', other)]
    list_strs = []
    for name, varlist in lists:
        if varlist:
            items_str = ', '.join(sorted(varlist))
            list_strs.append(f'{name}: {items_str}')
    lists_str = '\n'.join(list_strs)
    return f'Fire is starting a Python REPL with the following objects:\n{lists_str}\n'

Returns a string describing what objects are available in the Python REPL.

Args:
  variables: A dict of the object to be available in the REPL.
  verbose: Whether to include 'hidden' members, those keys starting with _.
Returns:
  A string fit for printing at the start of the REPL, indicating what objects
  are available for the user to use.


def _prepare_4d_attention_mask(attention_mask: Optional[torch.Tensor], sequence_length: int, dtype: torch.dtype, device: torch.device, is_causal: bool=True) -> Optional[torch.Tensor]:
    min_value = torch.finfo(dtype).min if dtype.is_floating_point else torch.iinfo(dtype).min
    if attention_mask is not None:
        attention_mask = attention_mask.view(attention_mask.shape[0], 1, 1, -1)
        attention_mask = attention_mask * min_value
    if is_causal:
        causal_mask = torch.triu(torch.ones((sequence_length, sequence_length), dtype=dtype, device=device) * min_value, diagonal=1)
        causal_mask = causal_mask.view(1, 1, sequence_length, sequence_length)
        if attention_mask is not None:
            attention_mask = torch.minimum(attention_mask, causal_mask)
        else:
            attention_mask = causal_mask
    return attention_mask

Creates 4D attention mask and combines causal and padding masks if needed.

Args:
    attention_mask: Optional tensor of shape (batch_size, seq_length) containing padding mask
    sequence_length: Length of the sequence
    dtype: Data type of the mask
    device: Device of the mask
    is_causal: Whether to apply causal masking

Returns:
    4D attention mask of shape (batch_size, 1, seq_length, seq_length)


def find(self, key, dynamic_default_value=None, name=None):
    with tf.name_scope(name or '%s_lookup_table_find' % self._name):
        key = tf.convert_to_tensor(key, dtype=self._key_dtype, name='key')
        if dynamic_default_value is not None:
            dynamic_default_value = tf.convert_to_tensor(dynamic_default_value, dtype=self._value_dtype, name='default_value')
        value = gen_simple_hash_table_op.examples_simple_hash_table_find(self.resource_handle, key, dynamic_default_value if dynamic_default_value is not None else self._default_value)
    return value

Looks up `key` in a table, outputs the corresponding value.

The `default_value` is used if key not present in the table.

Args:
  key: Key to look up. Must match the table's key_dtype.
  dynamic_default_value: The value to use if the key is missing in the
    table. If None (by default), the `table.default_value` will be used.
  name: A name for the operation (optional).

Returns:
  A tensor containing the value in the same shape as `key` using the
    table's value type.

Raises:
  TypeError: when `key` do not match the table data types.


class RunEnsembleDetector(beam.PTransform[beam.PCollection[NestedKeyedInputT], beam.PCollection[NestedKeyedOutputT]]):

    def __init__(self, ensemble_detector: EnsembleAnomalyDetector):
        self._ensemble_detector = ensemble_detector

    def expand(self, input: beam.PCollection[NestedKeyedInputT]) -> beam.PCollection[NestedKeyedOutputT]:
        model_uuid = f'{self._ensemble_detector._model_id}:{uuid.uuid4().hex[:6]}'
        assert self._ensemble_detector._sub_detectors is not None
        if not self._ensemble_detector._sub_detectors:
            raise ValueError(f'No detectors found at {model_uuid}')
        results = []
        for idx, detector in enumerate(self._ensemble_detector._sub_detectors):
            if isinstance(detector, EnsembleAnomalyDetector):
                results.append(input | f'Run Ensemble Detector at index {idx} ({model_uuid})' >> RunEnsembleDetector(detector))
            elif isinstance(detector, OfflineDetector):
                results.append(input | f'Run Offline Detector at index {idx} ({model_uuid})' >> RunOfflineDetector(detector))
            else:
                results.append(input | f'Run One Detector at index {idx} ({model_uuid})' >> RunOneDetector(detector))
        if self._ensemble_detector._aggregation_strategy is None:
            aggregation_type = 'Simple'
        else:
            aggregation_type = 'Custom'
        ret = results | beam.Flatten() | f'Run {aggregation_type} Aggregation Strategy ({model_uuid})' >> RunAggregationStrategy(self._ensemble_detector._aggregation_strategy, self._ensemble_detector._model_id)
        if self._ensemble_detector._threshold_criterion:
            ret = ret | f'Run Threshold Criterion ({model_uuid})' >> RunThresholdCriterion(self._ensemble_detector._threshold_criterion)
        return ret

Runs an ensemble of anomaly detectors on a PCollection of data.

This PTransform applies an `EnsembleAnomalyDetector` to the input data,
running each sub-detector and aggregating the results.

Args:
  ensemble_detector: The `EnsembleAnomalyDetector` to run.


def unique_fn_name(scope, name):
    return ('%s%s_%s' % (scope, name, ops.uid())).replace('/', '_')

Returns a unique name to use for a control flow function.

Args:
  scope: A name scope string.
  name: An identifier for this function (e.g. "true", "body").

Returns:
  A string, the name to use for the function.


def pop(self, rebuild=True):
    layer = self._layers.pop()
    self.built = False
    self._functional = None
    if rebuild:
        self._maybe_rebuild()
    return layer

Removes the last layer in the model.

Args:
    rebuild: `bool`. Whether to rebuild the model after removing
    the layer. Defaults to `True`.

Returns:
    layer: layer instance.


def get_number_of_image_tokens(self, height: int, width: int, images_kwargs=None):
    min_patches = images_kwargs.get('min_patches', None) or self.min_patches
    max_patches = images_kwargs.get('max_patches', None) or self.max_patches
    patch_size = images_kwargs.get('size', None) or self.size
    crop_to_patches = images_kwargs.get('crop_to_patches', None) or self.crop_to_patches
    num_patches = 1
    if crop_to_patches and max_patches > 1:
        num_columns, num_rows = get_optimal_tiled_canvas((height, width), (patch_size['height'], patch_size['width']), min_patches, max_patches)
        num_patches += num_columns * num_rows
    return num_patches

A utility that returns number patches for a given image size.

Args:
    height (`int`):
        Height of the input image.
    width (`int`):
        Width of the input image.
    images_kwargs (`dict`, *optional*)
        Any kwargs to override defaults of the image processor.
Returns:
    `int`: Number of patches per image.


def _crossed_column(keys, hash_bucket_size, hash_key=None):
    if not hash_bucket_size or hash_bucket_size < 1:
        raise ValueError('hash_bucket_size must be > 1. hash_bucket_size: {}'.format(hash_bucket_size))
    if not keys or len(keys) < 2:
        raise ValueError('keys must be a list with length > 1. Given: {}'.format(keys))
    for key in keys:
        if not isinstance(key, six.string_types) and (not isinstance(key, _CategoricalColumn)):
            raise ValueError('Unsupported key type. All keys must be either string, or categorical column except _HashedCategoricalColumn. Given: {}'.format(key))
        if isinstance(key, _HashedCategoricalColumn):
            raise ValueError('categorical_column_with_hash_bucket is not supported for crossing. Hashing before crossing will increase probability of collision. Instead, use the feature name as a string. Given: {}'.format(key))
    return _CrossedColumn(keys=tuple(keys), hash_bucket_size=hash_bucket_size, hash_key=hash_key)

Returns a column for performing crosses of categorical features.

Crossed features are hashed according to `hash_bucket_size`. Conceptually,
the transformation can be thought of as:
  Hash(cartesian product of features) % `hash_bucket_size`

For example, if the input features are:

* SparseTensor referred by first key:

  ```python
  shape = [2, 2]
  {
      [0, 0]: "a"
      [1, 0]: "b"
      [1, 1]: "c"
  }
  ```

* SparseTensor referred by second key:

  ```python
  shape = [2, 1]
  {
      [0, 0]: "d"
      [1, 0]: "e"
  }
  ```

then crossed feature will look like:

```python
 shape = [2, 2]
{
    [0, 0]: Hash64("d", Hash64("a")) % hash_bucket_size
    [1, 0]: Hash64("e", Hash64("b")) % hash_bucket_size
    [1, 1]: Hash64("e", Hash64("c")) % hash_bucket_size
}
```

Here is an example to create a linear model with crosses of string features:

```python
keywords_x_doc_terms = crossed_column(['keywords', 'doc_terms'], 50K)
columns = [keywords_x_doc_terms, ...]
features = tf.io.parse_example(..., features=make_parse_example_spec(columns))
linear_prediction = linear_model(features, columns)
```

You could also use vocabulary lookup before crossing:

```python
keywords = categorical_column_with_vocabulary_file(
    'keywords', '/path/to/vocabulary/file', vocabulary_size=1K)
keywords_x_doc_terms = crossed_column([keywords, 'doc_terms'], 50K)
columns = [keywords_x_doc_terms, ...]
features = tf.io.parse_example(..., features=make_parse_example_spec(columns))
linear_prediction = linear_model(features, columns)
```

If an input feature is of numeric type, you can use
`categorical_column_with_identity`, or `bucketized_column`, as in the example:

```python
# vertical_id is an integer categorical feature.
vertical_id = categorical_column_with_identity('vertical_id', 10K)
price = numeric_column('price')
# bucketized_column converts numerical feature to a categorical one.
bucketized_price = bucketized_column(price, boundaries=[...])
vertical_id_x_price = crossed_column([vertical_id, bucketized_price], 50K)
columns = [vertical_id_x_price, ...]
features = tf.io.parse_example(..., features=make_parse_example_spec(columns))
linear_prediction = linear_model(features, columns)
```

To use crossed column in DNN model, you need to add it in an embedding column
as in this example:

```python
vertical_id_x_price = crossed_column([vertical_id, bucketized_price], 50K)
vertical_id_x_price_embedded = embedding_column(vertical_id_x_price, 10)
dense_tensor = input_layer(features, [vertical_id_x_price_embedded, ...])
```

Args:
  keys: An iterable identifying the features to be crossed. Each element can
    be either:
    * string: Uses the corresponding feature which must be of string type.
    * `_CategoricalColumn`: Uses the transformed tensor produced by this
      column. Does not support hashed categorical column.
  hash_bucket_size: An int > 1. The number of buckets.
  hash_key: Specify the hash_key that will be used by the `FingerprintCat64`
    function to combine the crosses fingerprints on SparseCrossOp (optional).

Returns:
  A `_CrossedColumn`.

Raises:
  ValueError: If `len(keys) < 2`.
  ValueError: If any of the keys is neither a string nor `_CategoricalColumn`.
  ValueError: If any of the keys is `_HashedCategoricalColumn`.
  ValueError: If `hash_bucket_size < 1`.


def __init__(self, *, pattern: str, word_start: str | None=None, substream_input: str='', substream_output: str='', flush_fn: Callable[[content_api.ProcessorPart], bool] | None=None, remove_from_input_stream: bool=True):
    self._word_start = word_start
    self._pattern = re.compile(pattern, re.DOTALL)
    self._substream_input = substream_input
    self._substream_output = substream_output
    self._flush_fn = flush_fn or (lambda _: False)
    self._remove_from_input_stream = remove_from_input_stream

Extracts text parts from the input stream that match the pattern.

Only considers text parts from the input stream `substream_input`.

See class docstring for more details.

This processor buffers input parts until one of the following happens:

1. the `pattern` is found, then it outputs all parts in the buffer up to the
  end of the match.
2. when the `flush_fn` returns True, then it outputs all parts in the
  buffer.

When `word_start` is set, the following happens:

3. the word_start is found, then it outputs all the parts before the
  `word_start`.
4. `word_start` is not found in buffer_text[-len(word_start):], where
   buffer_text is the concatenation of all the text parts in the buffer. All
   parts whose text is in buffer_text[-len(word_start):] are returned.

While in the buffer, the parts are not output which means they can lead to
delays in the output stream.

To avoid such delays, set the `remove_from_input_stream` to False and/or
define a `flush_fn` that returns True often to discard the parts in the
buffer frequently.

Args:
  pattern: pattern to match a text to extract into a part. When
    `remove_from_input_stream` is True, the matched text will be removed
    from the stream and will be replaced by a single extracted part. The
    parts before and after this match will be returned as is. Note that
    re.DOTALL is used to match newlines.
  word_start: text to match the start of the text that needs to be captured.
    `word_start` is not a regular expression but a plain string that will be
    matched exactly. `word_start` should be a substring of the pattern and
    should indicate that the pattern is about to be matched. Whenever
    `word_start` is found, the parts after it will be buffered (not
    returned) until the pattern is found, the `flush_fn` returns True, or
    the input stream is exhausted. When set to None (default), this logic is
    not applied.
  substream_input: name of the substream to use for the input part.
  substream_output: name of the substream to use for the extracted part.
  flush_fn: function to check when to reset the buffer and yield all the
    parts in the buffer. The part where `flush_fn` returns True will be
    returned as is and will not be matched against the pattern.
  remove_from_input_stream: if True, the processor will remove the matched
    parts from the input stream. If False, the input stream will be
    preserved and the parts will be returned as is quickly. The processor
    will output into its `substream_output` substream once a match is found.


def _delete_minibatch(self, bucket, keys):
    request = messages.DeleteBatchRequest(bucket, keys)
    results = {}
    try:
        response = self.client.delete_batch(request)
        for key in response.deleted:
            results[bucket, key] = None
        for key, error in zip(response.failed, response.errors):
            results[bucket, key] = error
    except messages.S3ClientError as e:
        for key in keys:
            results[bucket, key] = e
    return results

A helper method. Boto3 allows batch deletions
for files within the same bucket.

Args:
  bucket: String bucket name
  keys: List of keys to be deleted in the bucket

Returns: dict of the form {(bucket, key): error}, where error is None if the
operation succeeded


def __init__(self, axis=-1, **kwargs):
    super(Concatenate, self).__init__(**kwargs)
    self.axis = axis
    self.supports_masking = True
    self._reshape_required = False


def cumprod(self: EventSetOrNode, sampling: Optional[EventSetOrNode]=None) -> EventSetOrNode:
    from temporian.core.operators.window.moving_product import cumprod
    return cumprod(self, sampling=sampling)

Computes the cumulative product of values over each feature in an
[`EventSet`][temporian.EventSet].

This operation only supports floating-point features.

Missing (NaN) values are not accounted for. The output will be NaN until
the input contains at least one numeric value.

Warning: The `cumprod` function leverages an infinite window length for
its calculations, which may lead to considerable computational overhead
with increasing dataset sizes.

Example:
    ```python
    >>> a = tp.event_set(
    ...     timestamps=[0, 1, 2, 3],
    ...     features={"value": [1.0, 2.0, 10.0, 12.0]},
    ... )

    >>> b = a.cumprod()
    >>> b
    indexes: ...
        (4 events):
            timestamps: [0. 1. 2. 3.]
            'value': [  1.   2.  20. 240.]
    ...

    ```

Examples with sampling:
    ```python
    >>> a = tp.event_set(
    ...     timestamps=[0, 1, 2, 5, 6, 7],
    ...     features={"value": [1, 2, 10, 12, np.nan, 2]},
    ... )

    >>> # Cumulative product at 5 and 10
    >>> b = tp.event_set(timestamps=[5, 10])
    >>> c = a.cumprod(sampling=b)
    >>> c
    indexes: ...
        (2 events):
            timestamps: [ 5. 10.]
            'value': [240. 480.]
    ...

    >>> # Product all values in the EventSet
    >>> c = a.cumprod(sampling=a.end())
    >>> c
    indexes: ...
        (1 events):
            timestamps: [7.]
            'value': [480.]
    ...

    ```

Args:
    sampling: Timestamps to sample the sliding window's value at. If not
        provided, timestamps in the input are used.

Returns:
    Cumulative product of each feature.