def forward(self, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, position_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[List[torch.FloatTensor]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, image_pixel_values: Optional[torch.FloatTensor]=None, image_sizes: Optional[torch.LongTensor]=None, image_attention_mask=None, audio_input_features: Optional[torch.FloatTensor]=None, audio_embed_sizes=None, audio_attention_mask=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, cache_position: Optional[torch.LongTensor]=None, logits_to_keep: Union[int, torch.Tensor]=0, **kwargs) -> CausalLMOutputWithPast:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    outputs: BaseModelOutputWithPast = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, image_pixel_values=image_pixel_values, image_sizes=image_sizes, image_attention_mask=image_attention_mask, audio_input_features=audio_input_features, audio_embed_sizes=audio_embed_sizes, audio_attention_mask=audio_attention_mask, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, cache_position=cache_position, **kwargs)
    hidden_states = outputs.last_hidden_state
    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
    logits = self.lm_head(hidden_states[:, slice_indices, :])
    loss = None
    if labels is not None:
        loss = self.loss_function(logits, labels, self.vocab_size)
    return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

image_pixel_values (`torch.FloatTensor`, *optional*):
    If the input contains images, these correspond to the pixel values after transformations (as returned by
    the Processor)
image_sizes (`torch.LongTensor`, *optional*):
    If the input contains images, these correspond to size of each image.
image_attention_mask (`torch.LongTensor`, *optional*):
    Attention mask for the images.
audio_input_features (`torch.FloatTensor`, *optional*):
    If the input contains audio samples, these correspond to the values after transformation (as returned by
    the Processor).
audio_embed_sizes (`torch.Tensor`, *optional*):
    Size of the audio inputs.
audio_attention_mask (`torch.Tensor, *optional*):
    Attention mask for the audio inputs.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:
```python
>>> from transformers import AutoTokenizer, Phi4MultimodalForCausalLM
>>> model = Phi4MultimodalForCausalLM.from_pretrained("TBA")
>>> tokenizer = AutoTokenizer.from_pretrained("TBA")
>>> prompt = "This is an example script ."
>>> inputs = tokenizer(prompt, return_tensors="pt")
>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum'
```


def _rpc(self, rpc_func_name, *args, **kwargs):
    try:
        self.check_server_proc_running()
    except Exception:
        self.log.error('Server process running check failed, skip sending RPC method(%s).', rpc_func_name)
        raise
    with self._lock:
        rpc_id = next(self._counter)
        request = self._gen_rpc_request(rpc_id, rpc_func_name, *args, **kwargs)
        self.log.debug('Sending RPC request %s.', request)
        response = self.send_rpc_request(request)
        self.log.debug('RPC request sent.')
        if self.verbose_logging or _MAX_RPC_RESP_LOGGING_LENGTH >= len(response):
            self.log.debug('Snippet received: %s', response)
        else:
            self.log.debug('Snippet received: %s... %d chars are truncated', response[:_MAX_RPC_RESP_LOGGING_LENGTH], len(response) - _MAX_RPC_RESP_LOGGING_LENGTH)
    response_decoded = self._decode_response_string_and_validate_format(rpc_id, response)
    return self._handle_rpc_response(rpc_func_name, response_decoded)

Sends an RPC to the server.

Args:
  rpc_func_name: str, the name of the snippet function to execute on the
    server.
  *args: any, the positional arguments of the RPC request.
  **kwargs: any, the keyword arguments of the RPC request.

Returns:
  The result of the RPC.

Raises:
  errors.ProtocolError: something went wrong when exchanging data with the
    server.
  errors.ApiError: the RPC went through, however executed with errors.


class TFMaskedLMOutput(ModelOutput):
    loss: tf.Tensor | None = None
    logits: Optional[tf.Tensor] = None
    hidden_states: Tuple[tf.Tensor] | None = None
    attentions: Tuple[tf.Tensor] | None = None

Base class for masked language models outputs.

Args:
    loss (`tf.Tensor` of shape `(n,)`, *optional*, where n is the number of non-masked labels, returned when `labels` is provided):
        Masked language modeling (MLM) loss.
    logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
        `(batch_size, sequence_length, hidden_size)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs.
    attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.


class ImageClassifierOutput(ModelOutput):
    loss: Optional[torch.FloatTensor] = None
    logits: Optional[torch.FloatTensor] = None
    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None

Base class for outputs of image classification models.

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states
        (also called feature maps) of the model at the output of each stage.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.


def forward(self, hidden_states: torch.FloatTensor, attention_mask: Optional[torch.Tensor]=None, head_mask: Optional[torch.Tensor]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None) -> Union[Tuple, BaseModelOutput]:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    if attention_mask is not None:
        attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)
    hidden_states = self.layer_norm(hidden_states)
    hidden_states = self.dropout(hidden_states)
    position_bias = self.embed_positions(hidden_states)
    synced_gpus = is_deepspeed_zero3_enabled() or is_fsdp_managed_module(self)
    all_hidden_states = () if output_hidden_states else None
    all_self_attentions = () if output_attentions else None
    if head_mask is not None:
        if head_mask.size()[0] != len(self.layers):
            raise ValueError(f'The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}.')
    for idx, encoder_layer in enumerate(self.layers):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)
        skip_the_layer = False
        if self.training:
            dropout_probability = torch.rand([])
            skip_the_layer = dropout_probability < self.layerdrop
        if not skip_the_layer or synced_gpus:
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(encoder_layer.__call__, hidden_states, attention_mask, head_mask[idx] if head_mask is not None else None, position_bias, output_attentions)
            else:
                layer_outputs = encoder_layer(hidden_states, attention_mask=attention_mask, position_bias=position_bias, layer_head_mask=head_mask[idx] if head_mask is not None else None, output_attentions=output_attentions)
            hidden_states = layer_outputs[0]
        if skip_the_layer:
            layer_outputs = (None, None)
        if output_attentions:
            all_self_attentions = all_self_attentions + (layer_outputs[1],)
    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)
    if not return_dict:
        return tuple((v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None))
    return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_self_attentions)

Args:
    hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`):
        Features extracted from the speech or text input by the encoder prenet.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing convolution and attention on padding token indices. Mask values selected in
        `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.

        [What are attention masks?](../glossary#attention-mask)
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
        Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

        - 1 indicates the head is **not masked**,
        - 0 indicates the head is **masked**.

    output_hidden_states (`bool`, *optional*):
        Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
        for more detail.
    return_dict (`bool`, *optional*):
        Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.


def _convert_to_sparse_tensor(sp_input):
    if isinstance(sp_input, sparse_tensor.SparseTensorValue):
        return sparse_tensor.SparseTensor.from_value(sp_input)
    if not isinstance(sp_input, sparse_tensor.SparseTensor):
        raise TypeError('Input must be a SparseTensor.')
    return sp_input

Convert `sp_input` to `SparseTensor` and return it.

Args:
  sp_input: `SparseTensor` or `SparseTensorValue`.

Returns:
  `sp_input` converted to `SparseTensor`.

Raises:
  ValueError: if `sp_input` is neither `SparseTensor` nor `SparseTensorValue`.


def fuse_resize_and_conv(input_graph_def: graph_pb2.GraphDef, output_node_names: Sequence[str]) -> graph_pb2.GraphDef:
    input_node_map = {}
    for node in input_graph_def.node:
        if node.name not in input_node_map:
            input_node_map[node.name] = node
        else:
            raise ValueError('Duplicate node names detected for ', node.name)
    node_reference_count = collections.defaultdict(int)
    for node in input_graph_def.node:
        for input_name in node.input:
            stripped_name = node_name_from_input(input_name)
            node_reference_count[stripped_name] += 1
    for output_name in output_node_names:
        node_reference_count[output_name] += 1
    new_ops = []
    for node in input_graph_def.node:
        if node.op != 'Conv2D':
            continue
        conv_op = node
        input_op = node_from_map(input_node_map, conv_op.input[0])
        if input_op.op == 'MirrorPad':
            mirror_pad_op = input_op
            resize_op = node_from_map(input_node_map, mirror_pad_op.input[0])
            if resize_op.op != 'ResizeBilinear':
                resize_op = None
        else:
            mirror_pad_op = None
            if input_op.op == 'ResizeBilinear':
                resize_op = input_op
            else:
                resize_op = None
        if not mirror_pad_op and (not resize_op):
            continue
        node_reference_count[conv_op.name] = 0
        if mirror_pad_op:
            node_reference_count[mirror_pad_op.name] -= 1
        if resize_op:
            node_reference_count[resize_op.name] -= 1
        fused_conv_op = node_def_pb2.NodeDef()
        if resize_op:
            fused_conv_op.op = 'FusedResizeAndPadConv2D'
        else:
            fused_conv_op.op = 'FusedPadConv2D'
        fused_conv_op.name = conv_op.name
        if mirror_pad_op:
            mirror_paddings_name = mirror_pad_op.input[1]
            mirror_paddings_mode = mirror_pad_op.attr['mode']
        else:
            paddings_op = node_def_pb2.NodeDef()
            paddings_op.op = 'Const'
            paddings_op.name = conv_op.name + '_dummy_paddings'
            paddings_op.attr['dtype'].CopyFrom(attr_value_pb2.AttrValue(type=dtypes.int32.as_datatype_enum))
            paddings_op.attr['value'].CopyFrom(attr_value_pb2.AttrValue(tensor=tensor_util.make_tensor_proto([0, 0, 0, 0, 0, 0, 0, 0], dtypes.int32, [4, 2])))
            new_ops.extend([paddings_op])
            mirror_paddings_name = paddings_op.name
            mirror_paddings_mode = attr_value_pb2.AttrValue(s=b'REFLECT')
        if resize_op:
            fused_conv_op.input.extend([resize_op.input[0], resize_op.input[1], mirror_paddings_name, conv_op.input[1]])
            fused_conv_op.attr['resize_align_corners'].CopyFrom(resize_op.attr['align_corners'])
        else:
            fused_conv_op.input.extend([mirror_pad_op.input[0], mirror_paddings_name, conv_op.input[1]])
        fused_conv_op.attr['T'].CopyFrom(conv_op.attr['T'])
        fused_conv_op.attr['mode'].CopyFrom(mirror_paddings_mode)
        fused_conv_op.attr['strides'].CopyFrom(conv_op.attr['strides'])
        fused_conv_op.attr['padding'].CopyFrom(conv_op.attr['padding'])
        new_ops.extend([fused_conv_op])
    result_graph_def = graph_pb2.GraphDef()
    for node in input_graph_def.node:
        if node_reference_count[node.name] < 1:
            continue
        new_node = node_def_pb2.NodeDef()
        new_node.CopyFrom(node)
        result_graph_def.node.extend([new_node])
    result_graph_def.node.extend(new_ops)
    return result_graph_def

Merges preceding resize and mirror pad ops into a specialized convolution.

There's a common pattern of enlarging the input to a convolution using a
resize operation, and also using MirrorPad to extend the boundaries to that
zero edge pixels don't bleed inwards when convolving. This routine looks for
that pattern of operations, and fuses them together into a Conv2DWithResizeOp.

Args:
  input_graph_def: A GraphDef containing a model.
  output_node_names: A list of names of the nodes that produce the final
    results.

Returns:
  Modified graph with resize and pad ops merged.

Raises:
  ValueError: If the graph is badly formed with duplicate node names.


def py_func_from_autograph(python_func, autograph_options=None):
    _, original_func = tf_decorator.unwrap(python_func)

    def autograph_handler(*args, **kwargs):
        """Calls a converted version of original_func."""
        try:
            return api.converted_call(original_func, args, kwargs, options=converter.ConversionOptions(recursive=True, optional_features=autograph_options, user_requested=True))
        except Exception as e:
            if hasattr(e, 'ag_error_metadata'):
                raise e.ag_error_metadata.to_exception(e)
            else:
                raise
    converted_func = tf_decorator.make_decorator(original_func, autograph_handler)
    return tf_decorator.rewrap(python_func, original_func, converted_func)

Compile a python function using autograph, for use with FuncGraph.

Args:
  python_func: the Python function to compile.
  autograph_options: additional knobs to control when `autograph=True`.
    See https://www.tensorflow.org/guide/autograph for more information.
Returns:
  python_func, converted using autograph.


class ChineseCLIPVisionEncoder(nn.Module):

    def __init__(self, config: ChineseCLIPConfig):
        super().__init__()
        self.config = config
        self.layers = nn.ModuleList([ChineseCLIPVisionLayer(config) for _ in range(config.num_hidden_layers)])
        self.gradient_checkpointing = False

    def forward(self, inputs_embeds, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None) -> Union[Tuple, BaseModelOutput]:
        """
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None
        hidden_states = inputs_embeds
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(encoder_layer.__call__, hidden_states, output_attentions)
            else:
                layer_outputs = encoder_layer(hidden_states, output_attentions=output_attentions)
            hidden_states = layer_outputs[0]
            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        if not return_dict:
            return tuple((v for v in [hidden_states, encoder_states, all_attentions] if v is not None))
        return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)

Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`ChineseCLIPVisionEncoderLayer`].

Args:
    config: ChineseCLIPConfig


def convert_old_keys_to_new_keys(state_dict_keys: Optional[dict]=None):
    output_dict = {}
    if state_dict_keys is not None:
        old_text = '\n'.join(state_dict_keys)
        new_text = old_text
        for pattern, replacement in ORIGINAL_TO_CONVERTED_KEY_MAPPING.items():
            if replacement is None:
                new_text = re.sub(pattern, '', new_text)
                continue
            new_text = re.sub(pattern, replacement, new_text)
        output_dict = dict(zip(old_text.split('\n'), new_text.split('\n')))
    return output_dict

Converts old keys to new keys using the mapping and dynamically removes the 'ijepa.' prefix if necessary.

Args:
    state_dict_keys (dict): The keys from the state_dict to convert.

Returns:
    dict: A mapping from old keys to new keys.


def get_entries(attr_name):
    assert attr_name in ['inputs', 'outputs']
    entries = {}
    for op_type in ops._gradient_registry.list():
        if op_type in _EXCLUDED_OPS:
            continue
        num_values = _get_num_inputs_outputs(op_type)[0 if attr_name == 'inputs' else 1]
        gradient_fn = ops._gradient_registry.lookup(op_type)
        if gradient_fn is None:
            if num_values != -1:
                entries[op_type] = '{"%s"},' % op_type
            continue
        used_tensors = _live_tensors(gradient_fn, attr_name=attr_name)
        if used_tensors is _ALL:
            continue
        elif not used_tensors:
            entries[op_type] = '{"%s"},' % op_type
        else:
            all_tensors = set(range(num_values))
            unused_tensors = all_tensors - used_tensors
            if unused_tensors:
                unused_tensor_list = sorted(list(unused_tensors))
                entries[op_type] = '{"%s", %d, {%s}},' % (op_type, len(unused_tensor_list), ', '.join((str(i) for i in unused_tensor_list)))
    return entries

Returns the dict of entries.

Each entry is of the form {op_name, {true|false, indices}}

true: All values are unused.
false: `indices` are the only unused indices.

Note: ops for which all values are used are not printed.

Args:
  attr_name: inputs or outputs.

Returns:
  A dict from op_type to formatted entry in the dict.


def __init__(self, feature_config: Union[tpu_embedding_v2_utils.FeatureConfig, Iterable], optimizer: Optional[tpu_embedding_v2_utils._Optimizer], pipeline_execution_with_tensor_core: bool=False):
    self._strategy = distribute_lib.get_strategy()
    self._using_tpu = isinstance(self._strategy, (tpu_strategy.TPUStrategy, tpu_strategy.TPUStrategyV2))
    self._pipeline_execution_with_tensor_core = pipeline_execution_with_tensor_core
    self._feature_config = feature_config
    self._output_shapes = []
    for feature in nest.flatten(feature_config):
        self._output_shapes.append(feature.output_shape)
    device_assignment = getattr(self._strategy.extended, '_device_assignment', None)
    self._num_cores_per_replica = device_assignment.num_cores_per_replica if device_assignment else None
    self._table_config = []
    for feature in nest.flatten(feature_config):
        if feature.table not in self._table_config:
            self._table_config.append(feature.table)
    table_names = []
    for i, table in enumerate(self._table_config):
        if table.optimizer is None:
            table.optimizer = optimizer
        if (table.optimizer is not None or self._using_tpu) and (not isinstance(table.optimizer, tpu_embedding_v2_utils._Optimizer)):
            raise ValueError('{} is an unsupported optimizer class. Please pass an instance of one of the optimizer classes under tf.tpu.experimental.embedding.'.format(type(table.optimizer)))
        if table.name is None:
            table.name = 'table_{}'.format(i)
        if table.name in table_names:
            raise ValueError(f'Tables must have a unique name. Multiple tables with name {table.name} found.')
        table_names.append(table.name)
    if self._using_tpu:
        self._dynamic_learning_rates = []
        for table in self._table_config:
            if callable(table.optimizer.learning_rate) and table.optimizer.learning_rate not in self._dynamic_learning_rates:
                self._dynamic_learning_rates.append(table.optimizer.learning_rate)
        self._hosts = tpu_embedding_v2_utils.get_list_of_hosts(self._strategy)
    self._built = False
    self._verify_output_shapes_on_enqueue = True

Creates the TPUEmbedding mid level API object.

```python
strategy = tf.distribute.TPUStrategy(...)
with strategy.scope():
  embedding = tf.tpu.experimental.embedding.TPUEmbedding(
      feature_config=tf.tpu.experimental.embedding.FeatureConfig(
          table=tf.tpu.experimental.embedding.TableConfig(
              dim=...,
              vocabulary_size=...)))
```

Args:
  feature_config: A nested structure of
    `tf.tpu.experimental.embedding.FeatureConfig` configs.
  optimizer: An instance of one of `tf.tpu.experimental.embedding.SGD`,
    `tf.tpu.experimental.embedding.Adagrad` or
    `tf.tpu.experimental.embedding.Adam`. When not created under
    TPUStrategy may be set to None to avoid the creation of the optimizer
    slot variables, useful for optimizing memory consumption when exporting
    the model for serving where slot variables aren't needed.
  pipeline_execution_with_tensor_core: If True, the TPU embedding
    computations will overlap with the TensorCore computations (and hence
    will be one step old). Set to True for improved performance.

Raises:
  ValueError: If optimizer is not one of tf.tpu.experimental.embedding.(SGD,
  Adam or Adagrad) or None when created under a TPUStrategy.


def vqt(input_qhbm: qhbm.QHBM, target_hamiltonian: Union[tf.Tensor, hamiltonian.Hamiltonian], beta: tf.Tensor):

    def f_vqt(bitstrings):
        h_expectations = tf.squeeze(input_qhbm.q_inference.expectation(bitstrings, target_hamiltonian), 1)
        beta_h_expectations = beta * h_expectations
        energies = tf.stop_gradient(input_qhbm.e_inference.energy(bitstrings))
        return beta_h_expectations - energies
    average_expectation = input_qhbm.e_inference.expectation(f_vqt)
    current_partition = tf.stop_gradient(input_qhbm.e_inference.log_partition())
    return average_expectation - current_partition

Computes the VQT loss of a given QHBM and Hamiltonian.

This function is differentiable within a `tf.GradientTape` scope.

Args:
  input_qhbm: Inference methods for the model.
  target_hamiltonian: The Hamiltonian whose thermal state is to be learned. If
    it is a `tf.Tensor`, it is of type `tf.string` with shape [1], result of
    calling `tfq.convert_to_tensor` on a list of `cirq.PauliSum`, `[op]`.
    Otherwise, a Hamiltonian.
  beta: A scalar `tf.Tensor` which is the inverse temperature at which the
    loss is calculated.

Returns:
  The VQT loss.


class ErrorMatcher:
    ERROR_RE = re.compile('^(?P<code>(\\w+-)+\\w+)(\\[(?P<mark>.+)\\])?((?P<cmp>([!=]=|[<>]=?))(?P<version>\\d+\\.\\d+))?$')

    def __init__(self, src):
        self.errorlog = None
        self.marks = None
        self.expected = self._parse_comments(src)

    def _fail(self, msg):
        if self.marks:
            self.errorlog.print_to_stderr()
        raise AssertionError(msg)

    def has_error(self):
        return self.errorlog and self.errorlog.has_error()

    def assert_errors_match_expected(self, errorlog):
        """Matches expected errors against the errorlog, populating self.marks."""

        def _format_error(line, code, mark=None):
            formatted = 'Line %d: %s' % (line, code)
            if mark:
                formatted += f'[{mark}]'
            return formatted
        self.errorlog = errorlog
        self.marks = {}
        expected = copy.deepcopy(self.expected)
        for error in self.errorlog.unique_sorted_errors():
            errs = expected[error.line]
            for i, (code, mark) in enumerate(errs):
                if code == error.name:
                    if mark:
                        self.marks[mark] = error
                    del errs[i]
                    break
            else:
                if errs:
                    code, mark = errs[0]
                    exp = _format_error(error.line, code, mark)
                    actual = _format_error(error.line, error.name)
                    self._fail(f'Error does not match:\nExpected: {exp}\nActual: {actual}')
                else:
                    self._fail(f'Unexpected error:\n{error}')
        leftover_errors = []
        for line in sorted(expected):
            leftover_errors.extend((_format_error(line, code, mark) for code, mark in expected[line]))
        if leftover_errors:
            self._fail('Errors not found:\n' + '\n'.join(leftover_errors))

    def _assert_error_messages(self, matchers):
        """Assert error messages."""
        assert self.marks is not None
        for mark, error in self.marks.items():
            try:
                matcher = matchers.pop(mark)
            except KeyError:
                self._fail(f'No matcher for mark {mark}')
            if not matcher.match(error.message):
                self._fail('Bad error message for mark %s: expected %r, got %r' % (mark, matcher, error.message))
        if matchers:
            self._fail(f'Marks not found in code: {', '.join(matchers)}')

    def assert_diagnostic_messages(self, matchers):
        """Assert error messages."""
        assert self.marks is not None
        for mark, error in self.marks.items():
            try:
                matcher = matchers.pop(mark)
            except KeyError:
                self._fail(f'No matcher for mark {mark}')
            if isinstance(matcher, str):
                match = matcher.__eq__
            else:
                match = matcher.match
            error_as_string = error.as_string()
            if not match(error_as_string):
                self._fail('Bad error message for mark %s: expected %r, got %r' % (mark, matcher, error_as_string))
        if matchers:
            self._fail(f'Marks not found in code: {', '.join(matchers)}')

    def assert_error_regexes(self, expected_regexes):
        matchers = {k: RegexMatcher(v) for k, v in expected_regexes.items()}
        self._assert_error_messages(matchers)

    def assert_error_sequences(self, expected_sequences):
        matchers = {k: SequenceMatcher(v) for k, v in expected_sequences.items()}
        self._assert_error_messages(matchers)

    def assert_diagnostic_regexes(self, expected_diagnostic_regexes):
        matchers = {k: RegexMatcher(v) for k, v in expected_diagnostic_regexes.items()}
        self.assert_diagnostic_messages(matchers)

    def _parse_comment(self, comment):
        comment = comment.strip()
        error_match = self.ERROR_RE.fullmatch(comment)
        if not error_match:
            return None
        version_cmp = error_match.group('cmp')
        if version_cmp:
            version = utils.version_from_string(error_match.group('version'))
            actual_version = sys.version_info[:2]
            if not slots.COMPARES[version_cmp](actual_version, version):
                return None
        return (error_match.group('code'), error_match.group('mark'))

    def _parse_comments(self, src):
        """Parse comments."""
        src = io.StringIO(src)
        expected = collections.defaultdict(list)
        used_marks = set()
        for tok, s, (line, _), _, _ in tokenize.generate_tokens(src.readline):
            if tok != tokenize.COMMENT:
                continue
            for comment in s.split('#'):
                parsed_comment = self._parse_comment(comment)
                if parsed_comment is None:
                    continue
                code, mark = parsed_comment
                if mark:
                    if mark in used_marks:
                        self._fail(f'Mark {mark} already used')
                    used_marks.add(mark)
                expected[line].append((code, mark))
        return expected

An ErrorLog matcher to help with test assertions.

Takes the source code as an init argument, and constructs two dictionaries
holding parsed comment directives.

Attributes:
  errorlog: The errorlog being matched against
  marks: { mark_name : errors.Error object }
  expected: { line number : sequence of expected error codes and mark names }

Adds an assertion matcher to match errorlog.errors against a list of expected
errors of the form [(line number, error code, message regex)].

See tests/test_base_test.py for usage examples.


def get_source_code_system(enum_value_descriptor: descriptor.EnumValueDescriptor) -> Optional[str]:
    return get_value_for_annotation_extension(enum_value_descriptor, annotations_pb2.source_code_system)

Returns the value associated with the source_code_system annotation.

Args:
  enum_value_descriptor: An EnumValueDescriptor describing a FHIR Code for
    which to return a source code system.

Returns:
  The string value of the source code system, if one exists. Otherwise,
  returns None.

Raises:
  ValueError: Unable to retrieve options for type: <type>.


def forward(self, input_ids: Optional[torch.LongTensor]=None, attention_mask: Optional[torch.Tensor]=None, position_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[List[torch.FloatTensor]]=None, inputs_embeds: Optional[torch.FloatTensor]=None, labels: Optional[torch.LongTensor]=None, use_cache: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, output_router_logits: Optional[bool]=None, cache_position: Optional[torch.LongTensor]=None, logits_to_keep: Union[int, torch.Tensor]=0, **kwargs: Unpack[KwargsForCausalLM]) -> MoeCausalLMOutputWithPast:
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_router_logits = output_router_logits if output_router_logits is not None else self.config.output_router_logits
    output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    outputs: MoeModelOutputWithPast = self.model(input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, output_router_logits=output_router_logits, cache_position=cache_position, **kwargs)
    hidden_states = outputs.last_hidden_state
    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
    logits = self.lm_head(hidden_states[:, slice_indices, :])
    loss = None
    if labels is not None:
        loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
    aux_loss = None
    if output_router_logits:
        aux_loss = load_balancing_loss_func(outputs.router_logits, self.num_experts, self.num_experts_per_tok, attention_mask)
        if labels is not None:
            loss += self.router_aux_loss_coef * aux_loss.to(loss.device)
    return MoeCausalLMOutputWithPast(loss=loss, aux_loss=aux_loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, router_logits=outputs.router_logits)

labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MixtralForCausalLM

>>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```


def bloom_gelu_back(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
    x = x[0]
    tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
    ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out)
    return ff * g

gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
0.3989423 * x * torch.exp(-0.5 * x * x)

Args:
    g (`torch.tensor`):
        gradient output tensor
    x (`torch.tensor`):
        input tensor


def set_options(cls, pipeline_options):
    cls._pipeline_options = pipeline_options

Set filesystem options.

Args:
  pipeline_options: Instance of ``PipelineOptions``.


def generate_dummy_inputs(self, tokenizer: PreTrainedTokenizer, batch_size: int=-1, seq_length: int=-1, is_pair: bool=False, framework: Optional[TensorType]=None) -> Mapping[str, Any]:
    input_dict = super().generate_dummy_inputs(tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework)
    box = [48, 84, 73, 128]
    if not framework == TensorType.PYTORCH:
        raise NotImplementedError('Exporting LayoutLM to ONNX is currently only supported for PyTorch.')
    if not is_torch_available():
        raise ValueError('Cannot generate dummy inputs without PyTorch installed.')
    import torch
    batch_size, seq_length = input_dict['input_ids'].shape
    input_dict['bbox'] = torch.tensor([*[box] * seq_length]).tile(batch_size, 1, 1)
    return input_dict

Generate inputs to provide to the ONNX exporter for the specific framework

Args:
    tokenizer: The tokenizer associated with this model configuration
    batch_size: The batch size (int) to export the model for (-1 means dynamic axis)
    seq_length: The sequence length (int) to export the model for (-1 means dynamic axis)
    is_pair: Indicate if the input is a pair (sentence 1, sentence 2)
    framework: The framework (optional) the tokenizer will generate tensor for

Returns:
    Mapping[str, Tensor] holding the kwargs to provide to the model's forward function


class MaskFormerForInstanceSegmentationOutput(ModelOutput):
    loss: Optional[torch.FloatTensor] = None
    class_queries_logits: Optional[torch.FloatTensor] = None
    masks_queries_logits: Optional[torch.FloatTensor] = None
    auxiliary_logits: Optional[torch.FloatTensor] = None
    encoder_last_hidden_state: Optional[torch.FloatTensor] = None
    pixel_decoder_last_hidden_state: Optional[torch.FloatTensor] = None
    transformer_decoder_last_hidden_state: Optional[torch.FloatTensor] = None
    encoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    pixel_decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    transformer_decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    attentions: Optional[Tuple[torch.FloatTensor]] = None

Class for outputs of [`MaskFormerForInstanceSegmentation`].

This output can be directly passed to [`~MaskFormerImageProcessor.post_process_semantic_segmentation`] or or
[`~MaskFormerImageProcessor.post_process_instance_segmentation`] or
[`~MaskFormerImageProcessor.post_process_panoptic_segmentation`] depending on the task. Please, see
[`~MaskFormerImageProcessor] for details regarding usage.

Args:
    loss (`torch.Tensor`, *optional*):
        The computed loss, returned when labels are present.
    class_queries_logits (`torch.FloatTensor`):
        A tensor of shape `(batch_size, num_queries, num_labels + 1)` representing the proposed classes for each
        query. Note the `+ 1` is needed because we incorporate the null class.
    masks_queries_logits (`torch.FloatTensor`):
        A tensor of shape `(batch_size, num_queries, height, width)` representing the proposed masks for each
        query.
    encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Last hidden states (final feature map) of the last stage of the encoder model (backbone).
    pixel_decoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Last hidden states (final feature map) of the last stage of the pixel decoder model (FPN).
    transformer_decoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Last hidden states (final feature map) of the last stage of the transformer decoder model.
    encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the encoder
        model at the output of each stage.
    pixel_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the pixel
        decoder model at the output of each stage.
    transformer_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the transformer decoder at the output
        of each stage.
    hidden_states `tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` containing `encoder_hidden_states`, `pixel_decoder_hidden_states` and
        `decoder_hidden_states`.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights from Detr's decoder after the attention softmax, used to compute the
        weighted average in the self-attention heads.


def __init__(self, on_ui_exit=None, config=None):
    self._on_ui_exit = on_ui_exit
    self._command_handler_registry = debugger_cli_common.CommandHandlerRegistry()
    self._tab_completion_registry = debugger_cli_common.TabCompletionRegistry()
    self._tab_completion_registry.register_tab_comp_context([''], self.CLI_EXIT_COMMANDS + [debugger_cli_common.CommandHandlerRegistry.HELP_COMMAND] + debugger_cli_common.CommandHandlerRegistry.HELP_COMMAND_ALIASES)
    self._config = config or cli_config.CLIConfig()
    self._config_argparser = argparse.ArgumentParser(description='config command', usage=argparse.SUPPRESS)
    subparsers = self._config_argparser.add_subparsers()
    set_parser = subparsers.add_parser('set')
    set_parser.add_argument('property_name', type=str)
    set_parser.add_argument('property_value', type=str)
    set_parser = subparsers.add_parser('show')
    self.register_command_handler('config', self._config_command_handler, self._config_argparser.format_help(), prefix_aliases=['cfg'])

Constructor of the base class.

Args:
  on_ui_exit: (`Callable`) the callback to be called when the UI exits.
  config: An instance of `cli_config.CLIConfig()` carrying user-facing
    configurations.


def create_token_type_ids_from_sequences(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

Create a mask from the two sequences passed to be used in a sequence-pair classification task. PLBart does not
make use of token type ids, therefore a list of zeros is returned.

Args:
    token_ids_0 (`List[int]`):
        List of IDs.
    token_ids_1 (`List[int]`, *optional*):
        Optional second list of IDs for sequence pairs.

Returns:
    `List[int]`: List of zeros.


def get_v2_optimizer(name, **kwargs):
    try:
        return _V2_OPTIMIZER_MAP[name](**kwargs)
    except KeyError:
        raise ValueError('Could not find requested v2 optimizer: {}\nValid choices: {}'.format(name, list(_V2_OPTIMIZER_MAP.keys())))

Get the v2 optimizer requested.

This is only necessary until v2 are the default, as we are testing in Eager,
and Eager + v1 optimizers fail tests. When we are in v2, the strings alone
should be sufficient, and this mapping can theoretically be removed.

Args:
  name: string name of Keras v2 optimizer.
  **kwargs: any kwargs to pass to the optimizer constructor.

Returns:
  Initialized Keras v2 optimizer.

Raises:
  ValueError: if an unknown name was passed.


class Concatenate(Merge):

    def __init__(self, axis=-1, **kwargs):
        super().__init__(**kwargs)
        self.axis = axis
        self.supports_masking = True
        self._reshape_required = False

    def build(self, input_shape):
        if len(input_shape) < 1 or not isinstance(input_shape[0], (tuple, list)):
            raise ValueError(f'A `Concatenate` layer should be called on a list of at least 1 input. Received: input_shape={input_shape}')
        if all((shape is None for shape in input_shape)):
            return
        reduced_inputs_shapes = [list(shape) for shape in input_shape]
        reduced_inputs_shapes_copy = copy.copy(reduced_inputs_shapes)
        shape_set = set()
        for i in range(len(reduced_inputs_shapes_copy)):
            concat_axis = self.axis % len(reduced_inputs_shapes_copy[i])
            for axis, axis_value in enumerate(reduced_inputs_shapes_copy, start=1):
                if axis != concat_axis and axis_value == 1:
                    del reduced_inputs_shapes[i][axis]
            if len(reduced_inputs_shapes[i]) > self.axis:
                del reduced_inputs_shapes[i][self.axis]
            shape_set.add(tuple(reduced_inputs_shapes[i]))
        if len(shape_set) != 1:
            err_msg = f'A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape={input_shape}'
            ranks = set((len(shape) for shape in shape_set))
            if len(ranks) != 1:
                raise ValueError(err_msg)
            rank, = ranks
            for axis in range(rank):
                unique_dims = set((shape[axis] for shape in shape_set if shape[axis] is not None))
                if len(unique_dims) > 1:
                    raise ValueError(err_msg)

    def _merge_function(self, inputs):
        return ops.concatenate(inputs, axis=self.axis)

    def compute_output_shape(self, input_shape):
        if not isinstance(input_shape, (tuple, list)) or not isinstance(input_shape[0], (tuple, list)):
            raise ValueError(f'A `Concatenate` layer should be called on a list of inputs. Received: input_shape={input_shape}')
        input_shapes = input_shape
        output_shape = list(input_shapes[0])
        for shape in input_shapes[1:]:
            if output_shape[self.axis] is None or shape[self.axis] is None:
                output_shape[self.axis] = None
                break
            output_shape[self.axis] += shape[self.axis]
        return tuple(output_shape)

    def compute_mask(self, inputs, mask=None):
        if mask is None:
            return None
        if not isinstance(mask, (tuple, list)):
            raise ValueError(f'`mask` should be a list. Received mask={mask}')
        if not isinstance(inputs, (tuple, list)):
            raise ValueError(f'`inputs` should be a list. Received: inputs={inputs}')
        if len(mask) != len(inputs):
            raise ValueError(f'The lists `inputs` and `mask` should have the same length. Received: inputs={inputs} of length {len(inputs)}, and mask={mask} of length {len(mask)}')
        if all((m is None for m in mask)):
            return None
        masks = []
        for input_i, mask_i in zip(inputs, mask):
            if mask_i is None:
                masks.append(ops.ones_like(input_i, dtype='bool'))
            elif mask_i.ndim < input_i.ndim:
                masks.append(ops.broadcast_to(ops.expand_dims(mask_i, axis=-1), ops.shape(input_i)))
            else:
                masks.append(mask_i)
        concatenated = ops.concatenate(masks, axis=self.axis)
        return ops.any(concatenated, axis=-1, keepdims=False)

    def get_config(self):
        config = {'axis': self.axis}
        base_config = super().get_config()
        return dict(list(base_config.items()) + list(config.items()))

Concatenates a list of inputs.

It takes as input a list of tensors, all of the same shape except
for the concatenation axis, and returns a single tensor that is the
concatenation of all inputs.

Examples:

>>> x = np.arange(20).reshape(2, 2, 5)
>>> y = np.arange(20, 30).reshape(2, 1, 5)
>>> keras.layers.Concatenate(axis=1)([x, y])

Usage in a Keras model:

>>> x1 = keras.layers.Dense(8)(np.arange(10).reshape(5, 2))
>>> x2 = keras.layers.Dense(8)(np.arange(10, 20).reshape(5, 2))
>>> y = keras.layers.Concatenate()([x1, x2])

Args:
    axis: Axis along which to concatenate.
    **kwargs: Standard layer keyword arguments.

Returns:
    A tensor, the concatenation of the inputs alongside axis `axis`.


def log_loss(labels, predictions, weights=1.0, epsilon=1e-07, scope=None, loss_collection=ops.GraphKeys.LOSSES, reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
    if labels is None:
        raise ValueError('Argument `labels` must not be None.')
    if predictions is None:
        raise ValueError('Argument `predictions` must not be None.')
    with ops.name_scope(scope, 'log_loss', (predictions, labels, weights)) as scope:
        predictions = math_ops.cast(predictions, dtype=dtypes.float32)
        labels = math_ops.cast(labels, dtype=dtypes.float32)
        predictions.get_shape().assert_is_compatible_with(labels.get_shape())
        losses = -math_ops.multiply(labels, math_ops.log(predictions + epsilon)) - math_ops.multiply(1 - labels, math_ops.log(1 - predictions + epsilon))
        return compute_weighted_loss(losses, weights, scope, loss_collection, reduction=reduction)

Adds a Log Loss term to the training procedure.

`weights` acts as a coefficient for the loss. If a scalar is provided, then
the loss is simply scaled by the given value. If `weights` is a tensor of size
`[batch_size]`, then the total loss for each sample of the batch is rescaled
by the corresponding element in the `weights` vector. If the shape of
`weights` matches the shape of `predictions`, then the loss of each
measurable element of `predictions` is scaled by the corresponding value of
`weights`.

Args:
  labels: The ground truth output tensor, same dimensions as 'predictions'.
  predictions: The predicted outputs.
  weights: Optional `Tensor` whose rank is either 0, or the same rank as
    `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
    be either `1`, or the same as the corresponding `losses` dimension).
  epsilon: A small increment to add to avoid taking a log of zero.
  scope: The scope for the operations performed in computing the loss.
  loss_collection: collection to which the loss will be added.
  reduction: Type of reduction to apply to loss.

Returns:
  Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
  shape as `labels`; otherwise, it is scalar.

Raises:
  ValueError: If the shape of `predictions` doesn't match that of `labels` or
    if the shape of `weights` is invalid.  Also if `labels` or `predictions`
    is None.

@compatibility(eager)
The `loss_collection` argument is ignored when executing eagerly. Consider
holding on to the return value or collecting losses via a `tf.keras.Model`.
@end_compatibility


def Get(self, request, global_params=None):
    config = self.GetMethodConfig('Get')
    return self._RunMethod(config, request, global_params=global_params)

Returns information about a `BuildTrigger`. This API is experimental.

Args:
  request: (CloudbuildProjectsTriggersGetRequest) input message
  global_params: (StandardQueryParameters, default: None) global arguments
Returns:
  (BuildTrigger) The response message.


class Blip2ImageTextMatchingModelOutput(ModelOutput):
    loss: Optional[torch.FloatTensor] = None
    logits_per_image: Optional[torch.FloatTensor] = None
    logits_per_text: Optional[torch.FloatTensor] = None
    text_embeds: Optional[torch.FloatTensor] = None
    image_embeds: Optional[torch.FloatTensor] = None
    text_model_output: BaseModelOutputWithPooling = None
    vision_model_output: BaseModelOutputWithPooling = None

    def to_tuple(self) -> Tuple[Any]:
        return tuple((self[k] if k not in ['text_model_output', 'vision_model_output'] else getattr(self, k).to_tuple() for k in self.keys()))

Args:
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output.
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The image embeddings obtained by applying the projection layer to the pooled output.
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`Blip2QFormerModel`].
    vision_model_output (`BaseModelOutputWithPooling`):
        The output of the [`Blip2VisionModel`].


def __init__(self, ctx):
    if ctx.options.use_fiddle_overlay:
        member_map = {'Config': overlay.add_name('Config', BuildableBuilder), 'Partial': overlay.add_name('Partial', BuildableBuilder)}
    else:
        member_map = {}
    ast = ctx.loader.import_name('fiddle')
    super().__init__(ctx, 'fiddle', member_map, ast)

Initializes the FiddleOverlay.

This function loads the AST for the fiddle module, which is used to
access type information for any members that are not explicitly provided by
the overlay. See get_attribute in attribute.py for how it's used.

Args:
  ctx: An instance of context.Context.


def set_current_position(self, position):
    raise NotImplementedError

Updates the last-consumed position to the given position.

A source may invoke this method for records that do not start at split
points. This may modify the internal state of the ``RangeTracker``. If the
record starts at a split point, method ``try_claim()`` **must** be invoked
instead of this method.

Args:
  position: starting position of a record being read by a source.


def index_to_string_table_from_tensor(vocabulary_list, default_value='UNK', name=None):
    if vocabulary_list is None:
        raise ValueError('`vocabulary_list` argument must be specified.')
    with ops.name_scope(name, 'index_to_string'):
        vocabulary_list = ops.convert_to_tensor(vocabulary_list, dtypes.string)
        num_elements = array_ops.size(vocabulary_list)
        keys = math_ops.cast(math_ops.range(num_elements), dtypes.int64)
        init = KeyValueTensorInitializer(keys, vocabulary_list, dtypes.int64, dtypes.string, name='table_init')
        return StaticHashTableV1(init, default_value)

Returns a lookup table that maps a `Tensor` of indices into strings.

This operation constructs a lookup table to map int64 indices into string
values. The mapping is initialized from a string `vocabulary_list` 1-D
`Tensor` where each element is a value and the corresponding index within the
tensor is the key.

Any input which does not have a corresponding index in 'vocabulary_list'
(an out-of-vocabulary entry) is assigned the `default_value`

The underlying table must be initialized by calling
`session.run(tf.compat.v1.tables_initializer())` or
`session.run(table.init())` once.

Elements in `vocabulary_list` cannot have duplicates, otherwise when executing
the table initializer op, it will throw a `FailedPreconditionError`.

Sample Usages:

```python
vocabulary_list = tf.constant(["emerson", "lake", "palmer"])
indices = tf.constant([1, 5], tf.int64)
table = tf.lookup.index_to_string_table_from_tensor(
    vocabulary_list, default_value="UNKNOWN")
values = table.lookup(indices)
...
tf.compat.v1.tables_initializer().run()

values.eval() ==> ["lake", "UNKNOWN"]
```

Args:
  vocabulary_list: A 1-D string `Tensor` that specifies the strings to map
    from indices.
  default_value: The value to use for out-of-vocabulary indices.
  name: A name for this op (optional).

Returns:
  The lookup table to map a string values associated to a given index `int64`
  `Tensors`.

Raises:
  ValueError: when `vocabulary_list` is not set.


def matmul(self, matmul_input: core.Tensor) -> Mapping[str, core.Tensor]:
    out = math_ops.matmul(matmul_input, self.matmul_filters)
    return {'output': out}

Performs a matrix multiplication.

Args:
  matmul_input: Input tensor to matmul with the filter.

Returns:
  A map of: output key -> output result.


def __init__(self, model_name: str, columns: list[str], title: Optional[str]=None, task_type: str=DEFAULT_TASK_TYPE, project: Optional[str]=None, location: Optional[str]=None, credentials: Optional[Credentials]=None, **kwargs):
    self.model_name = model_name
    self.project = project
    self.location = location
    self.credentials = credentials
    self.title = title
    self.task_type = task_type
    super().__init__(columns=columns, **kwargs)

Embedding Config for Vertex AI Text Embedding models following
https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings # pylint: disable=line-too-long
Text Embeddings are generated for a batch of text using the Vertex AI SDK.
Embeddings are returned in a list for each text in the batch. Look at
https://cloud.google.com/vertex-ai/docs/generative-ai/learn/model-versioning#stable-versions-available.md # pylint: disable=line-too-long
for more information on model versions and lifecycle.

Args:
  model_name: The name of the Vertex AI Text Embedding model.
  columns: The columns containing the text to be embedded.
  task_type: The downstream task for the embeddings. Valid values are
    RETRIEVAL_QUERY, RETRIEVAL_DOCUMENT, SEMANTIC_SIMILARITY,
    CLASSIFICATION, CLUSTERING. For more information on the task type,
    look at https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings # pylint: disable=line-too-long
  title: Identifier of the text content.
  project: The default GCP project for API calls.
  location: The default location for API calls.
  credentials: Custom credentials for API calls.
    Defaults to environment credentials.


def window_reverse(windows, window_size, height, width):
    num_channels = windows.shape[-1]
    windows = windows.view(-1, height // window_size, width // window_size, window_size, window_size, num_channels)
    windows = windows.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, height, width, num_channels)
    return windows

Merges windows to produce higher resolution features.
Args:
    windows (`torch.FloatTensor` of shape `(num_windows * batch_size, window_size, window_size, num_channels)`):
        Input windows
    window_size (`int`):
        Window size
    height (`int`):
        Height of the resized audio
    width (`int`):
        Width of the resized audio


def np_doc_only(np_fun_name, np_fun=None):
    np_fun_name, np_fun = _prepare_np_fun_name_and_fun(np_fun_name, np_fun)

    def decorator(f):
        f.__doc__ = _np_doc_helper(f, np_fun, np_fun_name=np_fun_name)
        return f
    return decorator

Attachs numpy docstring to a function.

This differs from np_doc in that it doesn't check for a match in signature.

Args:
  np_fun_name: name for the np_fun symbol. At least one of np_fun or
    np_fun_name shoud be set.
  np_fun: (optional) the numpy function whose docstring will be used.

Returns:
  A function decorator that attaches the docstring from `np_fun` to the
  decorated function.


def sparsemixer(scores, jitter_eps, training, top_k=2):
    if top_k != 2:
        raise ValueError('top_k must be equal to 2')
    with torch.no_grad():
        mask_logits_threshold, max_ind = scores.max(dim=-1, keepdim=True)
        factor = scores.abs().clamp(min=mask_logits_threshold)
        mask_logits_threshold = (mask_logits_threshold - scores) / factor > 2 * jitter_eps
    masked_gates = scores.masked_fill(mask_logits_threshold, float('-inf'))
    if training:
        selected_experts = (masked_gates - torch.empty_like(masked_gates, memory_format=torch.legacy_contiguous_format).exponential_().log()).max(dim=-1)[1].unsqueeze(-1)
    else:
        selected_experts = max_ind
    masked_gates = torch.softmax(masked_gates, dim=-1)
    multiplier_o = masked_gates.gather(dim=-1, index=selected_experts)
    if training:
        max_scores, max_ind = masked_gates.max(dim=-1, keepdim=True)
        mask_for_one = torch.logical_or(selected_experts == max_ind, torch.rand_like(max_scores) > 0.75)
        mask_for_one = torch.add(0.3333, mask_for_one, alpha=0.6667).type_as(masked_gates)
        multiplier = MultiplierProcessor.apply(scores, multiplier_o, selected_experts, masked_gates, mask_for_one)
    else:
        multiplier = multiplier_o
    masked_scores = torch.scatter(scores, -1, selected_experts, float('-inf'))
    with torch.no_grad():
        mask_logits_threshold, max_ind = masked_scores.max(dim=-1, keepdim=True)
        factor = scores.abs().clamp(min=mask_logits_threshold)
        mask_logits_threshold = (mask_logits_threshold - scores) / factor > 2 * jitter_eps
    masked_gates_top2 = masked_scores.masked_fill(mask_logits_threshold, float('-inf'))
    if training:
        selected_experts_top2 = (masked_gates_top2 - torch.empty_like(masked_gates_top2, memory_format=torch.legacy_contiguous_format).exponential_().log()).max(dim=-1)[1].unsqueeze(-1)
    else:
        selected_experts_top2 = max_ind
    masked_gates_top2 = torch.softmax(masked_gates_top2, dim=-1)
    multiplier_top2_o = masked_gates_top2.gather(dim=-1, index=selected_experts_top2)
    if training:
        max_scores, max_ind = masked_gates_top2.max(dim=-1, keepdim=True)
        mask_for_one_top2 = torch.logical_or(selected_experts_top2 == max_ind, torch.rand_like(max_scores).uniform_() > 0.75)
        mask_for_one_top2 = torch.add(0.3333, mask_for_one_top2, alpha=0.6667).type_as(masked_gates_top2)
        multiplier_top2 = MultiplierProcessor.apply(scores, multiplier_top2_o, selected_experts_top2, masked_gates_top2, mask_for_one_top2)
    else:
        multiplier_top2 = multiplier_top2_o
    multiplier = torch.concat((multiplier, multiplier_top2), dim=-1)
    selected_experts = torch.concat((selected_experts, selected_experts_top2), dim=-1)
    return (multiplier, selected_experts)

Sparse mixer function to select top-k experts and compute multipliers.
Based on the paper: https://huggingface.co/papers/2409.12136
We first replace the TopK(·) function as random sampling of discrete variables
in model training. Then, following Liu et al. (2023a) and Liu et al. (2023b), we apply Heun's
third order method to approximate the expert routing gradient and construct a modified
back-propagation to give a mathematically sound gradient estimation for expert routing.

Args:
    scores (torch.Tensor): Input scores tensor.
    jitter_eps (float): Jitter epsilon for numerical stability.
    training (bool): Flag indicating if the model is in training mode.
    top_k (int): Number of top experts to select.

Returns:
    Tuple[torch.Tensor, torch.Tensor]: Multiplier and selected experts tensors.


def list_adb_devices():
    out = adb.AdbProxy().devices()
    return parse_device_list(out, 'device')

List all android devices connected to the computer that are detected by
adb.

Returns:
  A list of android device serials. Empty if there's none.


def erosion2d(value, kernel, strides, rates, padding, name=None):
    with ops.name_scope(name, 'erosion2d', [value, kernel]) as name:
        return math_ops.negative(gen_nn_ops.dilation2d(input=math_ops.negative(value), filter=array_ops.reverse_v2(kernel, [0, 1]), strides=strides, rates=rates, padding=padding, name=name))

Computes the grayscale erosion of 4-D `value` and 3-D `kernel` tensors.

The `value` tensor has shape `[batch, in_height, in_width, depth]` and the
`kernel` tensor has shape `[kernel_height, kernel_width, depth]`, i.e.,
each input channel is processed independently of the others with its own
structuring function. The `output` tensor has shape
`[batch, out_height, out_width, depth]`. The spatial dimensions of the
output tensor depend on the `padding` algorithm. We currently only support the
default "NHWC" `data_format`.

In detail, the grayscale morphological 2-D erosion is given by:

    output[b, y, x, c] =
       min_{dy, dx} value[b,
                          strides[1] * y - rates[1] * dy,
                          strides[2] * x - rates[2] * dx,
                          c] -
                    kernel[dy, dx, c]

Duality: The erosion of `value` by the `kernel` is equal to the negation of
the dilation of `-value` by the reflected `kernel`.

Args:
  value: A `Tensor`. 4-D with shape `[batch, in_height, in_width, depth]`.
  kernel: A `Tensor`. Must have the same type as `value`.
    3-D with shape `[kernel_height, kernel_width, depth]`.
  strides: A list of `ints` that has length `>= 4`.
    1-D of length 4. The stride of the sliding window for each dimension of
    the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
  rates: A list of `ints` that has length `>= 4`.
    1-D of length 4. The input stride for atrous morphological dilation.
    Must be: `[1, rate_height, rate_width, 1]`.
  padding: A `string` from: `"SAME", "VALID"`.
    The type of padding algorithm to use.
  name: A name for the operation (optional). If not specified "erosion2d"
    is used.

Returns:
  A `Tensor`. Has the same type as `value`.
  4-D with shape `[batch, out_height, out_width, depth]`.
Raises:
  ValueError: If the `value` depth does not match `kernel`' shape, or if
    padding is other than `'VALID'` or `'SAME'`.


def opcode_to_name(model, op_code):
    op = model.operatorCodes[op_code]
    code = max(op.builtinCode, op.deprecatedBuiltinCode)
    for name, value in vars(schema_fb.BuiltinOperator).items():
        if value == code:
            return name
    return None

Converts a TFLite op_code to the human readable name.

Args:
  model: The input tflite model.
  op_code: The op_code to resolve to a readable name.

Returns:
  A string containing the human readable op name, or None if not resolvable.


def get_compile_flags():
    flags = []
    flags.append('-I%s' % get_include())
    flags.append('-D_GLIBCXX_USE_CXX11_ABI=%d' % _CXX11_ABI_FLAG)
    cxx_version_flag = None
    if _CXX_VERSION == 201103:
        cxx_version_flag = '--std=c++11'
    elif _CXX_VERSION == 201402:
        cxx_version_flag = '--std=c++14'
    elif _CXX_VERSION == 201703:
        cxx_version_flag = '--std=c++17'
    elif _CXX_VERSION == 202002:
        cxx_version_flag = '--std=c++20'
    if cxx_version_flag:
        flags.append(cxx_version_flag)
    flags.append('-DEIGEN_MAX_ALIGN_BYTES=%d' % pywrap_tf_session.get_eigen_max_align_bytes())
    return flags

Returns the compilation flags for compiling with TensorFlow.

The returned list of arguments can be passed to the compiler for compiling
against TensorFlow headers. The result is platform dependent.

For example, on a typical Linux system with Python 3.7 the following command
prints `['-I/usr/local/lib/python3.7/dist-packages/tensorflow/include',
'-D_GLIBCXX_USE_CXX11_ABI=1', '-DEIGEN_MAX_ALIGN_BYTES=64']`

>>> print(tf.sysconfig.get_compile_flags())

Returns:
  A list of strings for the compiler flags.


def has_atomic_move(path):
    try:
        return _pywrap_file_io.HasAtomicMove(compat.path_to_bytes(path))
    except errors.OpError:
        return True

Checks whether the file system supports atomic moves.

Returns whether or not the file system of the given path supports the atomic
move operation for a file or folder.  If atomic move is supported, it is
recommended to use a temp location for writing and then move to the final
location.

Args:
  path: string, path to a file

Returns:
  True, if the path is on a file system that supports atomic move
  False, if the file system does not support atomic move. In such cases
         we need to be careful about using moves. In some cases it is safer
         not to use temporary locations in this case.


def __init__(self, fn, buffer_size=_DEFAULT_BUFFER_SIZE):
    if not callable(fn):
        raise TypeError('Expected a callable object instead of: %r' % fn)
    super().__init__()
    self._fn = fn
    self._buffer_size = buffer_size

Initializes a CallableFn object wrapping a callable.

Args:
  fn: A callable object that reduces elements of an iterable to a single
    value (like the builtins sum and max). This callable must be capable of
    receiving the kind of values it generates as output in its input, and
    for best results, its operation must be commutative and associative.

Raises:
  TypeError: if fn parameter is not a callable type.


def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
    cos = cos.unsqueeze(unsqueeze_dim)
    sin = sin.unsqueeze(unsqueeze_dim)
    q_embed = q * cos + rotate_half(q) * sin
    k_embed = k * cos + rotate_half(k) * sin
    return (q_embed, k_embed)

Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.


def forward(self, inputs: torch.Tensor, loc: torch.Tensor, scale: torch.Tensor):
    mean = loc.transpose(-1, -2)
    mean = mean.unsqueeze(-2)
    mean = mean.repeat(1, 1, self.num_patches, 1)
    stdev = scale.transpose(-1, -2)
    stdev = stdev.unsqueeze(-2)
    stdev = stdev.repeat(1, 1, self.num_patches, 1)
    concat_stats = torch.cat([mean, stdev], dim=-1)
    concat_stats = self.map_scale_expansion(concat_stats)
    concat_stats = self.map_scale_compression(concat_stats)
    inputs = torch.cat([inputs, concat_stats], dim=-1)
    inputs = self.inverse_trans_expansion(inputs)
    inputs = self.inverse_trans_compression(inputs)
    return inputs

Args:
    inputs (`torch.Tensor` of shape `(batch_size, num_input_channels, num_patch, d_model)`)
    loc (`torch.Tensor` of shape `(batch_size, 1, num_input_channels)`)
    scale (`torch.Tensor` of shape `(batch_size, 1, num_input_channels)`)
Returns:
    `torch.Tensor` of shape `(batch_size, num_input_channels, num_patch, d_model)`


def _pad(self, encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], max_length: Optional[int]=None, padding_strategy: PaddingStrategy=PaddingStrategy.DO_NOT_PAD, pad_to_multiple_of: Optional[int]=None, padding_side: Optional[str]=None, return_attention_mask: Optional[bool]=None) -> dict:
    if return_attention_mask is None:
        return_attention_mask = 'attention_mask' in self.model_input_names
    required_input = encoded_inputs[self.model_input_names[0]]
    if padding_strategy == PaddingStrategy.LONGEST:
        max_length = len(required_input)
    if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
        max_length = (max_length // pad_to_multiple_of + 1) * pad_to_multiple_of
    needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
    if return_attention_mask and 'attention_mask' not in encoded_inputs:
        encoded_inputs['attention_mask'] = [1] * len(required_input)
    if needs_to_be_padded:
        difference = max_length - len(required_input)
        padding_side = padding_side if padding_side is not None else self.padding_side
        if padding_side == 'right':
            if return_attention_mask:
                encoded_inputs['attention_mask'] = encoded_inputs['attention_mask'] + [0] * difference
            if 'token_type_ids' in encoded_inputs:
                encoded_inputs['token_type_ids'] = encoded_inputs['token_type_ids'] + [self.pad_token_type_id] * difference
            if 'bbox' in encoded_inputs:
                encoded_inputs['bbox'] = encoded_inputs['bbox'] + [self.pad_token_box] * difference
            if 'labels' in encoded_inputs:
                encoded_inputs['labels'] = encoded_inputs['labels'] + [self.pad_token_label] * difference
            if 'special_tokens_mask' in encoded_inputs:
                encoded_inputs['special_tokens_mask'] = encoded_inputs['special_tokens_mask'] + [1] * difference
            encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
        elif padding_side == 'left':
            if return_attention_mask:
                encoded_inputs['attention_mask'] = [0] * difference + encoded_inputs['attention_mask']
            if 'token_type_ids' in encoded_inputs:
                encoded_inputs['token_type_ids'] = [self.pad_token_type_id] * difference + encoded_inputs['token_type_ids']
            if 'bbox' in encoded_inputs:
                encoded_inputs['bbox'] = [self.pad_token_box] * difference + encoded_inputs['bbox']
            if 'labels' in encoded_inputs:
                encoded_inputs['labels'] = [self.pad_token_label] * difference + encoded_inputs['labels']
            if 'special_tokens_mask' in encoded_inputs:
                encoded_inputs['special_tokens_mask'] = [1] * difference + encoded_inputs['special_tokens_mask']
            encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
        else:
            raise ValueError('Invalid padding strategy:' + str(padding_side))
    return encoded_inputs

Pad encoded inputs (on left/right and up to predefined length or max length in the batch)

Args:
    encoded_inputs:
        Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
    max_length: maximum length of the returned list and optionally padding length (see below).
        Will truncate by taking into account the special tokens.
    padding_strategy: PaddingStrategy to use for padding.

        - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
        - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
        - PaddingStrategy.DO_NOT_PAD: Do not pad
        The tokenizer padding sides are defined in self.padding_side:

            - 'left': pads on the left of the sequences
            - 'right': pads on the right of the sequences
    pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
        This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
        `>= 7.5` (Volta).
    padding_side:
        The side on which the model should have padding applied. Should be selected between ['right', 'left'].
        Default value is picked from the class attribute of the same name.
    return_attention_mask:
        (optional) Set to False to avoid returning attention mask (default: set to model specifics)


class CastFunction(StandardSqlExpression):
    expression: StandardSqlExpression
    cast_to: StandardSqlDataType
    _sql_alias: Optional[str] = None

    def __str__(self) -> str:
        return f'CAST(\n{self.expression} AS {self.cast_to.big_query_type_name})'

    @property
    def sql_alias(self) -> str:
        return self._sql_alias or 'cast_'

    @property
    def sql_data_type(self) -> StandardSqlDataType:
        return self.cast_to

Representation of a SQL cast.

Attributes:
  expression: The expression being cast.
  cast_to: The type the expression is being cast to.


def validate(self):
    errors = []
    for cls in self.OPTIONS:
        if 'validate' in cls.__dict__ and callable(cls.__dict__['validate']):
            errors.extend(self.options.view_as(cls).validate(self))
    return errors

Calls validate on subclassess and returns a list of errors.

validate will call validate method on subclasses, accumulate the returned
list of errors, and returns the aggregate list.

Returns:
  Aggregate list of errors after all calling all possible validate methods.


class ClvpTokenizer(PreTrainedTokenizer):
    vocab_files_names = VOCAB_FILES_NAMES
    model_input_names = ['input_ids', 'attention_mask']

    def __init__(self, vocab_file, merges_file, errors='replace', unk_token='[UNK]', bos_token='<|endoftext|>', eos_token='[STOP]', pad_token='[STOP]', add_prefix_space=False, add_bos_token=False, add_eos_token=False, **kwargs):
        bos_token = AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
        pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
        self.add_bos_token = add_bos_token
        self.add_eos_token = add_eos_token
        self._normalizer = None
        with open(vocab_file, encoding='utf-8') as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.errors = errors
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        with open(merges_file, encoding='utf-8') as merges_handle:
            bpe_merges = merges_handle.read().split('\n')[1:-1]
        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
        self.cache = {}
        self.add_prefix_space = add_prefix_space
        self.pat = re.compile("'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+")
        super().__init__(errors=errors, unk_token=unk_token, bos_token=bos_token, eos_token=eos_token, pad_token=pad_token, add_prefix_space=add_prefix_space, add_bos_token=add_bos_token, add_eos_token=add_eos_token, **kwargs)

    @property
    def vocab_size(self):
        return len(self.encoder)

    @property
    def normalizer(self):
        if self._normalizer is None:
            self._normalizer = EnglishNormalizer()
        return self._normalizer

    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        if token in self.cache:
            return self.cache[token]
        word = tuple(token)
        pairs = get_pairs(word)
        if not pairs:
            return token
        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j
                if word[i] == first and i < len(word) - 1 and (word[i + 1] == second):
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = ' '.join(word)
        self.cache[token] = word
        return word

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
        output = bos_token_id + token_ids_0 + eos_token_id
        if token_ids_1 is not None:
            output = output + bos_token_id + token_ids_1 + eos_token_id
        return output

    def get_special_tokens_mask(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None, already_has_special_tokens: bool=False) -> List[int]:
        """
        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True)
        if not self.add_bos_token:
            return super().get_special_tokens_mask(token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=False)
        if token_ids_1 is None:
            return [1] + [0] * len(token_ids_0)
        return [1] + [0] * len(token_ids_0) + [1] + [0] * len(token_ids_1)

    def _tokenize(self, text):
        """Tokenize a string."""
        bpe_tokens = []
        text = self.normalizer(text)
        for token in re.findall(self.pat, text):
            token = ''.join((self.byte_encoder[b] for b in token.encode('utf-8')))
            bpe_tokens.extend(('[SPACE]' if bpe_token == 'Ġ' and '[SPACE]' in self.encoder.keys() else bpe_token for bpe_token in self.bpe(token).split(' ')))
        return bpe_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.decoder.get(index)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        text = ''.join(tokens)
        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
        return text

    def clean_up_tokenization(self, text):
        text = ''.join(text)
        vocab_tokens = list(self.encoder.keys()) + list(self.added_tokens_encoder.keys())
        text = text.replace('[SPACE]', ' ') if '[SPACE]' in vocab_tokens else text
        text = text.replace('[STOP]', ' ') if '[STOP]' in vocab_tokens else text
        text = text.replace(self.unk_token, '').replace('   ', ' ').replace('  ', ' ')
        return text

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str]=None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
            return
        vocab_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
        merge_file = os.path.join(save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])
        with open(vocab_file, 'w', encoding='utf-8') as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + '\n')
        index = 0
        with open(merge_file, 'w', encoding='utf-8') as writer:
            writer.write('#version: 0.2\n')
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!')
                    index = token_index
                writer.write(' '.join(bpe_tokens) + '\n')
                index += 1
        return (vocab_file, merge_file)

Construct a CLVP tokenizer. Based on byte-level Byte-Pair-Encoding.

This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will
be encoded differently whether it is at the beginning of the sentence (without space) or not:

```python
>>> from transformers import ClvpTokenizer

>>> tokenizer = ClvpTokenizer.from_pretrained("susnato/clvp_dev")
>>> tokenizer("Hello world")["input_ids"]
[62, 84, 28, 2, 179, 79]

>>> tokenizer(" Hello world")["input_ids"]
[2, 62, 84, 28, 2, 179, 79]
```

You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you
call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.

<Tip>

When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).

</Tip>

This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.

Args:
    vocab_file (`str`):
        Path to the vocabulary file.
    merges_file (`str`):
        Path to the merges file.
    errors (`str`, *optional*, defaults to `"replace"`):
        Paradigm to follow when decoding bytes to UTF-8. See
        [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
    unk_token (`str`, *optional*, defaults to `"[UNK]"`):
        The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
        token instead.
    bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
        The beginning of sequence token.
    eos_token (`str`, *optional*, defaults to `"[STOP]"`):
        The end of sequence token.
    pad_token (`str`, *optional*, defaults to `"[STOP]"`):
        The pad token of the sequence.
    add_prefix_space (`bool`, *optional*, defaults to `False`):
        Whether or not to add an initial space to the input. This allows to treat the leading word just as any
        other word. (CLVP tokenizer detect beginning of words by the preceding space).
    add_bos_token (`bool`, *optional*, defaults to `False`):
        Whether to add `bos_token` in front of the sequence when add_special_tokens=True.
    add_eos_token (`bool`, *optional*, defaults to `False`):
        Whether to add `eos_token` in end of the sequence when add_special_tokens=True.


def _AddTokenOnCurrentLine(self, dry_run):
    current = self.next_token
    previous = current.previous_token
    spaces = current.spaces_required_before
    if isinstance(spaces, list):
        spaces = 0
    if not dry_run:
        current.AddWhitespacePrefix(newlines_before=0, spaces=spaces)
    if previous.OpensScope():
        if not current.is_comment:
            self.stack[-1].closing_scope_indent = self.column - 1
            if style.Get('ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'):
                self.stack[-1].closing_scope_indent += 1
            self.stack[-1].indent = self.column + spaces
        else:
            self.stack[-1].closing_scope_indent = self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH')
    self.column += spaces

Puts the token on the current line.

Appends the next token to the state and updates information necessary for
indentation.

Arguments:
  dry_run: (bool) Commit whitespace changes to the FormatToken if True.


def maybe_shuffle_batch_join(tensors_list, batch_size, capacity, min_after_dequeue, keep_input, seed=None, enqueue_many=False, shapes=None, allow_smaller_final_batch=False, shared_name=None, name=None):
    return _shuffle_batch_join(tensors_list, batch_size, capacity, min_after_dequeue, keep_input, seed=seed, enqueue_many=enqueue_many, shapes=shapes, allow_smaller_final_batch=allow_smaller_final_batch, shared_name=shared_name, name=name)

Create batches by randomly shuffling conditionally-enqueued tensors.

See docstring in `shuffle_batch_join` for more details.

Args:
  tensors_list: A list of tuples or dictionaries of tensors to enqueue.
  batch_size: An integer. The new batch size pulled from the queue.
  capacity: An integer. The maximum number of elements in the queue.
  min_after_dequeue: Minimum number elements in the queue after a
    dequeue, used to ensure a level of mixing of elements.
  keep_input: A `bool` Tensor.  This tensor controls whether the input is
    added to the queue or not.  If it is a scalar and evaluates `True`, then
    `tensors` are all added to the queue. If it is a vector and `enqueue_many`
    is `True`, then each example is added to the queue only if the
    corresponding value in `keep_input` is `True`. This tensor essentially
    acts as a filtering mechanism.
  seed: Seed for the random shuffling within the queue.
  enqueue_many: Whether each tensor in `tensor_list_list` is a single
    example.
  shapes: (Optional) The shapes for each example.  Defaults to the
    inferred shapes for `tensors_list[i]`.
  allow_smaller_final_batch: (Optional) Boolean. If `True`, allow the final
    batch to be smaller if there are insufficient items left in the queue.
  shared_name: (optional). If set, this queue will be shared under the given
    name across multiple sessions.
  name: (Optional) A name for the operations.

Returns:
  A list or dictionary of tensors with the same number and types as
  `tensors_list[i]`.

Raises:
  ValueError: If the `shapes` are not specified, and cannot be
    inferred from the elements of `tensors_list`.

@compatibility(eager)
Input pipelines based on Queues are not supported when eager execution is
enabled. Please use the `tf.data` API to ingest data under eager execution.
@end_compatibility


def __init__(self, msg, exception_details=None):
    message = '%s with exceptions %s' % (msg, exception_details)
    super().__init__(message)
    self.exception_details = exception_details

Class representing the errors thrown in the batch file operations.
Args:
  msg: Message string for the exception thrown
  exception_details: Optional map of individual input to exception for
    failed operations in batch. This parameter is optional so if specified
    the user can assume that the all errors in the filesystem operation
    have been reported. When the details are missing then the operation
    may have failed anywhere so the user should use match to determine
    the current state of the system.


def _FindStmtParent(node):
    if pytree_utils.NodeName(node) in _STATEMENT_NODES:
        return node
    else:
        return _FindStmtParent(node.parent)

Find the nearest parent of node that is a statement node.

Arguments:
  node: node to start from

Returns:
  Nearest parent (or node itself, if suitable).


def mean_pairwise_squared_error(labels, predictions, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES):
    if labels is None:
        raise ValueError('Argument `labels` must not be None.')
    if predictions is None:
        raise ValueError('Argument `predictions` must not be None.')
    with ops.name_scope(scope, 'mean_pairwise_squared_error', (predictions, labels, weights)) as scope:
        weights = math_ops.cast(weights, dtype=dtypes.float32)
        labels = math_ops.cast(labels, dtype=dtypes.float32)

        def compute_loss(labels, predictions, weights, loss_collection):
            predictions = math_ops.cast(predictions, dtype=dtypes.float32)
            predictions.get_shape().assert_is_compatible_with(labels.get_shape())
            diffs = math_ops.subtract(predictions, labels)
            axis = math_ops.range(1, array_ops.rank(diffs))
            sum_squares_diff_per_batch = math_ops.reduce_sum(math_ops.square(diffs), axis=axis, keepdims=True)
            num_present_per_batch = _num_present(diffs, weights, per_batch=True)
            term1 = 2.0 * math_ops.div_no_nan(sum_squares_diff_per_batch, math_ops.maximum(num_present_per_batch - 1, 0), name='value')
            sum_diff = math_ops.reduce_sum(diffs, axis=axis, keepdims=True)
            term2 = 2.0 * math_ops.div_no_nan(math_ops.square(sum_diff), math_ops.maximum(math_ops.multiply(num_present_per_batch, num_present_per_batch - 1), 0), name='value')
            weighted_losses = math_ops.multiply(term1 - term2, weights)
            loss = math_ops.reduce_sum(weighted_losses)
            mean_loss = array_ops.where(math_ops.reduce_sum(num_present_per_batch) > 0, loss, array_ops.zeros_like(loss), name='value')
            util.add_loss(mean_loss, loss_collection)
            return mean_loss
        if control_flow_ops.get_enclosing_xla_context() is not None:
            return compute_loss(labels, predictions, weights, loss_collection)
        else:
            with ops.control_dependencies((weights_broadcast_ops.assert_broadcastable(weights, labels),)):
                return compute_loss(labels, predictions, weights, loss_collection)

Adds a pairwise-errors-squared loss to the training procedure.

Unlike `mean_squared_error`, which is a measure of the differences between
corresponding elements of `predictions` and `labels`,
`mean_pairwise_squared_error` is a measure of the differences between pairs of
corresponding elements of `predictions` and `labels`.

For example, if `labels`=[a, b, c] and `predictions`=[x, y, z], there are
three pairs of differences are summed to compute the loss:
  loss = [ ((a-b) - (x-y)).^2 + ((a-c) - (x-z)).^2 + ((b-c) - (y-z)).^2 ] / 3

Note that since the inputs are of shape `[batch_size, d0, ... dN]`, the
corresponding pairs are computed within each batch sample but not across
samples within a batch. For example, if `predictions` represents a batch of
16 grayscale images of dimension [batch_size, 100, 200], then the set of pairs
is drawn from each image, but not across images.

`weights` acts as a coefficient for the loss. If a scalar is provided, then
the loss is simply scaled by the given value. If `weights` is a tensor of size
`[batch_size]`, then the total loss for each sample of the batch is rescaled
by the corresponding element in the `weights` vector.

Args:
  labels: The ground truth output tensor, whose shape must match the shape of
    `predictions`.
  predictions: The predicted outputs, a tensor of size
    `[batch_size, d0, .. dN]` where N+1 is the total number of dimensions in
    `predictions`.
  weights: Coefficients for the loss a scalar, a tensor of shape
    `[batch_size]` or a tensor whose shape matches `predictions`.
  scope: The scope for the operations performed in computing the loss.
  loss_collection: collection to which the loss will be added.

Returns:
  A scalar `Tensor` that returns the weighted loss.

Raises:
  ValueError: If the shape of `predictions` doesn't match that of `labels` or
    if the shape of `weights` is invalid.  Also if `labels` or `predictions`
    is None.

@compatibility(eager)
The `loss_collection` argument is ignored when executing eagerly. Consider
holding on to the return value or collecting losses via a `tf.keras.Model`.
@end_compatibility