diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..841f13be4c0d2f48f54eecc916acd826395449af --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_hiera import * + from .modeling_hiera import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/configuration_hiera.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/configuration_hiera.py new file mode 100644 index 0000000000000000000000000000000000000000..43ee69d4845e02b30a2f3289e7550a3870e38ba3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/configuration_hiera.py @@ -0,0 +1,122 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Hiera model configuration""" + +from huggingface_hub.dataclasses import strict + +from ...backbone_utils import BackboneConfigMixin +from ...configuration_utils import PreTrainedConfig +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="facebook/hiera-base-224") +@strict +class HieraConfig(BackboneConfigMixin, PreTrainedConfig): + r""" + patch_stride (`list(int)`, *optional*, defaults to `[4, 4]`): + The stride of the patch. + patch_padding (`list(int)`, *optional*, defaults to `[3, 3]`): + The padding of the patch. + num_heads (`list(int)`, *optional*, defaults to `[1, 2, 4, 8]`): + Number of attention heads in each layer of the Transformer encoder. + embed_dim_multiplier (`float`, *optional*, defaults to 2.0): + The multiplier to the dimensionality of patch embedding in each layer of the Transformer encoder. + num_query_pool (`int`, *optional*, defaults to 3): + The number of query pool stages. + query_stride (`list(int)`, *optional*, defaults to `[2, 2]`): + The stride of the query pool. + masked_unit_size (`list(int)`, *optional*, defaults to `[8, 8]`): + The size of the masked unit. + masked_unit_attention (`list(bool)`, *optional*, defaults to `[True, True, False, False]`): + Whether to use masked unit attention in each layer of the Transformer encoder. + layer_norm_init (`float`, *optional*, defaults to 1.0): + The initial weight value for layer normalization layers. + decoder_depth (`int`, *optional*): + Depth of the decoder for MAE pretraining. + normalize_pixel_loss (`bool`, *optional*, defaults to `True`): + Whether to normalize the pixel loss by the number of pixels. + mask_ratio (`float`, *optional*, defaults to 0.6): + The ratio of masked tokens in the input. + + Example: + + ```python + >>> from transformers import HieraConfig, HieraModel + + >>> # Initializing a Hiera hiera-base-patch16-224 style configuration + >>> configuration = HieraConfig() + + >>> # Initializing a model (with random weights) from the hiera-base-patch16-224 style configuration + >>> model = HieraModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "hiera" + + attribute_map = {"num_hidden_layers": "num_layers"} + + embed_dim: int = 96 + image_size: list[int] | tuple[int, ...] = (224, 224) + patch_size: list[int] | tuple[int, ...] = (7, 7) + patch_stride: list[int] | tuple[int, ...] = (4, 4) + patch_padding: list[int] | tuple[int, ...] = (3, 3) + mlp_ratio: float = 4.0 + depths: list[int] | tuple[int, ...] = (2, 3, 16, 3) + num_heads: list[int] | tuple[int, ...] = (1, 2, 4, 8) + embed_dim_multiplier: float | int = 2.0 + num_query_pool: int = 3 + query_stride: list[int] | tuple[int, ...] = (2, 2) + masked_unit_size: list[int] | tuple[int, ...] = (8, 8) + masked_unit_attention: list[bool] | tuple[bool, ...] = (True, True, False, False) + drop_path_rate: float | int = 0.0 + num_channels: int = 3 + hidden_act: str = "gelu" + initializer_range: float = 0.02 + layer_norm_init: float = 1.0 + layer_norm_eps: float = 1e-6 + decoder_hidden_size: int | None = None + decoder_depth: int | None = None + decoder_num_heads: int | None = None + normalize_pixel_loss: bool | None = True + mask_ratio: float = 0.6 + _out_features: list[str] | None = None + _out_indices: list[int] | None = None + + def __post_init__(self, **kwargs): + # we set the hidden_size attribute in order to make Hiera work with VisionEncoderDecoderModel + # this indicates the channel dimension after the last stage of the model + self.hidden_size = int(self.embed_dim * self.embed_dim_multiplier ** (len(self.depths) - 1)) + self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(self.depths) + 1)] + self.set_output_features_output_indices( + out_indices=kwargs.pop("out_indices", None), out_features=kwargs.pop("out_features", None) + ) + super().__post_init__(**kwargs) + + def validate_architecture(self): + """Part of `@strict`-powered validation. Validates the architecture of the config.""" + if self.masked_unit_size[0] % self.query_stride[0] ** (len(self.depths) - 1) != 0: + raise ValueError( + f"masked_unit_size[0] ({self.masked_unit_size[0]}) must be divisible by query_stride[0] ({self.query_stride[0]}) " + f"raised to the power of the number of layers ({len(self.depths) - 1})" + ) + + if self.num_query_pool >= len(self.depths): + raise ValueError( + f"num_query_pool ({self.num_query_pool}) must be less than the number of layers ({len(self.depths)})" + ) + + +__all__ = ["HieraConfig"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/modeling_hiera.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/modeling_hiera.py new file mode 100644 index 0000000000000000000000000000000000000000..ba638f240ac7b6ffb42a050dd474af08d34bc721 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hiera/modeling_hiera.py @@ -0,0 +1,1398 @@ +# Copyright 2024 Meta and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Hiera model.""" + +import math +from dataclasses import dataclass + +import torch +from torch import nn + +from ... import initialization as init +from ...activations import ACT2FN +from ...backbone_utils import BackboneMixin, filter_output_hidden_states +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import ( + BackboneOutput, + BaseModelOutput, + BaseModelOutputWithPooling, + ImageClassifierOutput, + ModelOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import auto_docstring, logging, torch_int +from ...utils.generic import can_return_tuple +from .configuration_hiera import HieraConfig + + +logger = logging.get_logger(__name__) + + +@auto_docstring( + custom_intro=""" + Hiera encoder's outputs, with potential hidden states and attentions. + """ +) +@dataclass +class HieraEncoderOutput(ModelOutput): + r""" + reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of + shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to + include the spatial dimensions. + """ + + last_hidden_state: torch.FloatTensor | None = None + hidden_states: tuple[torch.FloatTensor, ...] | None = None + attentions: tuple[torch.FloatTensor, ...] | None = None + reshaped_hidden_states: tuple[torch.FloatTensor, ...] | None = None + + +@auto_docstring( + custom_intro=""" + Hiera model's outputs that also contains a pooling of the last hidden states. + """ +) +@dataclass +class HieraModelOutput(ModelOutput): + r""" + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed): + Average pooling of the last layer hidden-state. + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Tensor indicating which patches are masked (0) and which are not (1). + ids_restore (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Tensor containing the original index of the (shuffled) masked patches. + reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of + shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to + include the spatial dimensions. + """ + + last_hidden_state: torch.FloatTensor | None = None + pooler_output: torch.FloatTensor | None = None + bool_masked_pos: torch.BoolTensor | None = None + ids_restore: torch.LongTensor | None = None + hidden_states: tuple[torch.FloatTensor, ...] | None = None + attentions: tuple[torch.FloatTensor, ...] | None = None + reshaped_hidden_states: tuple[torch.FloatTensor, ...] | None = None + + +@auto_docstring( + custom_intro=""" + Hiera image classification outputs. + """ +) +@dataclass +class HieraForImageClassificationOutput(ImageClassifierOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, `optional`): + Loss value for the training task. + logits (`torch.FloatTensor` of shape `(batch_size, num_labels)`): + Prediction scores of the classification head (logits of the output layer). + hidden_states (`tuple(torch.FloatTensor)`, `optional`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of + shape `(batch_size, sequence_length, hidden_size)`. These are the unrolled hidden states of the model. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, `optional`): + Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + reshaped_hidden_states (`tuple(torch.FloatTensor)`, `optional`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of + shape `(batch_size, height, width, hidden_size)`. These are the reshaped and re-rolled hidden states of the model. + + Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to + include the spatial dimensions. + """ + + loss: torch.FloatTensor | None = None + logits: torch.FloatTensor | None = None + hidden_states: tuple[torch.FloatTensor, ...] | None = None + attentions: tuple[torch.FloatTensor, ...] | None = None + reshaped_hidden_states: tuple[torch.FloatTensor, ...] | None = None + + +@auto_docstring( + custom_intro=""" + Class for HieraForPreTraining's outputs, with potential hidden states and attentions. + """ +) +@dataclass +class HieraForPreTrainingOutput(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`): + Pixel reconstruction loss. + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, patch_size ** 2 * num_channels)`): + Pixel reconstruction logits. + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Tensor indicating which patches are masked (0) and which are not (1). + ids_restore (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Tensor containing the original index of the (shuffled) masked patches. + reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of + shape `(batch_size, height, width, hidden_size)`. Hidden-states of the model at the output of each layer + plus the initial embedding outputs reshaped to include the spatial dimensions. + """ + + loss: torch.FloatTensor | None = None + logits: torch.FloatTensor | None = None + bool_masked_pos: torch.BoolTensor | None = None + ids_restore: torch.LongTensor | None = None + hidden_states: tuple[torch.FloatTensor] | None = None + attentions: tuple[torch.FloatTensor] | None = None + reshaped_hidden_states: tuple[torch.FloatTensor] | None = None + + +class HieraPatchEmbeddings(nn.Module): + """ + This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial + `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a + Transformer. + """ + + def __init__(self, config, is_mae: bool = False): + super().__init__() + + # Support any number of spatial dimensions + self.spatial_dims = len(config.patch_size) + if self.spatial_dims != 2: + raise ValueError(f"The number of dimensions of the input image should be 2, but got {self.spatial_dims}.") + self.num_channels = config.num_channels + self.image_size = config.image_size[-2:] + self.tokens_spatial_shape = [i // s for i, s in zip(config.image_size, config.patch_stride)] + self.mask_spatial_shape = [i // s for i, s in zip(self.tokens_spatial_shape, config.masked_unit_size)] + self.mask_ratio = config.mask_ratio + self.is_mae = is_mae + self.projection = nn.Conv2d( + self.num_channels, + config.embed_dim, + kernel_size=config.patch_size, + stride=config.patch_stride, + padding=config.patch_padding, + ) + + def masked_conv( + self, pixel_values: torch.FloatTensor, bool_masked_pos: torch.BoolTensor | None = None + ) -> torch.Tensor: + """Zero-out the masked regions of the input before conv. + Prevents leakage of masked regions when using overlapping kernels. + """ + if bool_masked_pos is None: + return self.projection(pixel_values) + + target_size = pixel_values.shape[2:] + # Reshape bool_masked_pos to (batch_size, 1, mask_unit_height, mask_unit_width) + bool_masked_pos = bool_masked_pos.view(pixel_values.shape[0], 1, *self.mask_spatial_shape) + + bool_masked_pos = nn.functional.interpolate(bool_masked_pos.float(), size=target_size) + + return self.projection(pixel_values * bool_masked_pos) + + def random_masking( + self, pixel_values: torch.FloatTensor, noise: torch.FloatTensor | None = None + ) -> tuple[torch.BoolTensor, torch.LongTensor]: + """ + Perform per-sample random masking by per-sample shuffling. Per-sample shuffling is done by argsort random + noise. + + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`) + noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*) which is + mainly used for testing purposes to control randomness and maintain the reproducibility + """ + batch_size = pixel_values.shape[0] + # Tokens selected for masking at mask unit level + num_windows = math.prod(self.mask_spatial_shape) + len_keep = int(num_windows * (1 - self.mask_ratio)) + + if noise is None: + noise = torch.rand(batch_size, num_windows, device=pixel_values.device) + + # Sort noise for each sample + ids_shuffle = torch.argsort(noise, dim=1) + # ascend: small is keep, large is remove + ids_restore = torch.argsort(ids_shuffle, dim=1).to(pixel_values.device) + + # Generate the binary bool_masked_pos: 1 is *keep*, 0 is *remove* + # Note this is opposite to original MAE + bool_masked_pos = torch.zeros([batch_size, num_windows], device=pixel_values.device) + bool_masked_pos[:, :len_keep] = 1 + # Unshuffle to get the binary bool_masked_pos + bool_masked_pos = torch.gather(bool_masked_pos, dim=1, index=ids_restore).bool() + + return bool_masked_pos, ids_restore + + def forward( + self, + pixel_values: torch.FloatTensor, + noise: torch.FloatTensor | None = None, + ) -> tuple[torch.Tensor, torch.BoolTensor | None, torch.LongTensor | None]: + (bool_masked_pos, ids_restore) = ( + self.random_masking(pixel_values, noise=noise) if self.is_mae else (None, None) + ) + + embeddings = self.masked_conv(pixel_values, bool_masked_pos) + embeddings = embeddings.flatten(2).transpose(2, 1) + + return embeddings, bool_masked_pos, ids_restore + + +class HieraEmbeddings(nn.Module): + """ + Construct position and patch embeddings. + """ + + def __init__(self, config: HieraConfig, is_mae: bool = False) -> None: + super().__init__() + self.patch_stride = config.patch_stride + tokens_spatial_shape = [i // s for i, s in zip(config.image_size, config.patch_stride)] + self.mask_spatial_shape = [i // s for i, s in zip(tokens_spatial_shape, config.masked_unit_size)] + self.num_tokens = math.prod(tokens_spatial_shape) + self.is_mae = is_mae + + self.patch_embeddings = HieraPatchEmbeddings(config, is_mae=is_mae) + + self.position_embeddings = nn.Parameter(torch.zeros(1, self.num_tokens, config.embed_dim)) + + def interpolate_pos_encoding( + self, embeddings: torch.Tensor, pos_embeds: torch.Tensor, height: int, width: int + ) -> torch.Tensor: + """ + This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution + images. This method is also adapted to support torch.jit tracing, no class embeddings, and different patch strides. + + Adapted from: + - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and + - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 + """ + + num_patches = embeddings.shape[1] + num_positions = pos_embeds.shape[1] + + # always interpolate when tracing to ensure the exported model works for dynamic input shapes + if not torch.jit.is_tracing() and num_patches == num_positions and height == width: + return pos_embeds + + dim = embeddings.shape[-1] + + new_height = height // self.patch_stride[0] + new_width = width // self.patch_stride[1] + + sqrt_num_positions = torch_int(num_positions**0.5) + pos_embeds = pos_embeds.reshape(1, sqrt_num_positions, sqrt_num_positions, dim) + pos_embeds = pos_embeds.permute(0, 3, 1, 2) + + pos_embeds = nn.functional.interpolate( + pos_embeds, + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + ) + + pos_embeds = pos_embeds.permute(0, 2, 3, 1).view(1, -1, dim) + return pos_embeds + + def get_position_embedding( + self, embeddings: torch.Tensor, height: int, width: int, interpolate_pos_encoding: bool + ) -> torch.FloatTensor: + return ( + self.interpolate_pos_encoding(embeddings, self.position_embeddings, height, width) + if interpolate_pos_encoding + else self.position_embeddings + ) + + def forward( + self, + pixel_values: torch.FloatTensor, + noise: torch.FloatTensor | None = None, + interpolate_pos_encoding: bool = False, + ) -> tuple[torch.Tensor, torch.BoolTensor | None, torch.LongTensor | None]: + height, width = pixel_values.shape[-2:] + embeddings, bool_masked_pos, ids_restore = self.patch_embeddings(pixel_values, noise=noise) + embeddings = embeddings + self.get_position_embedding(embeddings, height, width, interpolate_pos_encoding) + return embeddings, bool_masked_pos, ids_restore + + +class HieraMaskUnitAttention(nn.Module): + """ + Computes either Mask Unit or Global Attention. Also is able to perform query pooling. + + Note: this assumes the tokens have already been flattened and unrolled into mask units. + """ + + def __init__( + self, + hidden_size: int, + hidden_size_output: int, + num_heads: int, + query_stride: int = 1, + window_size: int = 0, + use_mask_unit_attn: bool = False, + ) -> None: + super().__init__() + self.num_heads = num_heads + self.query_stride = query_stride + self.hidden_size_output = hidden_size_output + + self.head_dim = hidden_size_output // num_heads + self.scale = (self.head_dim) ** -0.5 + + self.qkv = nn.Linear(hidden_size, 3 * hidden_size_output) + self.proj = nn.Linear(hidden_size_output, hidden_size_output) + + self.window_size = window_size + self.use_mask_unit_attn = use_mask_unit_attn + + def forward( + self, + hidden_states: torch.Tensor, + output_attentions: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Input should be of shape [batch, tokens, channels].""" + batch_size, seq_len, _ = hidden_states.shape + + num_windows = 1 + if self.use_mask_unit_attn: + num_windows = seq_len // (self.query_stride * self.window_size) + + qkv = self.qkv(hidden_states) + qkv = qkv.reshape(batch_size, -1, num_windows, 3, self.num_heads, self.head_dim) + qkv = qkv.permute(3, 0, 4, 2, 1, 5) + + query, key, value = qkv.unbind(0) + + if self.query_stride > 1: + # Refer to unroll to see how this performs a maxpool-Nd + query = query.view(batch_size, self.num_heads, num_windows, self.query_stride, -1, self.head_dim) + query = query.max(dim=3).values + + attn_weights = (query * self.scale) @ key.transpose(-1, -2) + attn_weights = attn_weights.softmax(dim=-1) + + attn_output = attn_weights @ value + attn_output = attn_output.transpose(1, 3).reshape(batch_size, -1, self.hidden_size_output) + attn_output = self.proj(attn_output) + + return (attn_output, attn_weights) if output_attentions else (attn_output, None) + + +class HieraMlp(nn.Module): + def __init__(self, config, dim: int) -> None: + super().__init__() + self.activation_fn = ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(dim, int(dim * config.mlp_ratio)) + self.fc2 = nn.Linear(int(dim * config.mlp_ratio), dim) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +# Copied from transformers.models.swin.modular_swin.SwinDropPath with SwinDropPath->HieraDropPath +class HieraDropPath(nn.Module): + """Stochastic depth (DropPath) per sample, for residual blocks. + + Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth + `_. + """ + + def __init__(self, drop_prob: float = 0.0) -> None: + super().__init__() + self.drop_prob = drop_prob + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.drop_prob == 0.0 or not self.training: + return hidden_states + keep_prob = 1 - self.drop_prob + shape = (hidden_states.shape[0],) + (1,) * (hidden_states.ndim - 1) + random_tensor = torch.rand(shape, dtype=hidden_states.dtype, device=hidden_states.device) + random_tensor = torch.floor(random_tensor + keep_prob) + return hidden_states.div(keep_prob) * random_tensor + + def extra_repr(self) -> str: + return f"p={self.drop_prob}" + + +class HieraLayer(nn.Module): + def __init__( + self, + config, + hidden_size: int, + hidden_size_output: int, + num_heads: int, + drop_path: float = 0.0, + query_stride: int = 1, + window_size: int = 0, + use_mask_unit_attn: bool = False, + ) -> None: + super().__init__() + + self.hidden_size = hidden_size + self.hidden_size_output = hidden_size_output + self.query_stride = query_stride + + self.layernorm_before = nn.LayerNorm(hidden_size, eps=config.layer_norm_eps) + self.attn = HieraMaskUnitAttention( + hidden_size=hidden_size, + hidden_size_output=hidden_size_output, + num_heads=num_heads, + query_stride=query_stride, + window_size=window_size, + use_mask_unit_attn=use_mask_unit_attn, + ) + + self.layernorm_after = nn.LayerNorm(hidden_size_output, eps=config.layer_norm_eps) + self.mlp = HieraMlp(config, hidden_size_output) + + self.drop_path = HieraDropPath(drop_path) if drop_path > 0 else nn.Identity() + if hidden_size != hidden_size_output: + self.proj = nn.Linear(hidden_size, hidden_size_output) + + def forward( + self, + hidden_states: torch.Tensor, + output_attentions: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + batch_size, seq_len, _ = hidden_states.shape + # Attention + Q Pooling + hidden_states_norm = self.layernorm_before(hidden_states) + if self.hidden_size != self.hidden_size_output: + hidden_states = self.proj(hidden_states_norm) + # Refer to unroll to see how this performs a maxpool-Nd + hidden_states = ( + hidden_states.view(batch_size, self.query_stride, -1, self.hidden_size_output).max(dim=1).values + ) + + (hidden_states_norm, attn_weights) = self.attn(hidden_states_norm, output_attentions=output_attentions) + hidden_states = hidden_states + self.drop_path(hidden_states_norm) + + residual = hidden_states + hidden_states = self.layernorm_after(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + self.drop_path(hidden_states) + + return (hidden_states, attn_weights) + + +class HieraStage(GradientCheckpointingLayer): + def __init__( + self, + config, + depth: int, + hidden_size: int, + hidden_size_output: int, + num_heads: int, + drop_path: list[float], + query_stride: list[int], + window_size: int, + use_mask_unit_attn: bool, + stage_num: int | None = None, + ) -> None: + super().__init__() + # we need to know if the previous stage used masked attention + # mask unit or global attention. + # lag by 1 layer, so that global attention, + # applied post pooling on lower resolution + previous_stage_used_masked_attention = False + if stage_num is not None: + previous_stage_used_masked_attention = config.masked_unit_attention[stage_num - 1 if stage_num > 0 else 0] + self.layers = nn.ModuleList( + [ + HieraLayer( + config=config, + hidden_size=hidden_size if i == 0 else hidden_size_output, + hidden_size_output=hidden_size_output, + num_heads=num_heads, + drop_path=drop_path[i], + query_stride=query_stride[i], + window_size=window_size, + use_mask_unit_attn=use_mask_unit_attn or (previous_stage_used_masked_attention and i == 0), + ) + for i in range(depth) + ] + ) + + def forward( + self, hidden_states: torch.Tensor, output_attentions: bool = False + ) -> tuple[torch.Tensor, torch.Tensor | None]: + for i, layer_module in enumerate(self.layers): + (hidden_states, attn_weights) = layer_module(hidden_states, output_attentions=output_attentions) + + return hidden_states, attn_weights + + +def undo_windowing(hidden_states: torch.Tensor, shape: list[int], mask_unit_shape: list[int]) -> torch.Tensor: + """ + Restore spatial organization by undoing windowed organization of mask units. + + Args: + hidden_states (`torch.Tensor`): The hidden states tensor of shape `[batch_size, num_mask_unit_height*num_mask_unit_width, hidden_size]`. + shape (`list[int]`): The original shape of the hidden states tensor before windowing. + mask_unit_shape (`list[int]`): The shape of the mask units used for windowing. + + Returns: + torch.Tensor: The restored hidden states tensor of shape [batch_size, num_mask_unit_height*mask_unit_height, num_mask_unit_width*mask_unit_width, hidden_size]. + """ + batch_size, hidden_size = hidden_states.shape[0], hidden_states.shape[-1] + # From: [batch_size, num_mask_unit_height*num_mask_unit_width, hidden_size] + # To: [batch_size, num_mask_unit_height, num_mask_unit_width, mask_unit_height, mask_unit_width, hidden_size] + num_mask_units = [s // mu for s, mu in zip(shape, mask_unit_shape)] + hidden_states = hidden_states.view(batch_size, *num_mask_units, *mask_unit_shape, hidden_size) + + # From: [batch_size, num_mask_unit_height, num_mask_unit_width, mask_unit_height, mask_unit_width, hidden_size] + # To: [batch_size, num_mask_unit_height*mask_unit_height, num_mask_unit_width*mask_unit_width, hidden_size] + hidden_states = hidden_states.permute(0, 1, 3, 2, 4, 5) + hidden_states = hidden_states.reshape(batch_size, *shape, hidden_size) + + return hidden_states + + +class HieraEncoder(nn.Module): + def __init__(self, config: HieraConfig) -> None: + super().__init__() + total_depth = sum(config.depths) + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, total_depth, device="cpu")] + # query strides rule + cumulative_depths = torch.tensor(config.depths, device="cpu").cumsum(0).tolist() + query_pool_layer = cumulative_depths[: config.num_query_pool] + query_strides = [math.prod(config.query_stride) if i in query_pool_layer else 1 for i in range(total_depth)] + + # Transformer blocks + self.stages = nn.ModuleList() + hidden_size = config.embed_dim + stage_ends = [0] + cumulative_depths + masked_unit_area = math.prod(config.masked_unit_size) + query_stride_area = math.prod(config.query_stride) + for idx_stage, depth in enumerate(config.depths): + hidden_size_output = int(config.embed_dim * config.embed_dim_multiplier**idx_stage) + + stage = HieraStage( + config=config, + depth=depth, + hidden_size=hidden_size, + hidden_size_output=hidden_size_output, + num_heads=config.num_heads[idx_stage], + drop_path=dpr[stage_ends[idx_stage] : stage_ends[idx_stage + 1]], + query_stride=query_strides[stage_ends[idx_stage] : stage_ends[idx_stage + 1]], + window_size=int(masked_unit_area * query_stride_area**-idx_stage), + use_mask_unit_attn=config.masked_unit_attention[idx_stage], + stage_num=idx_stage, + ) + + hidden_size = hidden_size_output + self.stages.append(stage) + + # Setting reroll schedule + # The first stage has to reverse everything + # The next stage has to reverse all but the first unroll, etc. + stage_size = [i // s for i, s in zip(config.image_size, config.patch_stride)] + unroll_schedule = [config.query_stride] * len(config.depths[:-1]) + + self.schedule = {} + for idx_stage in range(len(config.depths)): + self.schedule[idx_stage] = unroll_schedule, stage_size + if idx_stage < config.num_query_pool: + stage_size = [i // s for i, s in zip(stage_size, config.query_stride)] + unroll_schedule = unroll_schedule[1:] + + self.gradient_checkpointing = False + + def reroll( + self, hidden_states: torch.Tensor, stage_idx: int, bool_masked_pos: torch.BoolTensor | None = None + ) -> torch.Tensor: + """ + Roll the given tensor back up to spatial order assuming it's from the given block. + + If no bool_masked_pos is provided returns: + - [batch_size, height, width, hidden_size] + If a bool_masked_pos is provided returns: + - [batch_size, num_mask_units, mask_unit_height, mask_unit_width, hidden_size] + """ + schedule, size = self.schedule[stage_idx] + batch_size, seq_len, hidden_size = hidden_states.shape + + num_dim = len(size) + mask_unit_shape = [1] * num_dim + + for strides in schedule: + # Extract the current patch from seq_len + hidden_states = hidden_states.view( + batch_size, *strides, seq_len // math.prod(strides), *mask_unit_shape, hidden_size + ) + + # Move that patch into the current MU + # Input: [batch_size, stride, stride, seq_len//(stride*stride), mask_unit_height, mask_unit_width, hidden_size] + # Output: [batch_size, seq_len//(stride*stride), stride, mask_unit_height, stride, mask_unit_width, hidden_size] + hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5, 6) + + # Reshape to [batch_size, seq_len//(stride*stride), *mask_units, hidden_size] + for i in range(num_dim): + mask_unit_shape[i] *= strides[i] + hidden_states = hidden_states.reshape(batch_size, -1, *mask_unit_shape, hidden_size) + seq_len = hidden_states.shape[1] + + # Current shape (e.g., 2d: [batch_size, #num_mask_units_height*#num_mask_units_width, mask_unit_height, mask_unit_width, hidden_size]) + hidden_states = hidden_states.view(batch_size, seq_len, *mask_unit_shape, hidden_size) + + # If masked, return [batch_size, num_mask_units, mask_unit_height, mask_unit_width, hidden_size] + if bool_masked_pos is not None: + return hidden_states + + # If not masked, we can return [batch_size, height, width, hidden_size] + hidden_states = undo_windowing(hidden_states, size, mask_unit_shape) + + return hidden_states + + def forward( + self, + hidden_states: torch.Tensor, + bool_masked_pos: torch.BoolTensor | None = None, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + ) -> tuple | BaseModelOutput: + all_hidden_states = () if output_hidden_states else None + all_reshaped_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + reshaped_hidden_states = self.reroll(hidden_states, stage_idx=0, bool_masked_pos=bool_masked_pos) + all_reshaped_hidden_states = all_reshaped_hidden_states + (reshaped_hidden_states,) + + for i, stage_module in enumerate(self.stages): + layer_outputs = stage_module(hidden_states, output_attentions) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + reshaped_hidden_states = self.reroll(hidden_states, stage_idx=i, bool_masked_pos=bool_masked_pos) + all_reshaped_hidden_states = all_reshaped_hidden_states + (reshaped_hidden_states,) + + if not return_dict: + return tuple( + v + for v in [hidden_states, all_hidden_states, all_self_attentions, all_reshaped_hidden_states] + if v is not None + ) + return HieraEncoderOutput( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + reshaped_hidden_states=all_reshaped_hidden_states, + ) + + +def unroll( + hidden_states: torch.Tensor, image_shape: tuple[int, int], patch_stride: tuple[int, int], schedule: list[list[int]] +) -> torch.Tensor: + """ + Reorders the tokens such that patches are contiguous in memory. + E.g., given [batch_size, (height, width), hidden_size] and stride of (stride, stride), this will re-order the tokens as + [batch_size, (stride, stride, height // stride, width // stride), hidden_size] + + This allows operations like Max2d to be computed as x.view(batch_size, stride*stride, -1, hidden_size).max(dim=1). + Not only is this faster, but it also makes it easy to support inputs of arbitrary + dimensions in addition to patch-wise sparsity. + + Performing this operation multiple times in sequence puts entire windows as contiguous + in memory. For instance, if you applied the stride (2, 2) 3 times, entire windows of + size 8x8 would be contiguous in memory, allowing operations like mask unit attention + computed easily and efficiently, while also allowing max to be applied sequentially. + + Note: This means that intermediate values of the model are not in height x width order, so they + need to be re-rolled if you want to use the intermediate values as a height x width feature map. + The last block of the network is fine though, since by then the strides are all consumed. + """ + batch_size, _, hidden_size = hidden_states.shape + + size = [i // s for i, s in zip(image_shape, patch_stride)] + + current_size = size + hidden_states = hidden_states.view(*([batch_size] + current_size + [hidden_size])) + + for strides in schedule: + # Move patches with the given strides to the batch dimension + + # Create a view of the tensor with the patch stride as separate dims + # For example in 2d: [batch_size, height // stride, stride, width // stride, stride, C] + current_size = [i // s for i, s in zip(current_size, strides)] + # initialize new_shape with [height // stride, stride, width // stride, stride] + new_shape = [item for pair in zip(current_size, strides) for item in pair] + # add batch_size and hidden_size to new_shape + new_shape = [batch_size] + new_shape + [hidden_size] + hidden_states = hidden_states.view(new_shape) + + # Move the patch stride into the batch dimension + # For example in 2d: [batch_size, stride, stride, height // stride, width // stride, hidden_size] + num_dims = len(new_shape) + permute = [0] + list(range(2, num_dims - 1, 2)) + list(range(1, num_dims - 1, 2)) + [num_dims - 1] + hidden_states = hidden_states.permute(permute) + + # Now finally flatten the relevant dims into the batch dimension + hidden_states = hidden_states.flatten(0, len(strides)) + batch_size *= math.prod(strides) + + hidden_states = hidden_states.reshape(-1, math.prod(size), hidden_size) + return hidden_states + + +@auto_docstring +class HieraPreTrainedModel(PreTrainedModel): + config: HieraConfig + base_model_prefix = "hiera" + main_input_name = "pixel_values" + input_modalities = ("image",) + supports_gradient_checkpointing = True + + @torch.no_grad() + def _init_weights(self, module) -> None: + """Initialize the weights""" + std = self.config.initializer_range + + if isinstance(module, HieraEmbeddings): + init.trunc_normal_(module.position_embeddings, std=std) + + elif isinstance(module, HieraDecoder): + init.trunc_normal_(module.mask_token, std=std) + init.trunc_normal_(module.decoder_position_embeddings, std=std) + + elif isinstance(module, (nn.Linear, nn.Conv1d, nn.Conv2d)): + init.trunc_normal_(module.weight, std=std) + if module.bias is not None: + init.constant_(module.bias, std) + + elif isinstance(module, nn.LayerNorm): + init.constant_(module.bias, std) + init.constant_(module.weight, self.config.layer_norm_init) + + +class HieraPooler(nn.Module): + def __init__(self, config: HieraConfig): + super().__init__() + num_features = int(config.embed_dim * config.embed_dim_multiplier ** (len(config.depths) - 1)) + self.layernorm = nn.LayerNorm(num_features, eps=config.layer_norm_eps) + self.pooler = nn.AdaptiveAvgPool1d(1) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states.transpose(1, 2) + pooled_output = self.pooler(hidden_states) + pooled_output = torch.flatten(pooled_output, 1) + pooled_output = self.layernorm(pooled_output) + return pooled_output + + +@auto_docstring +class HieraModel(HieraPreTrainedModel): + def __init__(self, config: HieraConfig, add_pooling_layer: bool = True, is_mae: bool = False): + r""" + add_pooling_layer (`bool`, *optional*, defaults to `True`): + Whether or not to apply pooling layer. + is_mae (`bool`, *optional*, defaults to `False`): + Whether or not to run the model on MAE mode. + """ + super().__init__(config) + self.num_features = int(config.embed_dim * config.embed_dim_multiplier ** (len(config.depths) - 1)) + + self.embeddings = HieraEmbeddings(config, is_mae=is_mae) + self.encoder = HieraEncoder(config) + + self.unroll_schedule = [config.query_stride] * len(config.depths[:-1]) + + self.pooler = HieraPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> HieraPatchEmbeddings: + return self.embeddings.patch_embeddings + + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor | None = None, + noise: torch.FloatTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + interpolate_pos_encoding: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | BaseModelOutputWithPooling: + r""" + noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + embedding_output, bool_masked_pos, ids_restore = self.embeddings( + pixel_values, interpolate_pos_encoding=interpolate_pos_encoding, noise=noise + ) + + image_shape = (pixel_values.shape[-2], pixel_values.shape[-1]) + hidden_states = unroll( + embedding_output, + image_shape=image_shape, + patch_stride=self.config.patch_stride, + schedule=self.unroll_schedule, + ) + + # Discard masked tokens if bool_masked_pos is provided + if bool_masked_pos is not None: + mask_unit_area = math.prod(self.config.masked_unit_size) + batch_size, _, hidden_size = hidden_states.shape + positions = bool_masked_pos.unsqueeze(-1).tile(1, mask_unit_area, hidden_size) + hidden_states = hidden_states[positions] + hidden_states = hidden_states.view(batch_size, -1, hidden_size) + + encoder_outputs = self.encoder( + hidden_states, + bool_masked_pos=bool_masked_pos, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = None + if self.pooler is not None: + pooled_output = self.pooler(sequence_output) + + if not return_dict: + head_outputs = (sequence_output, pooled_output) if pooled_output is not None else (sequence_output,) + head_outputs = ( + head_outputs + (bool_masked_pos, ids_restore) if bool_masked_pos is not None else head_outputs + ) + return head_outputs + encoder_outputs[1:] + + return HieraModelOutput( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + bool_masked_pos=bool_masked_pos, + ids_restore=ids_restore, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + reshaped_hidden_states=encoder_outputs.reshaped_hidden_states, + ) + + +class HieraDecoder(nn.Module): + def __init__(self, config: HieraConfig): + super().__init__() + num_features = int(config.embed_dim * config.embed_dim_multiplier ** (len(config.depths) - 1)) + tokens_spatial_shape = [i // s for i, s in zip(config.image_size, config.patch_stride)] + self.tokens_spatial_shape_final = [ + i // s ** (config.num_query_pool) for i, s in zip(tokens_spatial_shape, config.query_stride) + ] + self.mask_unit_spatial_shape_final = [ + i // s ** (config.num_query_pool) for i, s in zip(config.masked_unit_size, config.query_stride) + ] + + self.decoder_embeddings = nn.Linear(num_features, config.decoder_hidden_size) + + self.mask_token = nn.Parameter(torch.zeros(1, 1, config.decoder_hidden_size)) + + self.decoder_position_embeddings = nn.Parameter( + torch.zeros(1, math.prod(self.tokens_spatial_shape_final), config.decoder_hidden_size) + ) + + self.decoder_block = HieraStage( + config=config, + hidden_size=config.decoder_hidden_size, + hidden_size_output=config.decoder_hidden_size, + num_heads=config.decoder_num_heads, + depth=config.decoder_depth, + use_mask_unit_attn=False, + drop_path=[0.0] * config.decoder_depth, + query_stride=[1] * config.decoder_depth, + window_size=0, + ) + + self.decoder_norm = nn.LayerNorm(config.decoder_hidden_size, eps=config.layer_norm_eps) + + # patch stride of prediction + self.pred_stride = config.patch_stride[-1] * (config.query_stride[-1] ** config.num_query_pool) + pred_dim = (self.pred_stride ** len(config.query_stride)) * config.num_channels + + self.decoder_pred = nn.Linear(config.decoder_hidden_size, pred_dim) + + def forward( + self, + encoder_hidden_states: torch.Tensor, + bool_masked_pos: torch.BoolTensor, + output_attentions: bool = False, + ) -> tuple[torch.Tensor, torch.BoolTensor]: + # Embed tokens + hidden_states = self.decoder_embeddings(encoder_hidden_states) + + # Combine visible and bool_masked_pos tokens + + # hidden_states : [batch_size, num_mask_units_visible, *mask_unit_spatial_shape_final, decoder_hidden_size] + # bool_masked_pos: [batch_size, num_mask_units] + mask_unit_height, mask_unit_width, decoder_hidden_size = hidden_states.shape[2:] + batch_size, num_mask_units = bool_masked_pos.shape + + decoder_hidden_states = torch.zeros( + batch_size, + num_mask_units, + mask_unit_height, + mask_unit_width, + decoder_hidden_size, + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + mask_tokens = self.mask_token.view(1, 1, 1, 1, -1) + bool_masked_pos = bool_masked_pos.reshape(batch_size, num_mask_units, 1, 1, 1) + bool_masked_pos = bool_masked_pos.expand(-1, -1, mask_unit_height, mask_unit_width, decoder_hidden_size) + decoder_hidden_states[bool_masked_pos] = hidden_states.flatten() + decoder_hidden_states = ( + 1 - bool_masked_pos.float() + ) * mask_tokens + bool_masked_pos.float() * decoder_hidden_states + + # Get back spatial order + hidden_states = undo_windowing( + decoder_hidden_states, + self.tokens_spatial_shape_final, + self.mask_unit_spatial_shape_final, + ) + bool_masked_pos = undo_windowing( + bool_masked_pos[..., 0:1], + self.tokens_spatial_shape_final, + self.mask_unit_spatial_shape_final, + ) + + # Flatten + hidden_states = hidden_states.reshape(hidden_states.shape[0], -1, hidden_states.shape[-1]) + bool_masked_pos = bool_masked_pos.view(hidden_states.shape[0], -1) + + # Add pos embed + hidden_states = hidden_states + self.decoder_position_embeddings + + # Apply decoder blocks + hidden_states, attn_weights = self.decoder_block(hidden_states, output_attentions=output_attentions) + hidden_states = self.decoder_norm(hidden_states) + + # Predictor projection + hidden_states = self.decoder_pred(hidden_states) + + return hidden_states, bool_masked_pos + + +class HieraMultiScaleHead(nn.Module): + def __init__(self, config: HieraConfig): + super().__init__() + self.mask_unit_spatial_shape_final = [ + i // s ** (config.num_query_pool) for i, s in zip(config.masked_unit_size, config.query_stride) + ] + self.stage_dimensions = [ + int(config.embed_dim * config.embed_dim_multiplier**i) for i in range(len(config.depths)) + ] + current_masked_unit_size = config.masked_unit_size + self.multi_scale_fusion_heads = nn.ModuleList() + + for idx in range(config.num_query_pool): + kernel = [i // s for i, s in zip(current_masked_unit_size, self.mask_unit_spatial_shape_final)] + current_masked_unit_size = [i // s for i, s in zip(current_masked_unit_size, config.query_stride)] + self.multi_scale_fusion_heads.append( + nn.Conv2d( + self.stage_dimensions[idx], + self.stage_dimensions[-1], + kernel_size=kernel, + stride=kernel, + ) + ) + self.multi_scale_fusion_heads.append(nn.Identity()) + + def apply_fusion_head(self, head: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: + if isinstance(head, nn.Identity): + return hidden_states + + batch_size, num_mask_units, mask_unit_height, mask_unit_width, hidden_size = hidden_states.shape + # From: [batch_size, num_mask_units, mask_unit_height, mask_unit_width, hidden_size] + # To: head([batch_size * num_mask_units, hidden_size, mask_unit_height, mask_unit_width]) + hidden_states = hidden_states.reshape( + batch_size * num_mask_units, mask_unit_height, mask_unit_width, hidden_size + ) + hidden_states = hidden_states.permute(0, 3, 1, 2) + hidden_states = head(hidden_states) + + # Restore original layout + hidden_states = hidden_states.permute(0, 2, 3, 1) + mask_unit_height_final, mask_unit_width_final, hidden_size = hidden_states.shape[1:] + hidden_states = hidden_states.reshape( + batch_size, num_mask_units, mask_unit_height_final, mask_unit_width_final, hidden_size + ) + + return hidden_states + + def forward(self, feature_maps: list[torch.Tensor]) -> torch.Tensor: + # Multi-scale fusion + hidden_states = 0.0 + for head, feature_map in zip(self.multi_scale_fusion_heads, feature_maps): + hidden_states = hidden_states + self.apply_fusion_head(head, feature_map) + + return hidden_states + + +@auto_docstring( + custom_intro=""" + The Hiera Model transformer with the decoder on top for self-supervised pre-training. + + + + Note that we provide a script to pre-train this model on custom data in our [examples + directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining). + + + """ +) +class HieraForPreTraining(HieraPreTrainedModel): + def __init__(self, config: HieraConfig) -> None: + super().__init__(config) + # Encoder + self.hiera = HieraModel(config, add_pooling_layer=False, is_mae=True) + self.encoder_norm = nn.LayerNorm(self.hiera.num_features, eps=config.layer_norm_eps) + # Multi-scale fusion heads + self.multiscale_fusion = HieraMultiScaleHead(config) + # Decoder + self.decoder = HieraDecoder(config) + self.pred_stride = self.decoder.pred_stride + + # Initialize weights and apply final processing + self.post_init() + + def get_pixel_label_2d(self, pixel_values: torch.Tensor, bool_masked_pos: torch.BoolTensor) -> torch.Tensor: + # bool_masked_pos (boolean tensor): True means *masked* + pixel_values = pixel_values.permute(0, 2, 3, 1) + + size = self.pred_stride + label = pixel_values.unfold(1, size, size).unfold(2, size, size) + label = label.flatten(1, 2).flatten(2) + label = label[bool_masked_pos] + if self.config.normalize_pixel_loss: + mean = label.mean(dim=-1, keepdim=True) + var = label.var(dim=-1, keepdim=True) + label = (label - mean) / (var + 1.0e-6) ** 0.5 + + return label + + def forward_loss(self, pixel_values: torch.Tensor, logits: torch.Tensor, bool_masked_pos: torch.BoolTensor): + # We invert the bool_masked_pos such that 1.0 is *masked* + bool_masked_pos = ~bool_masked_pos + label = self.get_pixel_label_2d(pixel_values, bool_masked_pos) + + logits = logits[bool_masked_pos] + loss = (logits - label) ** 2 + loss = loss.mean() + + return loss + + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor | None = None, + noise: torch.FloatTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + interpolate_pos_encoding: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | HieraForPreTrainingOutput: + r""" + noise (`torch.FloatTensor` of shape `(batch_size, num_mask_units)`, *optional*): + Mainly used for testing purposes to control randomness and maintain the reproducibility + + Examples: + ```python + >>> from transformers import AutoImageProcessor, HieraForPreTraining + >>> import torch + >>> from PIL import Image + >>> import httpx + >>> from io import BytesIO + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + + >>> image_processor = AutoImageProcessor.from_pretrained("facebook/hiera-tiny-224-mae-hf") + >>> model = HieraForPreTraining.from_pretrained("facebook/hiera-tiny-224-mae-hf") + + >>> inputs = image_processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> logits = outputs.logits + >>> loss = outputs.loss + >>> print(list(logits.shape)) + [1, 196, 768] + ```""" + return_dict = return_dict if return_dict is not None else self.config.return_dict + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + outputs = self.hiera( + pixel_values, + noise=noise, + output_attentions=output_attentions, + output_hidden_states=True, + interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=return_dict, + ) + + feature_maps = outputs[-1] + bool_masked_pos = outputs[1] + ids_to_restore = outputs[2] + # Take only the query pooled and last hidden states + feature_maps = feature_maps[1 : self.hiera.config.num_query_pool + 1] + (feature_maps[-1],) + fused_hidden_states = self.multiscale_fusion(feature_maps) + fused_hidden_states = self.encoder_norm(fused_hidden_states) + + # Reconstruct pixel values + logits, bool_masked_pos = self.decoder( + fused_hidden_states, + bool_masked_pos=bool_masked_pos, + output_attentions=output_attentions, + ) + + loss = self.forward_loss(pixel_values, logits, bool_masked_pos) + + if not return_dict: + output = (logits, bool_masked_pos, ids_to_restore) + if output_hidden_states: + output = output + (outputs[3],) + if output_attentions: + output = output + (outputs[4],) + if output_hidden_states: + output = output + (outputs[-1],) + return ((loss,) + output) if loss is not None else output + + return HieraForPreTrainingOutput( + loss=loss, + logits=logits, + bool_masked_pos=bool_masked_pos, + ids_restore=ids_to_restore, + hidden_states=outputs.hidden_states if output_hidden_states else None, + attentions=outputs.attentions, + reshaped_hidden_states=outputs.reshaped_hidden_states if output_hidden_states else None, + ) + + +@auto_docstring( + custom_intro=""" + Hiera Model transformer with an image classification head on top (a linear layer on top of the final hidden state with + average pooling) e.g. for ImageNet. + + + + Note that it's possible to fine-tune Hiera on higher resolution images than the ones it has been trained on, by + setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained + position embeddings to the higher resolution. + + + """ +) +class HieraForImageClassification(HieraPreTrainedModel): + def __init__(self, config: HieraConfig) -> None: + super().__init__(config) + + self.num_labels = config.num_labels + self.hiera = HieraModel(config, add_pooling_layer=True, is_mae=False) + + # Classifier head + self.classifier = ( + nn.Linear(self.hiera.num_features, config.num_labels) if config.num_labels > 0 else nn.Identity() + ) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values, + labels: torch.Tensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + interpolate_pos_encoding: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | HieraForImageClassificationOutput: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the image classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + outputs = self.hiera( + pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + interpolate_pos_encoding=interpolate_pos_encoding, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + loss = self.loss_function(labels, logits, self.config) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return HieraForImageClassificationOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + reshaped_hidden_states=outputs.reshaped_hidden_states, + ) + + +@auto_docstring( + custom_intro=""" + Hiera backbone, to be used with frameworks like DETR and MaskFormer. + """ +) +class HieraBackbone(BackboneMixin, HieraPreTrainedModel): + def __init__(self, config: HieraConfig): + super().__init__(config) + + self.num_features = [config.embed_dim] + [ + int(config.embed_dim * config.embed_dim_multiplier**i) for i in range(len(config.depths)) + ] + self.embeddings = HieraEmbeddings(config, is_mae=False) + self.encoder = HieraEncoder(config) + + # Add layer norms to hidden states of out_features + hidden_states_norms = {} + for stage, num_channels in zip(self.out_features, self.channels): + hidden_states_norms[stage] = nn.LayerNorm(num_channels) + self.hidden_states_norms = nn.ModuleDict(hidden_states_norms) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.patch_embeddings + + @can_return_tuple + @filter_output_hidden_states + def forward( + self, + pixel_values: torch.Tensor, + output_hidden_states: bool | None = None, + output_attentions: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> BackboneOutput: + """ + Returns: + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, AutoBackbone + >>> import torch + >>> from PIL import Image + >>> import httpx + >>> from io import BytesIO + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + + >>> processor = AutoImageProcessor.from_pretrained("facebook/hiera-tiny-224-hf") + >>> model = AutoBackbone.from_pretrained( + ... "facebook/hiera-tiny-224-hf", out_features=["stage1", "stage2", "stage3", "stage4"] + ... ) + + >>> inputs = processor(image, return_tensors="pt") + >>> outputs = model(**inputs) + >>> feature_maps = outputs.feature_maps + >>> list(feature_maps[-1].shape) + [1, 768, 7, 7] + ```""" + return_dict = return_dict if return_dict is not None else self.config.return_dict + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + + embedding_output, _, _ = self.embeddings(pixel_values) + + outputs = self.encoder( + embedding_output, + output_attentions=output_attentions, + output_hidden_states=True, + return_dict=return_dict, + ) + + hidden_states = outputs[-1] + + feature_maps = () + for stage, hidden_state in zip(self.stage_names, hidden_states): + if stage in self.out_features: + batch_size, height, width, num_channels = hidden_state.shape + hidden_state = hidden_state.view(batch_size, height * width, num_channels) + hidden_state = self.hidden_states_norms[stage](hidden_state) + hidden_state = hidden_state.view(batch_size, height, width, num_channels) + hidden_state = hidden_state.permute(0, 3, 1, 2).contiguous() + feature_maps += (hidden_state,) + + if not return_dict: + output = (feature_maps,) + if output_hidden_states: + output += (outputs[1],) + if output_attentions: + output += (outputs[2],) + return output + + return BackboneOutput( + feature_maps=feature_maps, + hidden_states=outputs[1] if output_hidden_states else None, + attentions=outputs[2] if output_attentions else None, + ) + + +__all__ = ["HieraForImageClassification", "HieraForPreTraining", "HieraBackbone", "HieraModel", "HieraPreTrainedModel"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6886241652534c506bffad22dda68e0c724b457d --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2026 NAVER CLOUD Corp. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_hyperclovax import * + from .modeling_hyperclovax import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/configuration_hyperclovax.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/configuration_hyperclovax.py new file mode 100644 index 0000000000000000000000000000000000000000..430a56bf024959c87a45df5bf75050a04bc1cb9f --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/configuration_hyperclovax.py @@ -0,0 +1,134 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/hyperclovax/modular_hyperclovax.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_hyperclovax.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 NAVER CLOUD Corp. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...modeling_rope_utils import RopeParameters +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="naver-hyperclovax/HyperCLOVAX-SEED-Think-14B") +@strict +class HyperCLOVAXConfig(PreTrainedConfig): + r""" + embedding_multiplier (`float`, *optional*, defaults to `1.0`): + Scaling factor applied to the token embedding outputs. Used in MuP to control the + scale of the embedding activations. + logits_scaling (`float`, *optional*, defaults to `1.0`): + Scaling factor **multiplied** to the final logits before loss computation or sampling. + Used in MuP to ensure consistent output scale across model sizes. Note: unlike + [`GraniteConfig`], this is a multiplier, not a divisor. + residual_multiplier (`float`, *optional*, defaults to `1.0`): + Scaling factor applied to each sub-layer output before adding to the residual stream. + Used in Maximal Update Parametrization (MuP) to stabilize training across model sizes. + attention_multiplier (`float`, *optional*, defaults to `head_dim ** -0.5`): + Scaling factor applied to attention logits before softmax, replacing the standard + `1 / sqrt(head_dim)` scaling. Set explicitly for MuP-based training; when `None`, + defaults to the standard value. + use_post_norm (`bool`, *optional*, defaults to `True`): + Whether to apply an extra RMSNorm after each sub-layer output (Peri-Layer Normalization). + + ```python + >>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig + + >>> # Initializing a HyperCLOVAX style configuration + >>> configuration = HyperCLOVAXConfig() + + >>> # Initializing a model from the configuration + >>> model = HyperCLOVAXModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "hyperclovax" + keys_to_ignore_at_inference = ["past_key_values"] + # Default tensor parallel plan for base model `HyperCLOVAXModel` + base_model_tp_plan = { + "layers.*.self_attn.q_proj": "colwise", + "layers.*.self_attn.k_proj": "colwise", + "layers.*.self_attn.v_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 32000 + hidden_size: int = 4096 + intermediate_size: int = 11008 + num_hidden_layers: int = 32 + num_attention_heads: int = 32 + num_key_value_heads: int | None = None + hidden_act: str = "silu" + max_position_embeddings: int = 2048 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-6 + use_cache: bool = True + pad_token_id: int | None = None + bos_token_id: int | None = 1 + eos_token_id: int | list[int] | None = 2 + tie_word_embeddings: bool = False + rope_parameters: RopeParameters | dict | None = None + attention_bias: bool = False + attention_dropout: float | int = 0.0 + mlp_bias: bool = False + embedding_multiplier: float | int = 1.0 + logits_scaling: float | int = 1.0 + residual_multiplier: float | int = 1.0 + + # MuP scaling factors: None means "resolve to the mathematically equivalent default". + attention_multiplier: float | None = None + + head_dim: int | None = None + + # Peri-Layer Normalization + use_post_norm: bool = True + + def __post_init__( + self, + **kwargs, + ): + if self.head_dim is None: + self.head_dim = self.hidden_size // self.num_attention_heads + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + + super().__post_init__(**kwargs) + + # Resolve None MuP values to their mathematically equivalent defaults. + if self.attention_multiplier is None: + self.attention_multiplier = self.head_dim**-0.5 + + def validate_architecture(self): + """Validates that `hidden_size` is divisible by `num_attention_heads`.""" + if self.hidden_size % self.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention " + f"heads ({self.num_attention_heads})." + ) + + +__all__ = ["HyperCLOVAXConfig"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/modeling_hyperclovax.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/modeling_hyperclovax.py new file mode 100644 index 0000000000000000000000000000000000000000..3608d215bfa9e5574d14abb15657912ad756b4ea --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/hyperclovax/modeling_hyperclovax.py @@ -0,0 +1,526 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/hyperclovax/modular_hyperclovax.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_hyperclovax.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 NAVER CLOUD Corp. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections.abc import Callable +from typing import Optional + +import torch +import torch.nn as nn + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub, use_kernel_func_from_hub, use_kernelized_func +from ...masking_utils import create_causal_mask +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import maybe_autocast, merge_with_config_defaults +from ...utils.output_capturing import capture_outputs +from .configuration_hyperclovax import HyperCLOVAXConfig + + +@use_kernel_forward_from_hub("RMSNorm") +class HyperCLOVAXRMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + HyperCLOVAXRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +class HyperCLOVAXRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: HyperCLOVAXConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters( + config: HyperCLOVAXConfig | None = None, + device: Optional["torch.device"] = None, + seq_len: int | None = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with maybe_autocast(device_type=device_type, enabled=False): # Force float32 + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +@use_kernel_func_from_hub("rotary_pos_emb") +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +@use_kernelized_func(apply_rotary_pos_emb) +class HyperCLOVAXAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: HyperCLOVAXConfig, layer_idx: int | None = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + self.scaling = config.attention_multiplier + self.attention_dropout = config.attention_dropout + self.is_causal = True + + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple[torch.Tensor, torch.Tensor]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_values is not None: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class HyperCLOVAXMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=config.mlp_bias) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +class HyperCLOVAXDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: HyperCLOVAXConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = HyperCLOVAXAttention(config=config, layer_idx=layer_idx) + + self.mlp = HyperCLOVAXMLP(config) + self.input_layernorm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.residual_multiplier = config.residual_multiplier + # Optional Peri-Layer Normalization: additional RMSNorm after each sub-layer output + self.post_norm1 = ( + HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps) if config.use_post_norm else nn.Identity() + ) + self.post_norm2 = ( + HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps) if config.use_post_norm else nn.Identity() + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): + attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, + query_sequence_length, key_sequence_length)` if default attention is used. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_values (`Cache`, *optional*): cached past key and value projection states + position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): + Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, + with `head_dim` being the embedding dimension of each attention head. + kwargs (`dict`, *optional*): + Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code + into the model + """ + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = self.post_norm1(hidden_states) + hidden_states = residual + hidden_states * self.residual_multiplier + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_norm2(hidden_states) + hidden_states = residual + hidden_states * self.residual_multiplier + return hidden_states + + +@auto_docstring +class HyperCLOVAXPreTrainedModel(PreTrainedModel): + config: HyperCLOVAXConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["HyperCLOVAXDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": HyperCLOVAXDecoderLayer, + "attentions": HyperCLOVAXAttention, + } + + +@auto_docstring +class HyperCLOVAXModel(HyperCLOVAXPreTrainedModel): + def __init__(self, config: HyperCLOVAXConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [HyperCLOVAXDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = HyperCLOVAXRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = HyperCLOVAXRotaryEmbedding(config=config) + self.gradient_checkpointing = False + self.embedding_multiplier = config.embedding_multiplier + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + inputs_embeds = inputs_embeds * self.embedding_multiplier + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if position_ids is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + position_embeddings=position_embeddings, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring +class HyperCLOVAXForCausalLM(HyperCLOVAXPreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = HyperCLOVAXModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM + + >>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B") + >>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked. + ```""" + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + # MuP: multiply logits by logits_scaling (cf. GraniteForCausalLM which divides) + logits = self.lm_head(hidden_states[:, slice_indices, :]) * self.config.logits_scaling + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["HyperCLOVAXPreTrainedModel", "HyperCLOVAXModel", "HyperCLOVAXForCausalLM"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cf34ec43ac1014d8c153b3aa259e394fc7b73570 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_ibert import * + from .modeling_ibert import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/configuration_ibert.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/configuration_ibert.py new file mode 100644 index 0000000000000000000000000000000000000000..7793f51aa23dc5039d3612578ed0896424258f42 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/configuration_ibert.py @@ -0,0 +1,62 @@ +# Copyright 2021 The I-BERT Authors (Sehoon Kim, Amir Gholami, Zhewei Yao, +# Michael Mahoney, Kurt Keutzer - UC Berkeley) and The HuggingFace Inc. team. +# Copyright (c) 20121, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""I-BERT configuration""" + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="kssteven/ibert-roberta-base") +@strict +class IBertConfig(PreTrainedConfig): + r""" + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling [`IBertModel`] + quant_mode (`bool`, *optional*, defaults to `False`): + Whether to quantize the model or not. + force_dequant (`str`, *optional*, defaults to `"none"`): + Force dequantize specific nonlinear layer. Dequantized layers are then executed with full precision. + `"none"`, `"gelu"`, `"softmax"`, `"layernorm"` and `"nonlinear"` are supported. As default, it is set as + `"none"`, which does not dequantize any layers. Please specify `"gelu"`, `"softmax"`, or `"layernorm"` to + dequantize GELU, Softmax, or LayerNorm, respectively. `"nonlinear"` will dequantize all nonlinear layers, + i.e., GELU, Softmax, and LayerNorm. + """ + + model_type = "ibert" + + vocab_size: int = 30522 + hidden_size: int = 768 + num_hidden_layers: int = 12 + num_attention_heads: int = 12 + intermediate_size: int = 3072 + hidden_act: str = "gelu" + hidden_dropout_prob: float | int = 0.1 + attention_probs_dropout_prob: float | int = 0.1 + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + layer_norm_eps: float = 1e-12 + pad_token_id: int | None = 1 + bos_token_id: int | None = 0 + eos_token_id: int | list[int] | None = 2 + quant_mode: bool = False + force_dequant: str = "none" + tie_word_embeddings: bool = True + + +__all__ = ["IBertConfig"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/modeling_ibert.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/modeling_ibert.py new file mode 100644 index 0000000000000000000000000000000000000000..90f3d794633cc9f59d59e6cba27c5bfd81c9d200 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ibert/modeling_ibert.py @@ -0,0 +1,1202 @@ +# Copyright 2021 The I-BERT Authors (Sehoon Kim, Amir Gholami, Zhewei Yao, +# Michael Mahoney, Kurt Keutzer - UC Berkeley) and The HuggingFace Inc. team. +# Copyright (c) 20121, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PyTorch I-BERT model.""" + +import math + +import torch +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ... import initialization as init +from ...activations import gelu +from ...masking_utils import create_bidirectional_mask +from ...modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + MaskedLMOutput, + MultipleChoiceModelOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import auto_docstring, logging +from .configuration_ibert import IBertConfig +from .quant_modules import IntGELU, IntLayerNorm, IntSoftmax, QuantAct, QuantEmbedding, QuantLinear + + +logger = logging.get_logger(__name__) + + +class IBertEmbeddings(nn.Module): + """ + Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. + """ + + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.embedding_bit = 8 + self.embedding_act_bit = 16 + self.act_bit = 8 + self.ln_input_bit = 22 + self.ln_output_bit = 32 + + self.word_embeddings = QuantEmbedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id, + weight_bit=self.embedding_bit, + quant_mode=self.quant_mode, + ) + self.token_type_embeddings = QuantEmbedding( + config.type_vocab_size, config.hidden_size, weight_bit=self.embedding_bit, quant_mode=self.quant_mode + ) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False + ) + + # End copy + self.padding_idx = config.pad_token_id + self.position_embeddings = QuantEmbedding( + config.max_position_embeddings, + config.hidden_size, + padding_idx=self.padding_idx, + weight_bit=self.embedding_bit, + quant_mode=self.quant_mode, + ) + + # Integer-only addition between embeddings + self.embeddings_act1 = QuantAct(self.embedding_act_bit, quant_mode=self.quant_mode) + self.embeddings_act2 = QuantAct(self.embedding_act_bit, quant_mode=self.quant_mode) + + self.LayerNorm = IntLayerNorm( + config.hidden_size, + eps=config.layer_norm_eps, + output_bit=self.ln_output_bit, + quant_mode=self.quant_mode, + force_dequant=config.force_dequant, + ) + self.output_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward( + self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0 + ): + if position_ids is None: + if input_ids is not None: + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = create_position_ids_from_input_ids( + input_ids, self.padding_idx, past_key_values_length + ).to(input_ids.device) + else: + position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds) + + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=self.position_ids.device) + + if inputs_embeds is None: + inputs_embeds, inputs_embeds_scaling_factor = self.word_embeddings(input_ids) + else: + inputs_embeds_scaling_factor = None + token_type_embeddings, token_type_embeddings_scaling_factor = self.token_type_embeddings(token_type_ids) + + embeddings, embeddings_scaling_factor = self.embeddings_act1( + inputs_embeds, + inputs_embeds_scaling_factor, + identity=token_type_embeddings, + identity_scaling_factor=token_type_embeddings_scaling_factor, + ) + + position_embeddings, position_embeddings_scaling_factor = self.position_embeddings(position_ids) + embeddings, embeddings_scaling_factor = self.embeddings_act1( + embeddings, + embeddings_scaling_factor, + identity=position_embeddings, + identity_scaling_factor=position_embeddings_scaling_factor, + ) + + embeddings, embeddings_scaling_factor = self.LayerNorm(embeddings, embeddings_scaling_factor) + embeddings = self.dropout(embeddings) + embeddings, embeddings_scaling_factor = self.output_activation(embeddings, embeddings_scaling_factor) + return embeddings, embeddings_scaling_factor + + def create_position_ids_from_inputs_embeds(self, inputs_embeds): + """ + We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. + + Args: + inputs_embeds: torch.Tensor + + Returns: torch.Tensor + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange( + self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=torch.long, device=inputs_embeds.device + ) + return position_ids.unsqueeze(0).expand(input_shape) + + +class IBertSelfAttention(nn.Module): + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"): + raise ValueError( + f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention " + f"heads ({config.num_attention_heads})" + ) + self.quant_mode = config.quant_mode + self.weight_bit = 8 + self.bias_bit = 32 + self.act_bit = 8 + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + # Q, K, V Linear layers + self.query = QuantLinear( + config.hidden_size, + self.all_head_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + self.key = QuantLinear( + config.hidden_size, + self.all_head_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + self.value = QuantLinear( + config.hidden_size, + self.all_head_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + + # Requantization (32bit -> 8bit) for Q, K, V activations + self.query_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.key_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.value_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.output_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + self.softmax = IntSoftmax(self.act_bit, quant_mode=self.quant_mode, force_dequant=config.force_dequant) + + def forward( + self, + hidden_states, + hidden_states_scaling_factor, + attention_mask=None, + output_attentions=False, + ): + # Projection + mixed_query_layer, mixed_query_layer_scaling_factor = self.query(hidden_states, hidden_states_scaling_factor) + mixed_key_layer, mixed_key_layer_scaling_factor = self.key(hidden_states, hidden_states_scaling_factor) + mixed_value_layer, mixed_value_layer_scaling_factor = self.value(hidden_states, hidden_states_scaling_factor) + + # Requantization + query_layer, query_layer_scaling_factor = self.query_activation( + mixed_query_layer, mixed_query_layer_scaling_factor + ) + key_layer, key_layer_scaling_factor = self.key_activation(mixed_key_layer, mixed_key_layer_scaling_factor) + value_layer, value_layer_scaling_factor = self.value_activation( + mixed_value_layer, mixed_value_layer_scaling_factor + ) + + # Transpose + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.attention_head_size) + query_layer = query_layer.view(hidden_shape).transpose(1, 2) + key_layer = key_layer.view(hidden_shape).transpose(1, 2) + value_layer = value_layer.view(hidden_shape).transpose(1, 2) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + scale = math.sqrt(self.attention_head_size) + attention_scores = attention_scores / scale + if self.quant_mode: + attention_scores_scaling_factor = query_layer_scaling_factor * key_layer_scaling_factor / scale + else: + attention_scores_scaling_factor = None + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in IBertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs, attention_probs_scaling_factor = self.softmax( + attention_scores, attention_scores_scaling_factor + ) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + context_layer = torch.matmul(attention_probs, value_layer) + if attention_probs_scaling_factor is not None: + context_layer_scaling_factor = attention_probs_scaling_factor * value_layer_scaling_factor + else: + context_layer_scaling_factor = None + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + # requantization: 32-bit -> 8-bit + context_layer, context_layer_scaling_factor = self.output_activation( + context_layer, context_layer_scaling_factor + ) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + output_scaling_factor = ( + (context_layer_scaling_factor, attention_probs_scaling_factor) + if output_attentions + else (context_layer_scaling_factor,) + ) + + return outputs, output_scaling_factor + + +class IBertSelfOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.act_bit = 8 + self.weight_bit = 8 + self.bias_bit = 32 + self.ln_input_bit = 22 + self.ln_output_bit = 32 + + self.dense = QuantLinear( + config.hidden_size, + config.hidden_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + self.ln_input_act = QuantAct(self.ln_input_bit, quant_mode=self.quant_mode) + self.LayerNorm = IntLayerNorm( + config.hidden_size, + eps=config.layer_norm_eps, + output_bit=self.ln_output_bit, + quant_mode=self.quant_mode, + force_dequant=config.force_dequant, + ) + self.output_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, hidden_states_scaling_factor, input_tensor, input_tensor_scaling_factor): + hidden_states, hidden_states_scaling_factor = self.dense(hidden_states, hidden_states_scaling_factor) + hidden_states = self.dropout(hidden_states) + hidden_states, hidden_states_scaling_factor = self.ln_input_act( + hidden_states, + hidden_states_scaling_factor, + identity=input_tensor, + identity_scaling_factor=input_tensor_scaling_factor, + ) + hidden_states, hidden_states_scaling_factor = self.LayerNorm(hidden_states, hidden_states_scaling_factor) + + hidden_states, hidden_states_scaling_factor = self.output_activation( + hidden_states, hidden_states_scaling_factor + ) + return hidden_states, hidden_states_scaling_factor + + +class IBertAttention(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.self = IBertSelfAttention(config) + self.output = IBertSelfOutput(config) + + def forward( + self, + hidden_states, + hidden_states_scaling_factor, + attention_mask=None, + output_attentions=False, + ): + self_outputs, self_outputs_scaling_factor = self.self( + hidden_states, + hidden_states_scaling_factor, + attention_mask, + output_attentions, + ) + attention_output, attention_output_scaling_factor = self.output( + self_outputs[0], self_outputs_scaling_factor[0], hidden_states, hidden_states_scaling_factor + ) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + outputs_scaling_factor = (attention_output_scaling_factor,) + self_outputs_scaling_factor[1:] + return outputs, outputs_scaling_factor + + +class IBertIntermediate(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.act_bit = 8 + self.weight_bit = 8 + self.bias_bit = 32 + self.dense = QuantLinear( + config.hidden_size, + config.intermediate_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + if config.hidden_act != "gelu": + raise ValueError("I-BERT only supports 'gelu' for `config.hidden_act`") + self.intermediate_act_fn = IntGELU(quant_mode=self.quant_mode, force_dequant=config.force_dequant) + self.output_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + + def forward(self, hidden_states, hidden_states_scaling_factor): + hidden_states, hidden_states_scaling_factor = self.dense(hidden_states, hidden_states_scaling_factor) + hidden_states, hidden_states_scaling_factor = self.intermediate_act_fn( + hidden_states, hidden_states_scaling_factor + ) + + # Requantization: 32bit -> 8-bit + hidden_states, hidden_states_scaling_factor = self.output_activation( + hidden_states, hidden_states_scaling_factor + ) + return hidden_states, hidden_states_scaling_factor + + +class IBertOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.act_bit = 8 + self.weight_bit = 8 + self.bias_bit = 32 + self.ln_input_bit = 22 + self.ln_output_bit = 32 + + self.dense = QuantLinear( + config.intermediate_size, + config.hidden_size, + bias=True, + weight_bit=self.weight_bit, + bias_bit=self.bias_bit, + quant_mode=self.quant_mode, + per_channel=True, + ) + self.ln_input_act = QuantAct(self.ln_input_bit, quant_mode=self.quant_mode) + self.LayerNorm = IntLayerNorm( + config.hidden_size, + eps=config.layer_norm_eps, + output_bit=self.ln_output_bit, + quant_mode=self.quant_mode, + force_dequant=config.force_dequant, + ) + self.output_activation = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, hidden_states_scaling_factor, input_tensor, input_tensor_scaling_factor): + hidden_states, hidden_states_scaling_factor = self.dense(hidden_states, hidden_states_scaling_factor) + hidden_states = self.dropout(hidden_states) + hidden_states, hidden_states_scaling_factor = self.ln_input_act( + hidden_states, + hidden_states_scaling_factor, + identity=input_tensor, + identity_scaling_factor=input_tensor_scaling_factor, + ) + hidden_states, hidden_states_scaling_factor = self.LayerNorm(hidden_states, hidden_states_scaling_factor) + + hidden_states, hidden_states_scaling_factor = self.output_activation( + hidden_states, hidden_states_scaling_factor + ) + return hidden_states, hidden_states_scaling_factor + + +class IBertLayer(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.act_bit = 8 + + self.seq_len_dim = 1 + self.attention = IBertAttention(config) + self.intermediate = IBertIntermediate(config) + self.output = IBertOutput(config) + + self.pre_intermediate_act = QuantAct(self.act_bit, quant_mode=self.quant_mode) + self.pre_output_act = QuantAct(self.act_bit, quant_mode=self.quant_mode) + + def forward( + self, + hidden_states, + hidden_states_scaling_factor, + attention_mask=None, + output_attentions=False, + ): + self_attention_outputs, self_attention_outputs_scaling_factor = self.attention( + hidden_states, + hidden_states_scaling_factor, + attention_mask, + output_attentions=output_attentions, + ) + attention_output = self_attention_outputs[0] + attention_output_scaling_factor = self_attention_outputs_scaling_factor[0] + + outputs = self_attention_outputs[1:] # add self attentions if we output attention weights + + layer_output, layer_output_scaling_factor = self.feed_forward_chunk( + attention_output, attention_output_scaling_factor + ) + outputs = (layer_output,) + outputs + + return outputs + + def feed_forward_chunk(self, attention_output, attention_output_scaling_factor): + attention_output, attention_output_scaling_factor = self.pre_intermediate_act( + attention_output, attention_output_scaling_factor + ) + intermediate_output, intermediate_output_scaling_factor = self.intermediate( + attention_output, attention_output_scaling_factor + ) + + intermediate_output, intermediate_output_scaling_factor = self.pre_output_act( + intermediate_output, intermediate_output_scaling_factor + ) + layer_output, layer_output_scaling_factor = self.output( + intermediate_output, intermediate_output_scaling_factor, attention_output, attention_output_scaling_factor + ) + return layer_output, layer_output_scaling_factor + + +class IBertEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.quant_mode = config.quant_mode + self.layer = nn.ModuleList([IBertLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states, + hidden_states_scaling_factor, + attention_mask=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = None # `config.add_cross_attention` is not supported + + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module( + hidden_states, + hidden_states_scaling_factor, + attention_mask, + output_attentions, + ) + + hidden_states = layer_outputs[0] + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class IBertPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.quant_mode = config.quant_mode + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +@auto_docstring +class IBertPreTrainedModel(PreTrainedModel): + config: IBertConfig + base_model_prefix = "ibert" + + @torch.no_grad() + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, (QuantLinear, nn.Linear)): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + init.zeros_(module.bias) + if getattr(module, "weight_integer", None) is not None: + init.zeros_(module.weight_integer) + init.zeros_(module.fc_scaling_factor) + if getattr(module, "bias_integer", None) is not None: + init.zeros_(module.bias_integer) + elif isinstance(module, (QuantEmbedding, nn.Embedding)): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + # Here we need the check explicitly, as we slice the weight in the `zeros_` call, so it looses the flag + if module.padding_idx is not None and not getattr(module.weight, "_is_hf_initialized", False): + init.zeros_(module.weight[module.padding_idx]) + if getattr(module, "weight_scaling_factor", None) is not None: + init.zeros_(module.weight_scaling_factor) + init.zeros_(module.weight_integer) + elif isinstance(module, (IntLayerNorm, nn.LayerNorm)): + init.zeros_(module.bias) + init.ones_(module.weight) + if getattr(module, "shift", None) is not None: + init.zeros_(module.shift) + elif isinstance(module, IBertLMHead): + init.zeros_(module.bias) + elif isinstance(module, IBertEmbeddings): + init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1))) + elif isinstance(module, QuantAct): + init.constant_(module.x_min, -1e-5) + init.constant_(module.x_max, 1e-5) + init.zeros_(module.act_scaling_factor) + + def resize_token_embeddings(self, new_num_tokens=None): + raise NotImplementedError("`resize_token_embeddings` is not supported for I-BERT.") + + +@auto_docstring +class IBertModel(IBertPreTrainedModel): + """ + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in [Attention is + all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + """ + + def __init__(self, config, add_pooling_layer=True): + r""" + add_pooling_layer (bool, *optional*, defaults to `True`): + Whether to add a pooling layer + """ + super().__init__(config) + self.config = config + self.quant_mode = config.quant_mode + + self.embeddings = IBertEmbeddings(config) + self.encoder = IBertEncoder(config) + + self.pooler = IBertPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> BaseModelOutputWithPoolingAndCrossAttentions | tuple[torch.FloatTensor]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask) + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + batch_size, seq_length = input_shape + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(((batch_size, seq_length)), device=device) + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + embedding_output, embedding_output_scaling_factor = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + ) + + attention_mask = create_bidirectional_mask( + config=self.config, + inputs_embeds=embedding_output, + attention_mask=attention_mask, + ) + + encoder_outputs = self.encoder( + embedding_output, + embedding_output_scaling_factor, + attention_mask=attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +@auto_docstring +class IBertForMaskedLM(IBertPreTrainedModel): + _tied_weights_keys = { + "lm_head.decoder.weight": "ibert.embeddings.word_embeddings.weight$", + "lm_head.decoder.bias": "lm_head.bias", + } + + def __init__(self, config): + super().__init__(config) + + self.ibert = IBertModel(config, add_pooling_layer=False) + self.lm_head = IBertLMHead(config) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.lm_head.decoder + + def set_output_embeddings(self, new_embeddings): + self.lm_head.decoder = new_embeddings + self.lm_head.bias = new_embeddings.bias + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> MaskedLMOutput | tuple[torch.FloatTensor]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + + outputs = self.ibert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + prediction_scores = self.lm_head(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class IBertLMHead(nn.Module): + """I-BERT Head for masked language modeling.""" + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + self.decoder = nn.Linear(config.hidden_size, config.vocab_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + def forward(self, features, **kwargs): + x = self.dense(features) + x = gelu(x) + x = self.layer_norm(x) + + # project back to size of vocabulary with bias + x = self.decoder(x) + + return x + + +@auto_docstring( + custom_intro=""" + I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled + output) e.g. for GLUE tasks. + """ +) +class IBertForSequenceClassification(IBertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.ibert = IBertModel(config, add_pooling_layer=False) + self.classifier = IBertClassificationHead(config) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> SequenceClassifierOutput | tuple[torch.FloatTensor]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + + outputs = self.ibert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@auto_docstring +class IBertForMultipleChoice(IBertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.ibert = IBertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> MultipleChoiceModelOutput | tuple[torch.FloatTensor]: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, + 1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + + [What are token type IDs?](../glossary#token-type-ids) + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., + num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See + `input_ids` above) + position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] + + flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None + flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + flat_inputs_embeds = ( + inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1)) + if inputs_embeds is not None + else None + ) + + outputs = self.ibert( + flat_input_ids, + position_ids=flat_position_ids, + token_type_ids=flat_token_type_ids, + attention_mask=flat_attention_mask, + inputs_embeds=flat_inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + + if not return_dict: + output = (reshaped_logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return MultipleChoiceModelOutput( + loss=loss, + logits=reshaped_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@auto_docstring +class IBertForTokenClassification(IBertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.ibert = IBertModel(config, add_pooling_layer=False) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> TokenClassifierOutput | tuple[torch.FloatTensor]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + + outputs = self.ibert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class IBertClassificationHead(nn.Module): + """Head for sentence-level classification tasks.""" + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.out_proj = nn.Linear(config.hidden_size, config.num_labels) + + def forward(self, features, **kwargs): + hidden_states = features[:, 0, :] # take token (equiv. to [CLS]) + hidden_states = self.dropout(hidden_states) + hidden_states = self.dense(hidden_states) + hidden_states = torch.tanh(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.out_proj(hidden_states) + return hidden_states + + +@auto_docstring +class IBertForQuestionAnswering(IBertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.ibert = IBertModel(config, add_pooling_layer=False) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.FloatTensor | None = None, + token_type_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + start_positions: torch.LongTensor | None = None, + end_positions: torch.LongTensor | None = None, + output_attentions: bool | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> QuestionAnsweringModelOutput | tuple[torch.FloatTensor]: + return_dict = return_dict if return_dict is not None else self.config.return_dict + + outputs = self.ibert( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions = start_positions.clamp(0, ignored_index) + end_positions = end_positions.clamp(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if not return_dict: + output = (start_logits, end_logits) + outputs[2:] + return ((total_loss,) + output) if total_loss is not None else output + + return QuestionAnsweringModelOutput( + loss=total_loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0): + """ + Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols + are ignored. This is modified from fairseq's *utils.make_positions*. + + Args: + input_ids (`torch.LongTensor`): + Indices of input sequence tokens in the vocabulary. + + Returns: torch.Tensor + """ + # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA. + mask = input_ids.ne(padding_idx).int() + incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask + return incremental_indices.long() + padding_idx + + +__all__ = [ + "IBertForMaskedLM", + "IBertForMultipleChoice", + "IBertForQuestionAnswering", + "IBertForSequenceClassification", + "IBertForTokenClassification", + "IBertModel", + "IBertPreTrainedModel", +] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15e10a1d9815935629fadf9405effdae5a16fbb2 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_mobilevitv2 import * + from .modeling_mobilevitv2 import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.py new file mode 100644 index 0000000000000000000000000000000000000000..2e28695ae1f2d1521a5b606903335bf3985cadb6 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.py @@ -0,0 +1,942 @@ +# Copyright 2023 Apple Inc. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Original license: https://github.com/apple/ml-cvnets/blob/main/LICENSE +"""PyTorch MobileViTV2 model.""" + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss + +from ... import initialization as init +from ...activations import ACT2FN +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import ( + BaseModelOutputWithNoAttention, + BaseModelOutputWithPoolingAndNoAttention, + ImageClassifierOutputWithNoAttention, + SemanticSegmenterOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import auto_docstring, logging +from .configuration_mobilevitv2 import MobileViTV2Config + + +logger = logging.get_logger(__name__) + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.make_divisible +def make_divisible(value: int, divisor: int = 8, min_value: int | None = None) -> int: + """ + Ensure that all layers have a channel count that is divisible by `divisor`. + """ + if min_value is None: + min_value = divisor + new_value = max(min_value, int(value + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_value < 0.9 * value: + new_value += divisor + return int(new_value) + + +def clip(value: float, min_val: float = float("-inf"), max_val: float = float("inf")) -> float: + return max(min_val, min(max_val, value)) + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTConvLayer with MobileViT->MobileViTV2 +class MobileViTV2ConvLayer(nn.Module): + def __init__( + self, + config: MobileViTV2Config, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + groups: int = 1, + bias: bool = False, + dilation: int = 1, + use_normalization: bool = True, + use_activation: bool | str = True, + ) -> None: + super().__init__() + padding = int((kernel_size - 1) / 2) * dilation + + if in_channels % groups != 0: + raise ValueError(f"Input channels ({in_channels}) are not divisible by {groups} groups.") + if out_channels % groups != 0: + raise ValueError(f"Output channels ({out_channels}) are not divisible by {groups} groups.") + + self.convolution = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias, + padding_mode="zeros", + ) + + if use_normalization: + self.normalization = nn.BatchNorm2d( + num_features=out_channels, + eps=1e-5, + momentum=0.1, + affine=True, + track_running_stats=True, + ) + else: + self.normalization = None + + if use_activation: + if isinstance(use_activation, str): + self.activation = ACT2FN[use_activation] + elif isinstance(config.hidden_act, str): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + else: + self.activation = None + + def forward(self, features: torch.Tensor) -> torch.Tensor: + features = self.convolution(features) + if self.normalization is not None: + features = self.normalization(features) + if self.activation is not None: + features = self.activation(features) + return features + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTInvertedResidual with MobileViT->MobileViTV2 +class MobileViTV2InvertedResidual(nn.Module): + """ + Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381 + """ + + def __init__( + self, config: MobileViTV2Config, in_channels: int, out_channels: int, stride: int, dilation: int = 1 + ) -> None: + super().__init__() + expanded_channels = make_divisible(int(round(in_channels * config.expand_ratio)), 8) + + if stride not in [1, 2]: + raise ValueError(f"Invalid stride {stride}.") + + self.use_residual = (stride == 1) and (in_channels == out_channels) + + self.expand_1x1 = MobileViTV2ConvLayer( + config, in_channels=in_channels, out_channels=expanded_channels, kernel_size=1 + ) + + self.conv_3x3 = MobileViTV2ConvLayer( + config, + in_channels=expanded_channels, + out_channels=expanded_channels, + kernel_size=3, + stride=stride, + groups=expanded_channels, + dilation=dilation, + ) + + self.reduce_1x1 = MobileViTV2ConvLayer( + config, + in_channels=expanded_channels, + out_channels=out_channels, + kernel_size=1, + use_activation=False, + ) + + def forward(self, features: torch.Tensor) -> torch.Tensor: + residual = features + + features = self.expand_1x1(features) + features = self.conv_3x3(features) + features = self.reduce_1x1(features) + + return residual + features if self.use_residual else features + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTMobileNetLayer with MobileViT->MobileViTV2 +class MobileViTV2MobileNetLayer(nn.Module): + def __init__( + self, config: MobileViTV2Config, in_channels: int, out_channels: int, stride: int = 1, num_stages: int = 1 + ) -> None: + super().__init__() + + self.layer = nn.ModuleList() + for i in range(num_stages): + layer = MobileViTV2InvertedResidual( + config, + in_channels=in_channels, + out_channels=out_channels, + stride=stride if i == 0 else 1, + ) + self.layer.append(layer) + in_channels = out_channels + + def forward(self, features: torch.Tensor) -> torch.Tensor: + for layer_module in self.layer: + features = layer_module(features) + return features + + +class MobileViTV2LinearSelfAttention(nn.Module): + """ + This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper: + https://huggingface.co/papers/2206.02680 + + Args: + config (`MobileVitv2Config`): + Model configuration object + embed_dim (`int`): + `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)` + """ + + def __init__(self, config: MobileViTV2Config, embed_dim: int) -> None: + super().__init__() + + self.qkv_proj = MobileViTV2ConvLayer( + config=config, + in_channels=embed_dim, + out_channels=1 + (2 * embed_dim), + bias=True, + kernel_size=1, + use_normalization=False, + use_activation=False, + ) + + self.attn_dropout = nn.Dropout(p=config.attn_dropout) + self.out_proj = MobileViTV2ConvLayer( + config=config, + in_channels=embed_dim, + out_channels=embed_dim, + bias=True, + kernel_size=1, + use_normalization=False, + use_activation=False, + ) + self.embed_dim = embed_dim + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # (batch_size, embed_dim, num_pixels_in_patch, num_patches) --> (batch_size, 1+2*embed_dim, num_pixels_in_patch, num_patches) + qkv = self.qkv_proj(hidden_states) + + # Project hidden_states into query, key and value + # Query --> [batch_size, 1, num_pixels_in_patch, num_patches] + # value, key --> [batch_size, embed_dim, num_pixels_in_patch, num_patches] + query, key, value = torch.split(qkv, split_size_or_sections=[1, self.embed_dim, self.embed_dim], dim=1) + + # apply softmax along num_patches dimension + context_scores = torch.nn.functional.softmax(query, dim=-1) + context_scores = self.attn_dropout(context_scores) + + # Compute context vector + # [batch_size, embed_dim, num_pixels_in_patch, num_patches] x [batch_size, 1, num_pixels_in_patch, num_patches] -> [batch_size, embed_dim, num_pixels_in_patch, num_patches] + context_vector = key * context_scores + # [batch_size, embed_dim, num_pixels_in_patch, num_patches] --> [batch_size, embed_dim, num_pixels_in_patch, 1] + context_vector = torch.sum(context_vector, dim=-1, keepdim=True) + + # combine context vector with values + # [batch_size, embed_dim, num_pixels_in_patch, num_patches] * [batch_size, embed_dim, num_pixels_in_patch, 1] --> [batch_size, embed_dim, num_pixels_in_patch, num_patches] + out = torch.nn.functional.relu(value) * context_vector.expand_as(value) + out = self.out_proj(out) + return out + + +class MobileViTV2FFN(nn.Module): + def __init__( + self, + config: MobileViTV2Config, + embed_dim: int, + ffn_latent_dim: int, + ffn_dropout: float = 0.0, + ) -> None: + super().__init__() + self.conv1 = MobileViTV2ConvLayer( + config=config, + in_channels=embed_dim, + out_channels=ffn_latent_dim, + kernel_size=1, + stride=1, + bias=True, + use_normalization=False, + use_activation=True, + ) + self.dropout1 = nn.Dropout(ffn_dropout) + + self.conv2 = MobileViTV2ConvLayer( + config=config, + in_channels=ffn_latent_dim, + out_channels=embed_dim, + kernel_size=1, + stride=1, + bias=True, + use_normalization=False, + use_activation=False, + ) + self.dropout2 = nn.Dropout(ffn_dropout) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv1(hidden_states) + hidden_states = self.dropout1(hidden_states) + hidden_states = self.conv2(hidden_states) + hidden_states = self.dropout2(hidden_states) + return hidden_states + + +class MobileViTV2TransformerLayer(nn.Module): + def __init__( + self, + config: MobileViTV2Config, + embed_dim: int, + ffn_latent_dim: int, + dropout: float = 0.0, + ) -> None: + super().__init__() + self.layernorm_before = nn.GroupNorm(num_groups=1, num_channels=embed_dim, eps=config.layer_norm_eps) + self.attention = MobileViTV2LinearSelfAttention(config, embed_dim) + self.dropout1 = nn.Dropout(p=dropout) + self.layernorm_after = nn.GroupNorm(num_groups=1, num_channels=embed_dim, eps=config.layer_norm_eps) + self.ffn = MobileViTV2FFN(config, embed_dim, ffn_latent_dim, config.ffn_dropout) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + layernorm_1_out = self.layernorm_before(hidden_states) + attention_output = self.attention(layernorm_1_out) + hidden_states = attention_output + hidden_states + + layer_output = self.layernorm_after(hidden_states) + layer_output = self.ffn(layer_output) + + layer_output = layer_output + hidden_states + return layer_output + + +class MobileViTV2Transformer(nn.Module): + def __init__(self, config: MobileViTV2Config, n_layers: int, d_model: int) -> None: + super().__init__() + + ffn_multiplier = config.ffn_multiplier + + ffn_dims = [ffn_multiplier * d_model] * n_layers + + # ensure that dims are multiple of 16 + ffn_dims = [int((d // 16) * 16) for d in ffn_dims] + + self.layer = nn.ModuleList() + for block_idx in range(n_layers): + transformer_layer = MobileViTV2TransformerLayer( + config, embed_dim=d_model, ffn_latent_dim=ffn_dims[block_idx] + ) + self.layer.append(transformer_layer) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + for layer_module in self.layer: + hidden_states = layer_module(hidden_states) + return hidden_states + + +class MobileViTV2Layer(GradientCheckpointingLayer): + """ + MobileViTV2 layer: https://huggingface.co/papers/2206.02680 + """ + + def __init__( + self, + config: MobileViTV2Config, + in_channels: int, + out_channels: int, + attn_unit_dim: int, + n_attn_blocks: int = 2, + dilation: int = 1, + stride: int = 2, + ) -> None: + super().__init__() + self.patch_width = config.patch_size + self.patch_height = config.patch_size + + cnn_out_dim = attn_unit_dim + + if stride == 2: + self.downsampling_layer = MobileViTV2InvertedResidual( + config, + in_channels=in_channels, + out_channels=out_channels, + stride=stride if dilation == 1 else 1, + dilation=dilation // 2 if dilation > 1 else 1, + ) + in_channels = out_channels + else: + self.downsampling_layer = None + + # Local representations + self.conv_kxk = MobileViTV2ConvLayer( + config, + in_channels=in_channels, + out_channels=in_channels, + kernel_size=config.conv_kernel_size, + groups=in_channels, + ) + self.conv_1x1 = MobileViTV2ConvLayer( + config, + in_channels=in_channels, + out_channels=cnn_out_dim, + kernel_size=1, + use_normalization=False, + use_activation=False, + ) + + # Global representations + self.transformer = MobileViTV2Transformer(config, d_model=attn_unit_dim, n_layers=n_attn_blocks) + + # self.layernorm = MobileViTV2LayerNorm2D(attn_unit_dim, eps=config.layer_norm_eps) + self.layernorm = nn.GroupNorm(num_groups=1, num_channels=attn_unit_dim, eps=config.layer_norm_eps) + + # Fusion + self.conv_projection = MobileViTV2ConvLayer( + config, + in_channels=cnn_out_dim, + out_channels=in_channels, + kernel_size=1, + use_normalization=True, + use_activation=False, + ) + + def unfolding(self, feature_map: torch.Tensor) -> tuple[torch.Tensor, tuple[int, int]]: + batch_size, in_channels, img_height, img_width = feature_map.shape + patches = nn.functional.unfold( + feature_map, + kernel_size=(self.patch_height, self.patch_width), + stride=(self.patch_height, self.patch_width), + ) + patches = patches.reshape(batch_size, in_channels, self.patch_height * self.patch_width, -1) + + return patches, (img_height, img_width) + + def folding(self, patches: torch.Tensor, output_size: tuple[int, int]) -> torch.Tensor: + batch_size, in_dim, patch_size, n_patches = patches.shape + patches = patches.reshape(batch_size, in_dim * patch_size, n_patches) + + feature_map = nn.functional.fold( + patches, + output_size=output_size, + kernel_size=(self.patch_height, self.patch_width), + stride=(self.patch_height, self.patch_width), + ) + + return feature_map + + def forward(self, features: torch.Tensor) -> torch.Tensor: + # reduce spatial dimensions if needed + if self.downsampling_layer: + features = self.downsampling_layer(features) + + # local representation + features = self.conv_kxk(features) + features = self.conv_1x1(features) + + # convert feature map to patches + patches, output_size = self.unfolding(features) + + # learn global representations + patches = self.transformer(patches) + patches = self.layernorm(patches) + + # convert patches back to feature maps + # [batch_size, patch_height, patch_width, input_dim] --> [batch_size, input_dim, patch_height, patch_width] + features = self.folding(patches, output_size) + + features = self.conv_projection(features) + return features + + +class MobileViTV2Encoder(nn.Module): + def __init__(self, config: MobileViTV2Config) -> None: + super().__init__() + self.config = config + + self.layer = nn.ModuleList() + self.gradient_checkpointing = False + + # segmentation architectures like DeepLab and PSPNet modify the strides + # of the classification backbones + dilate_layer_4 = dilate_layer_5 = False + if config.output_stride == 8: + dilate_layer_4 = True + dilate_layer_5 = True + elif config.output_stride == 16: + dilate_layer_5 = True + + dilation = 1 + + layer_0_dim = make_divisible( + clip(value=32 * config.width_multiplier, min_val=16, max_val=64), divisor=8, min_value=16 + ) + + layer_1_dim = make_divisible(64 * config.width_multiplier, divisor=16) + layer_2_dim = make_divisible(128 * config.width_multiplier, divisor=8) + layer_3_dim = make_divisible(256 * config.width_multiplier, divisor=8) + layer_4_dim = make_divisible(384 * config.width_multiplier, divisor=8) + layer_5_dim = make_divisible(512 * config.width_multiplier, divisor=8) + + layer_1 = MobileViTV2MobileNetLayer( + config, + in_channels=layer_0_dim, + out_channels=layer_1_dim, + stride=1, + num_stages=1, + ) + self.layer.append(layer_1) + + layer_2 = MobileViTV2MobileNetLayer( + config, + in_channels=layer_1_dim, + out_channels=layer_2_dim, + stride=2, + num_stages=2, + ) + self.layer.append(layer_2) + + layer_3 = MobileViTV2Layer( + config, + in_channels=layer_2_dim, + out_channels=layer_3_dim, + attn_unit_dim=make_divisible(config.base_attn_unit_dims[0] * config.width_multiplier, divisor=8), + n_attn_blocks=config.n_attn_blocks[0], + ) + self.layer.append(layer_3) + + if dilate_layer_4: + dilation *= 2 + + layer_4 = MobileViTV2Layer( + config, + in_channels=layer_3_dim, + out_channels=layer_4_dim, + attn_unit_dim=make_divisible(config.base_attn_unit_dims[1] * config.width_multiplier, divisor=8), + n_attn_blocks=config.n_attn_blocks[1], + dilation=dilation, + ) + self.layer.append(layer_4) + + if dilate_layer_5: + dilation *= 2 + + layer_5 = MobileViTV2Layer( + config, + in_channels=layer_4_dim, + out_channels=layer_5_dim, + attn_unit_dim=make_divisible(config.base_attn_unit_dims[2] * config.width_multiplier, divisor=8), + n_attn_blocks=config.n_attn_blocks[2], + dilation=dilation, + ) + self.layer.append(layer_5) + + def forward( + self, + hidden_states: torch.Tensor, + output_hidden_states: bool = False, + return_dict: bool = True, + ) -> tuple | BaseModelOutputWithNoAttention: + all_hidden_states = () if output_hidden_states else None + + for i, layer_module in enumerate(self.layer): + hidden_states = layer_module(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) + + return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states, hidden_states=all_hidden_states) + + +@auto_docstring +class MobileViTV2PreTrainedModel(PreTrainedModel): + config: MobileViTV2Config + base_model_prefix = "mobilevitv2" + main_input_name = "pixel_values" + input_modalities = ("image",) + supports_gradient_checkpointing = True + _no_split_modules = ["MobileViTV2Layer"] + + @torch.no_grad() + def _init_weights(self, module: nn.Module) -> None: + """Initialize the weights""" + if isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + init.zeros_(module.bias) + if getattr(module, "running_mean", None) is not None: + init.zeros_(module.running_mean) + init.ones_(module.running_var) + init.zeros_(module.num_batches_tracked) + elif isinstance(module, nn.GroupNorm): + init.zeros_(module.bias) + init.ones_(module.weight) + + +@auto_docstring +class MobileViTV2Model(MobileViTV2PreTrainedModel): + def __init__(self, config: MobileViTV2Config, expand_output: bool = True): + r""" + expand_output (`bool`, *optional*, defaults to `True`): + Whether to expand the output of the model. If `True`, the model will output pooled features in addition to + hidden states. If `False`, only the hidden states will be returned. + """ + super().__init__(config) + self.config = config + self.expand_output = expand_output + + layer_0_dim = make_divisible( + clip(value=32 * config.width_multiplier, min_val=16, max_val=64), divisor=8, min_value=16 + ) + + self.conv_stem = MobileViTV2ConvLayer( + config, + in_channels=config.num_channels, + out_channels=layer_0_dim, + kernel_size=3, + stride=2, + use_normalization=True, + use_activation=True, + ) + self.encoder = MobileViTV2Encoder(config) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | BaseModelOutputWithPoolingAndNoAttention: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + embedding_output = self.conv_stem(pixel_values) + + encoder_outputs = self.encoder( + embedding_output, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if self.expand_output: + last_hidden_state = encoder_outputs[0] + + # global average pooling: (batch_size, channels, height, width) -> (batch_size, channels) + pooled_output = torch.mean(last_hidden_state, dim=[-2, -1], keepdim=False) + else: + last_hidden_state = encoder_outputs[0] + pooled_output = None + + if not return_dict: + output = (last_hidden_state, pooled_output) if pooled_output is not None else (last_hidden_state,) + return output + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndNoAttention( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + ) + + +@auto_docstring( + custom_intro=""" + MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for + ImageNet. + """ +) +class MobileViTV2ForImageClassification(MobileViTV2PreTrainedModel): + def __init__(self, config: MobileViTV2Config) -> None: + super().__init__(config) + + self.num_labels = config.num_labels + self.mobilevitv2 = MobileViTV2Model(config) + + out_channels = make_divisible(512 * config.width_multiplier, divisor=8) # layer 5 output dimension + # Classifier head + self.classifier = ( + nn.Linear(in_features=out_channels, out_features=config.num_labels) + if config.num_labels > 0 + else nn.Identity() + ) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + labels: torch.Tensor | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | ImageClassifierOutputWithNoAttention: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the image classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + + outputs = self.mobilevitv2(pixel_values, output_hidden_states=output_hidden_states, return_dict=return_dict) + + pooled_output = outputs.pooler_output if return_dict else outputs[1] + + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + loss = self.loss_function(labels, logits, self.config) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return ImageClassifierOutputWithNoAttention( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + ) + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTASPPPooling with MobileViT->MobileViTV2 +class MobileViTV2ASPPPooling(nn.Module): + def __init__(self, config: MobileViTV2Config, in_channels: int, out_channels: int) -> None: + super().__init__() + + self.global_pool = nn.AdaptiveAvgPool2d(output_size=1) + + self.conv_1x1 = MobileViTV2ConvLayer( + config, + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + use_normalization=True, + use_activation="relu", + ) + + def forward(self, features: torch.Tensor) -> torch.Tensor: + spatial_size = features.shape[-2:] + features = self.global_pool(features) + features = self.conv_1x1(features) + features = nn.functional.interpolate(features, size=spatial_size, mode="bilinear", align_corners=False) + return features + + +class MobileViTV2ASPP(nn.Module): + """ + ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587 + """ + + def __init__(self, config: MobileViTV2Config) -> None: + super().__init__() + + encoder_out_channels = make_divisible(512 * config.width_multiplier, divisor=8) # layer 5 output dimension + in_channels = encoder_out_channels + out_channels = config.aspp_out_channels + + if len(config.atrous_rates) != 3: + raise ValueError("Expected 3 values for atrous_rates") + + self.convs = nn.ModuleList() + + in_projection = MobileViTV2ConvLayer( + config, + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + use_activation="relu", + ) + self.convs.append(in_projection) + + self.convs.extend( + [ + MobileViTV2ConvLayer( + config, + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + dilation=rate, + use_activation="relu", + ) + for rate in config.atrous_rates + ] + ) + + pool_layer = MobileViTV2ASPPPooling(config, in_channels, out_channels) + self.convs.append(pool_layer) + + self.project = MobileViTV2ConvLayer( + config, in_channels=5 * out_channels, out_channels=out_channels, kernel_size=1, use_activation="relu" + ) + + self.dropout = nn.Dropout(p=config.aspp_dropout_prob) + + def forward(self, features: torch.Tensor) -> torch.Tensor: + pyramid = [] + for conv in self.convs: + pyramid.append(conv(features)) + pyramid = torch.cat(pyramid, dim=1) + + pooled_features = self.project(pyramid) + pooled_features = self.dropout(pooled_features) + return pooled_features + + +# Copied from transformers.models.mobilevit.modeling_mobilevit.MobileViTDeepLabV3 with MobileViT->MobileViTV2 +class MobileViTV2DeepLabV3(nn.Module): + """ + DeepLabv3 architecture: https://huggingface.co/papers/1706.05587 + """ + + def __init__(self, config: MobileViTV2Config) -> None: + super().__init__() + self.aspp = MobileViTV2ASPP(config) + + self.dropout = nn.Dropout2d(config.classifier_dropout_prob) + + self.classifier = MobileViTV2ConvLayer( + config, + in_channels=config.aspp_out_channels, + out_channels=config.num_labels, + kernel_size=1, + use_normalization=False, + use_activation=False, + bias=True, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + features = self.aspp(hidden_states[-1]) + features = self.dropout(features) + features = self.classifier(features) + return features + + +@auto_docstring( + custom_intro=""" + MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. + """ +) +class MobileViTV2ForSemanticSegmentation(MobileViTV2PreTrainedModel): + def __init__(self, config: MobileViTV2Config) -> None: + super().__init__(config) + + self.num_labels = config.num_labels + self.mobilevitv2 = MobileViTV2Model(config, expand_output=False) + self.segmentation_head = MobileViTV2DeepLabV3(config) + + # Initialize weights and apply final processing + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values: torch.Tensor | None = None, + labels: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | SemanticSegmenterOutput: + r""" + labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): + Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). + + Examples: + + ```python + >>> import httpx + >>> from io import BytesIO + >>> import torch + >>> from PIL import Image + >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + + >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") + >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") + + >>> inputs = image_processor(images=image, return_tensors="pt") + + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> # logits are of shape (batch_size, num_labels, height, width) + >>> logits = outputs.logits + ```""" + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if labels is not None and self.config.num_labels == 1: + raise ValueError("The number of labels should be greater than one") + + outputs = self.mobilevitv2( + pixel_values, + output_hidden_states=True, # we need the intermediate hidden states + return_dict=return_dict, + ) + + encoder_hidden_states = outputs.hidden_states if return_dict else outputs[1] + + logits = self.segmentation_head(encoder_hidden_states) + + loss = None + if labels is not None: + # upsample logits to the images' original size + upsampled_logits = nn.functional.interpolate( + logits, size=labels.shape[-2:], mode="bilinear", align_corners=False + ) + loss_fct = CrossEntropyLoss(ignore_index=self.config.semantic_loss_ignore_index) + loss = loss_fct(upsampled_logits, labels) + + if not return_dict: + if output_hidden_states: + output = (logits,) + outputs[1:] + else: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SemanticSegmenterOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states if output_hidden_states else None, + attentions=None, + ) + + +__all__ = [ + "MobileViTV2ForImageClassification", + "MobileViTV2ForSemanticSegmentation", + "MobileViTV2Model", + "MobileViTV2PreTrainedModel", +] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..51456aac76b0e0c5666acadf0d63008a7c3b50d9 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_musicgen_melody import * + from .modeling_musicgen_melody import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py new file mode 100644 index 0000000000000000000000000000000000000000..1811fa11e630e2acb7d0065357493c89c74d54dc --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/feature_extraction_musicgen_melody.py @@ -0,0 +1,334 @@ +# Copyright 2024 Meta AI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Feature extractor class for Musicgen Melody +""" + +import copy +from typing import Any + +import numpy as np + +from ...audio_utils import chroma_filter_bank +from ...feature_extraction_sequence_utils import SequenceFeatureExtractor +from ...feature_extraction_utils import BatchFeature +from ...utils import TensorType, is_torch_available, is_torchaudio_available, logging +from ...utils.import_utils import requires + + +if is_torch_available(): + import torch + +if is_torchaudio_available(): + import torchaudio + +logger = logging.get_logger(__name__) + + +@requires(backends=("torchaudio",)) +class MusicgenMelodyFeatureExtractor(SequenceFeatureExtractor): + r""" + Constructs a MusicgenMelody feature extractor. + + This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains + most of the main methods. Users should refer to this superclass for more information regarding those methods. + + This class extracts chroma features from audio processed by [Demucs](https://github.com/adefossez/demucs/tree/main) or + directly from raw audio waveform. + + Args: + feature_size (`int`, *optional*, defaults to 12): + The feature dimension of the extracted features. + sampling_rate (`int`, *optional*, defaults to 32000): + The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). + hop_length (`int`, *optional*, defaults to 4096): + Length of the overlapping windows for the STFT used to obtain the Mel Frequency coefficients. + chunk_length (`int`, *optional*, defaults to 30): + The maximum number of chunks of `sampling_rate` samples used to trim and pad longer or shorter audio + sequences. + n_fft (`int`, *optional*, defaults to 16384): + Size of the Fourier transform. + num_chroma (`int`, *optional*, defaults to 12): + Number of chroma bins to use. + padding_value (`float`, *optional*, defaults to 0.0): + Padding value used to pad the audio. + return_attention_mask (`bool`, *optional*, defaults to `False`): + Whether to return the attention mask. Can be overwritten when calling the feature extractor. + + [What are attention masks?](../glossary#attention-mask) + + + + For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle + bugs. + + + stem_indices (`list[int]`, *optional*, defaults to `[3, 2]`): + Stem channels to extract if demucs outputs are passed. + """ + + model_input_names = ["input_features"] + + def __init__( + self, + feature_size=12, + sampling_rate=32000, + hop_length=4096, + chunk_length=30, + n_fft=16384, + num_chroma=12, + padding_value=0.0, + return_attention_mask=False, # pad inputs to max length with silence token (zero) and no attention mask + stem_indices=[3, 2], + **kwargs, + ): + super().__init__( + feature_size=feature_size, + sampling_rate=sampling_rate, + padding_value=padding_value, + return_attention_mask=return_attention_mask, + **kwargs, + ) + self.n_fft = n_fft + self.hop_length = hop_length + self.chunk_length = chunk_length + self.n_samples = chunk_length * sampling_rate + self.sampling_rate = sampling_rate + self.chroma_filters = torch.from_numpy( + chroma_filter_bank(sampling_rate=sampling_rate, num_frequency_bins=n_fft, tuning=0, num_chroma=num_chroma) + ).float() + self.spectrogram = torchaudio.transforms.Spectrogram( + n_fft=n_fft, win_length=n_fft, hop_length=hop_length, power=2, center=True, pad=0, normalized=True + ) + self.stem_indices = stem_indices + + def _torch_extract_fbank_features(self, waveform: torch.Tensor) -> torch.Tensor: + """ + Compute the chroma spectrogram of the provided audio using the torchaudio spectrogram implementation and the librosa chroma features. + """ + + # if wav length is not long enough, pad it + wav_length = waveform.shape[-1] + if wav_length < self.n_fft: + pad = self.n_fft - wav_length + rest = 0 if pad % 2 == 0 else 1 + waveform = torch.nn.functional.pad(waveform, (pad // 2, pad // 2 + rest), "constant", 0) + + # squeeze alongside channel dimension + spec = self.spectrogram(waveform).squeeze(1) + + # sum along the frequency dimension + raw_chroma = torch.einsum("cf, ...ft->...ct", self.chroma_filters, spec) + + # normalise with max value + norm_chroma = torch.nn.functional.normalize(raw_chroma, p=float("inf"), dim=-2, eps=1e-6) + + # transpose time and chroma dimension -> (batch, time, chroma) + norm_chroma = norm_chroma.transpose(1, 2) + + # replace max value alongside chroma dimension with 1 and replace the rest with 0 + idx = norm_chroma.argmax(-1, keepdim=True) + norm_chroma[:] = 0 + norm_chroma.scatter_(dim=-1, index=idx, value=1) + + return norm_chroma + + def _extract_stem_indices(self, audio, sampling_rate=None): + """ + Extracts stems from the output of the [Demucs](https://github.com/adefossez/demucs/tree/main) audio separation model, + then converts to mono-channel and resample to the feature extractor sampling rate. + + Args: + audio (`torch.Tensor` of shape `(batch_size, num_stems, channel_size, audio_length)`): + The output of the Demucs model to be processed. + sampling_rate (`int`, *optional*): + Demucs sampling rate. If not specified, defaults to `44000`. + """ + sampling_rate = 44000 if sampling_rate is None else sampling_rate + + # extract "vocals" and "others" sources from audio encoder (demucs) output + # [batch_size, num_stems, channel_size, audio_length] + wav = audio[:, torch.tensor(self.stem_indices)] + + # merge extracted stems to single waveform + wav = wav.sum(1) + + # convert to mono-channel waveform + wav = wav.mean(dim=1, keepdim=True) + + # resample to model sampling rate + # not equivalent to julius.resample + if sampling_rate != self.sampling_rate: + wav = torchaudio.functional.resample( + wav, sampling_rate, self.sampling_rate, rolloff=0.945, lowpass_filter_width=24 + ) + + # [batch_size, 1, audio_length] -> [batch_size, audio_length] + wav = wav.squeeze(1) + + return wav + + def __call__( + self, + audio: np.ndarray | list[float] | list[np.ndarray] | list[list[float]], + truncation: bool = True, + pad_to_multiple_of: int | None = None, + return_tensors: str | TensorType | None = None, + return_attention_mask: bool | None = None, + padding: str | None = True, + max_length: int | None = None, + sampling_rate: int | None = None, + **kwargs, + ) -> BatchFeature: + """ + Main method to featurize and prepare for the model one or several sequence(s). + + Args: + audio (`torch.Tensor`, `np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[torch.Tensor]`, `list[list[float]]`): + The sequence or batch of sequences to be padded. Each sequence can be a torch tensor, a numpy array, a list of float + values, a list of numpy arrays, a list of torch tensors, or a list of list of float values. + If `audio` is the output of Demucs, it has to be a torch tensor of shape `(batch_size, num_stems, channel_size, audio_length)`. + Otherwise, it must be mono or stereo channel audio. + truncation (`bool`, *optional*, default to `True`): + Activates truncation to cut input sequences longer than *max_length* to *max_length*. + pad_to_multiple_of (`int`, *optional*, defaults to None): + If set will pad the sequence to a multiple of the provided value. + + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability + `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors instead of list of python integers. Acceptable values are: + + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return Numpy `np.ndarray` objects. + return_attention_mask (`bool`, *optional*): + Whether to return the attention mask. If left to the default, will return the attention mask according + to the specific feature_extractor's default. + + [What are attention masks?](../glossary#attention-mask) + + + For Musicgen Melody models, audio `attention_mask` is not necessary. + + + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + sampling_rate (`int`, *optional*): + The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass + `sampling_rate` at the forward call to prevent silent errors. + Note that if `audio` is the output of Demucs, `sampling_rate` must be the sampling rate at which Demucs operates. + """ + + if sampling_rate is None: + logger.warning_once( + f"It is strongly recommended to pass the `sampling_rate` argument to `{self.__class__.__name__}()`. " + "Failing to do so can result in silent errors that might be hard to debug." + ) + + if isinstance(audio, torch.Tensor) and len(audio.shape) == 4: + logger.warning_once( + "`audio` is a 4-dimensional torch tensor and has thus been recognized as the output of `Demucs`. " + "If this is not the case, make sure to read Musicgen Melody docstrings and " + "to correct `audio` to get the right behaviour." + "Link to the docstrings: https://huggingface.co/docs/transformers/main/en/model_doc/musicgen_melody" + ) + audio = self._extract_stem_indices(audio, sampling_rate=sampling_rate) + elif sampling_rate is not None and sampling_rate != self.sampling_rate: + audio = torchaudio.functional.resample( + audio, sampling_rate, self.sampling_rate, rolloff=0.945, lowpass_filter_width=24 + ) + + is_batched = isinstance(audio, (np.ndarray, torch.Tensor)) and len(audio.shape) > 1 + is_batched = is_batched or ( + isinstance(audio, (list, tuple)) and (isinstance(audio[0], (torch.Tensor, np.ndarray, tuple, list))) + ) + + if is_batched and not isinstance(audio[0], torch.Tensor): + audio = [torch.tensor(speech, dtype=torch.float32).unsqueeze(-1) for speech in audio] + elif is_batched: + audio = [speech.unsqueeze(-1) for speech in audio] + elif not is_batched and not isinstance(audio, torch.Tensor): + audio = torch.tensor(audio, dtype=torch.float32).unsqueeze(-1) + + if isinstance(audio[0], torch.Tensor) and audio[0].dtype is torch.float64: + audio = [speech.to(torch.float32) for speech in audio] + + # always return batch + if not is_batched: + audio = [audio] + + if len(audio[0].shape) == 3: + logger.warning_once( + "`audio` has been detected as a batch of stereo signals. Will be convert to mono signals. " + "If this is an undesired behaviour, make sure to read Musicgen Melody docstrings and " + "to correct `audio` to get the right behaviour." + "Link to the docstrings: https://huggingface.co/docs/transformers/main/en/model_doc/musicgen_melody" + ) + # convert to mono-channel waveform + audio = [stereo.mean(dim=0) for stereo in audio] + + batched_speech = BatchFeature({"input_features": audio}) + + padded_inputs = self.pad( + batched_speech, + padding=padding, + max_length=max_length if max_length else self.n_samples, + truncation=truncation, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + return_tensors="pt", + ) + + input_features = self._torch_extract_fbank_features(padded_inputs["input_features"].squeeze(-1)) + + padded_inputs["input_features"] = input_features + + if return_attention_mask: + # rescale from raw audio length to spectrogram length + padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length] + + if return_tensors is not None: + padded_inputs = padded_inputs.convert_to_tensors(return_tensors) + + return padded_inputs + + def to_dict(self) -> dict[str, Any]: + """ + Serializes this instance to a Python dictionary. Returns: + `dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + output["feature_extractor_type"] = self.__class__.__name__ + if "mel_filters" in output: + del output["mel_filters"] + if "window" in output: + del output["window"] + if "chroma_filters" in output: + del output["chroma_filters"] + if "spectrogram" in output: + del output["spectrogram"] + return output + + +__all__ = ["MusicgenMelodyFeatureExtractor"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/modeling_musicgen_melody.py new file mode 100644 index 0000000000000000000000000000000000000000..a3e3f5c3251c39f97b0dc445cfbc4f024248cc95 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -0,0 +1,2093 @@ +# Copyright 2024 Meta AI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Musicgen Melody model.""" + +import inspect +import math +import random +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional + +import torch +import torch.nn as nn +from torch.nn import CrossEntropyLoss + +from ... import initialization as init +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache +from ...generation import ( + ClassifierFreeGuidanceLogitsProcessor, + GenerationConfig, + GenerationMixin, + GenerationMode, + LogitsProcessorList, + StoppingCriteriaList, +) +from ...masking_utils import create_causal_mask +from ...modeling_flash_attention_utils import ( + FlashAttentionKwargs, +) +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, ModelOutput +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, logging +from ...utils.generic import merge_with_config_defaults +from ...utils.output_capturing import OutputRecorder, capture_outputs +from ..auto.configuration_auto import AutoConfig +from ..auto.modeling_auto import AutoModel, AutoModelForTextEncoding +from .configuration_musicgen_melody import MusicgenMelodyConfig, MusicgenMelodyDecoderConfig + + +if TYPE_CHECKING: + from ...generation.streamers import BaseStreamer + +logger = logging.get_logger(__name__) + + +@auto_docstring( + custom_intro=""" + Base class for Musicgen Melody autoregressive outputs. + """ +) +@dataclass +class MusicgenMelodyOutputWithPast(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of conditional hidden-states representing the concatenation of the projected text encoder output and the projected audio encoder output. + Used as a conditional signal. + """ + + loss: torch.FloatTensor | None = None + logits: torch.FloatTensor | None = None + past_key_values: Cache | None = None + hidden_states: tuple[torch.FloatTensor] | None = None + attentions: tuple[torch.FloatTensor] | None = None + encoder_hidden_states: torch.FloatTensor | None = None + + +# Copied from transformers.models.musicgen.modeling_musicgen.shift_tokens_right +def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): + """ + Shift input ids one token to the right. + """ + # transpose to get (bsz, num_codebooks, seq_len) + input_ids = input_ids.transpose(1, 2) + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + if decoder_start_token_id is None: + raise ValueError("Make sure to set the decoder_start_token_id attribute of the model's configuration.") + shifted_input_ids[..., 0] = decoder_start_token_id + + if pad_token_id is None: + raise ValueError("Make sure to set the pad_token_id attribute of the model's configuration.") + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenSinusoidalPositionalEmbedding with Musicgen->MusicgenMelody +class MusicgenMelodySinusoidalPositionalEmbedding(nn.Module): + """This module produces sinusoidal positional embeddings of any length.""" + + def __init__(self, num_positions: int, embedding_dim: int): + super().__init__() + self.embedding_dim = embedding_dim + self.num_positions = num_positions + self.make_weights(num_positions, embedding_dim) + + def make_weights(self, num_embeddings: int, embedding_dim: int): + emb_weights = self.get_embedding(num_embeddings, embedding_dim) + if hasattr(self, "weights"): + # in forward put the weights on the correct dtype and device of the param + emb_weights = emb_weights.to(dtype=self.weights.dtype, device=self.weights.device) + + self.register_buffer("weights", emb_weights, persistent=False) + + @staticmethod + def get_embedding(num_embeddings: int, embedding_dim: int): + """ + Build sinusoidal embeddings. This matches the implementation in tensor2tensor, but differs slightly from the + description in Section 3.5 of "Attention Is All You Need". + """ + half_dim = embedding_dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.int64).float() * -emb) + emb = torch.arange(num_embeddings, dtype=torch.int64).float().unsqueeze(1) * emb.unsqueeze(0) + emb = torch.cat([torch.cos(emb), torch.sin(emb)], dim=1).view(num_embeddings, -1) + if embedding_dim % 2 == 1: + # zero pad + emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) + return emb.to(torch.get_default_dtype()) + + @torch.no_grad() + # Ignore copy + def forward(self, inputs_embeds: torch.Tensor, past_key_values_length: int = 0): + bsz, seq_len, _ = inputs_embeds.size() + # Create the position ids from the input token ids. + position_ids = (torch.arange(seq_len) + past_key_values_length).to(inputs_embeds.device) + # expand embeddings if needed + if seq_len > self.weights.size(0): + self.make_weights(seq_len, self.embedding_dim) + return self.weights.index_select(0, position_ids.view(-1)).detach() + + +# Copied from transformers.models.bert.modeling_bert.eager_attention_forward +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float | None = None, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + if scaling is None: + scaling = query.size(-1) ** -0.5 + + # Take the dot product between "query" and "key" to get the raw attention scores. + attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling + + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + + attn_output = torch.matmul(attn_weights, value) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenAttention with Musicgen->MusicgenMelody +class MusicgenMelodyAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float | None = 0.0, + is_decoder: bool | None = False, + bias: bool | None = True, + is_causal: bool | None = False, + config: MusicgenMelodyConfig | None = None, + layer_idx: int | None = None, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + self.config = config + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + self.is_causal = is_causal + self.layer_idx = layer_idx + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: torch.Tensor | None = None, + past_key_values: Cache | None = None, + attention_mask: torch.Tensor | None = None, + output_attentions: bool | None = False, + # TODO: we need a refactor so that the different attention modules can get their specific kwargs + # ATM, we have mixed things encoder, decoder, and encoder-decoder attn + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + # determine input shapes + input_shape = hidden_states.shape[:-1] + + hidden_shape = (*input_shape, -1, self.head_dim) + + # get query proj + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + is_updated = False + if past_key_values is not None: + if isinstance(past_key_values, EncoderDecoderCache): + is_updated = past_key_values.is_updated.get(self.layer_idx) + if is_cross_attention: + # after the first generated id, we can subsequently re-use all key/value_layer from cache + curr_past_key_values = past_key_values.cross_attention_cache + else: + curr_past_key_values = past_key_values.self_attention_cache + else: + curr_past_key_values = past_key_values + + current_states = key_value_states if is_cross_attention else hidden_states + if is_cross_attention and past_key_values is not None and is_updated: + # reuse k,v, cross_attentions + key_states = curr_past_key_values.layers[self.layer_idx].keys + value_states = curr_past_key_values.layers[self.layer_idx].values + else: + kv_shape = (*current_states.shape[:-1], -1, self.head_dim) + key_states = self.k_proj(current_states).view(kv_shape).transpose(1, 2) + value_states = self.v_proj(current_states).view(kv_shape).transpose(1, 2) + + if past_key_values is not None: + # save all key/value_states to cache to be re-used for fast auto-regressive generation + key_states, value_states = curr_past_key_values.update(key_states, value_states, self.layer_idx) + # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls + if is_cross_attention and isinstance(past_key_values, EncoderDecoderCache): + past_key_values.is_updated[self.layer_idx] = True + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.dropout, + scaling=self.scaling, + output_attentions=output_attentions, + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights + + +class MusicgenMelodyDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: MusicgenMelodyDecoderConfig, layer_idx=None): + super().__init__() + self.embed_dim = config.hidden_size + + self.self_attn = MusicgenMelodyAttention( + embed_dim=self.embed_dim, + num_heads=config.num_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + bias=False, + is_causal=True, + config=config, + layer_idx=layer_idx, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + + self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim, bias=False) + self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=False) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = True, + **kwargs, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + past_key_values (`Cache`): cached past key and value projection states + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + past_key_values=past_key_values, + attention_mask=attention_mask, + **kwargs, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + return hidden_states, self_attn_weights + + +@auto_docstring +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenPreTrainedModel with Musicgen->MusicgenMelody +class MusicgenMelodyPreTrainedModel(PreTrainedModel): + config: MusicgenMelodyDecoderConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["MusicgenMelodyDecoderLayer", "MusicgenMelodyAttention"] + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + @torch.no_grad() + def _init_weights(self, module): + std = self.config.initializer_factor + if isinstance(module, nn.Linear): + init.normal_(module.weight, mean=0.0, std=std) + if module.bias is not None: + init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + init.ones_(module.weight) + init.zeros_(module.bias) + elif isinstance(module, nn.Embedding): + init.normal_(module.weight, mean=0.0, std=std) + # Here we need the check explicitly, as we slice the weight in the `zeros_` call, so it looses the flag + if module.padding_idx is not None and not getattr(module.weight, "_is_hf_initialized", False): + init.zeros_(module.weight[module.padding_idx]) + elif isinstance(module, MusicgenMelodySinusoidalPositionalEmbedding): + emb_weights = module.get_embedding(module.num_positions, module.embedding_dim) + init.copy_(module.weights, emb_weights) + + +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenDecoder with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody +class MusicgenMelodyDecoder(MusicgenMelodyPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MusicgenMelodyDecoderLayer`] + """ + + _can_record_outputs = { + "hidden_states": MusicgenMelodyDecoderLayer, + "attentions": OutputRecorder(MusicgenMelodyAttention, index=1, layer_name="self_attn"), + "cross_attentions": OutputRecorder(MusicgenMelodyAttention, index=1, layer_name="encoder_attn"), + } + + def __init__(self, config: MusicgenMelodyDecoderConfig): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.layerdrop + self.max_target_positions = config.max_position_embeddings + self.d_model = config.hidden_size + self.num_codebooks = config.num_codebooks + self.embed_scale = math.sqrt(config.hidden_size) if config.scale_embedding else 1.0 + + embed_dim = config.vocab_size + 1 + self.embed_tokens = nn.ModuleList( + [nn.Embedding(embed_dim, config.hidden_size) for _ in range(config.num_codebooks)] + ) + + self.embed_positions = MusicgenMelodySinusoidalPositionalEmbedding( + config.max_position_embeddings, + config.hidden_size, + ) + + self.layers = nn.ModuleList( + [MusicgenMelodyDecoderLayer(config, layer_idx=i) for i in range(config.num_hidden_layers)] + ) + self.layer_norm = nn.LayerNorm(config.hidden_size) + self.attn_implementation = config._attn_implementation + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + # Ignore copy + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + encoder_hidden_states: torch.FloatTensor | None = None, + encoder_attention_mask: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPast: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """ + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + # (bsz * codebooks, seq_len) -> (bsz, codebooks, seq_len) + input = input_ids.reshape(-1, self.num_codebooks, input_ids.shape[-1]) + bsz, num_codebooks, seq_len = input.shape + elif inputs_embeds is not None: + input = inputs_embeds[:, :, -1:] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + if use_cache and past_key_values is None: + past_key_values = EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config)) + + past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 0 + if inputs_embeds is None: + inputs_embeds = sum(self.embed_tokens[codebook](input[:, codebook]) for codebook in range(num_codebooks)) + + if encoder_hidden_states is not None: + # take care of attention masks + if encoder_attention_mask is not None and attention_mask is None: + attention_mask = torch.ones(inputs_embeds.shape[:2], device=inputs_embeds.device) + + if attention_mask is not None: + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_states.shape[:2], device=attention_mask.device) + attention_mask = torch.cat([encoder_attention_mask, attention_mask], dim=1) + + # fuse encoder_hidden_states and inputs_embeds + inputs_embeds = torch.cat([encoder_hidden_states, inputs_embeds], dim=1) + + attention_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + ) + + # embed positions + positions = self.embed_positions(inputs_embeds, past_key_values_length) + hidden_states = inputs_embeds + positions.to(inputs_embeds.device) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + # decoder layers + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): + continue + + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + **kwargs, + ) + hidden_states = layer_outputs[0] + + hidden_states = self.layer_norm(hidden_states) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenModel with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody +class MusicgenMelodyModel(MusicgenMelodyPreTrainedModel): + def __init__(self, config: MusicgenMelodyDecoderConfig): + super().__init__(config) + self.decoder = MusicgenMelodyDecoder(config) + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.decoder.embed_tokens = value + + @auto_docstring + # Ignore copy + @capture_outputs + @merge_with_config_defaults + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + encoder_hidden_states: torch.FloatTensor | None = None, + encoder_attention_mask: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | BaseModelOutputWithPast: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + """ + decoder_outputs: BaseModelOutputWithPast = self.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + return BaseModelOutputWithPast( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + hidden_states=decoder_outputs.hidden_states, + attentions=decoder_outputs.attentions, + ) + + +@auto_docstring( + custom_intro=""" + The Musicgen Melody decoder model with a language modelling head on top. + """ +) +# Copied from transformers.models.musicgen.modeling_musicgen.MusicgenForCausalLM with MUSICGEN->MUSICGEN_MELODY,Musicgen->MusicgenMelody,MusicGen->Musicgen Melody +class MusicgenMelodyForCausalLM(MusicgenMelodyPreTrainedModel, GenerationMixin): + output_modalities = ("audio",) + + def __init__(self, config: MusicgenMelodyDecoderConfig): + super().__init__(config) + + self.model = MusicgenMelodyModel(config) + + self.num_codebooks = config.num_codebooks + self.lm_heads = nn.ModuleList( + [nn.Linear(config.hidden_size, config.vocab_size, bias=False) for _ in range(config.num_codebooks)] + ) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.model.decoder.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_heads + + def set_output_embeddings(self, new_embeddings): + self.lm_heads = new_embeddings + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + # Ignore copy + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + encoder_hidden_states: torch.FloatTensor | None = None, + encoder_attention_mask: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + use_cache: bool | None = None, + labels: torch.LongTensor | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | MusicgenMelodyOutputWithPast: + r""" + input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, sequence_length)`): + Indices of input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are input IDs?](../glossary#input-ids) + + + + The `input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `input_ids`. + + + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states representing the concatenation of the text encoder output and the processed audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing attention on conditional hidden states. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + """ + + if (labels is not None) and (input_ids is None and inputs_embeds is None): + input_ids = shift_tokens_right(labels, self.config.pad_token_id, self.config.bos_token_id) + + outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + + lm_logits = torch.stack([head(hidden_states) for head in self.lm_heads], dim=1) + + loss = None + if labels is not None: + # since encoder hidden states have been concatenated to the decoder hidden states, + # we take the last timestamps corresponding to labels + logits = lm_logits[:, :, -labels.shape[1] :] + + loss_fct = CrossEntropyLoss() + loss = torch.zeros([], device=self.device) + + # per codebook cross-entropy + # ref: https://github.com/facebookresearch/audiocraft/blob/69fea8b290ad1b4b40d28f92d1dfc0ab01dbab85/audiocraft/solvers/musicgen.py#L242-L243 + # -100 labels are ignored + labels = labels.masked_fill(labels == self.config.pad_token_id, -100) + + # per codebook cross-entropy + for codebook in range(self.config.num_codebooks): + codebook_logits = logits[:, codebook].contiguous().view(-1, logits.shape[-1]) + codebook_labels = labels[..., codebook].contiguous().view(-1) + loss += loss_fct(codebook_logits, codebook_labels) + + loss = loss / self.config.num_codebooks + + # (bsz, num_codebooks, seq_len, vocab_size) -> (bsz * num_codebooks, seq_len, vocab_size) + lm_logits = lm_logits.reshape(-1, *lm_logits.shape[2:]) + + return MusicgenMelodyOutputWithPast( + loss=loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + # Ignore copy + def prepare_inputs_for_generation( + self, + input_ids, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=True, + delay_pattern_mask=None, + guidance_scale=None, + **kwargs, + ): + # Overwritten -- MusicGen has custom processing + if delay_pattern_mask is None: + input_ids, delay_pattern_mask = self.build_delay_pattern_mask( + input_ids, + pad_token_id=self.generation_config.pad_token_id, + max_length=self.generation_config.max_length, + ) + + # apply the delay pattern mask + input_ids = self.apply_delay_pattern_mask(input_ids, delay_pattern_mask) + + if guidance_scale is not None and guidance_scale > 1: + # for classifier free guidance we need to replicate the decoder args across the batch dim (we'll split these + # before sampling) + input_ids = input_ids.repeat((2, 1)) + if attention_mask is not None: + attention_mask = attention_mask.repeat((2, 1)) + + if encoder_hidden_states is not None: + encoder_hidden_states = torch.concatenate( + [encoder_hidden_states, torch.zeros_like(encoder_hidden_states)], dim=0 + ) + + if encoder_attention_mask is not None: + encoder_attention_mask = torch.concatenate( + encoder_attention_mask, torch.zeros_like(encoder_attention_mask), dim=0 + ) + + if past_key_values is not None: + input_ids = input_ids[:, -1:] + + # we only want to use conditional signal in the 1st generation step but keeping the attention mask + encoder_hidden_states = None + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "encoder_hidden_states": encoder_hidden_states, + "encoder_attention_mask": encoder_attention_mask, + "past_key_values": past_key_values, + "use_cache": use_cache, + } + + def build_delay_pattern_mask(self, input_ids: torch.LongTensor, pad_token_id: int, max_length: int | None = None): + """Build a delayed pattern mask to the input_ids. Each codebook is offset by the previous codebook by + one, giving a delayed pattern mask at the start of sequence and end of sequence. Take the example where there + are 4 codebooks and a max sequence length of 8, we have the delayed pattern mask of shape `(codebooks, + seq_len)`: + - [P, -1, -1, -1, -1, P, P, P] + - [P, P, -1, -1, -1, -1, P, P] + - [P, P, P, -1, -1, -1, -1, P] + - [P, P, P, P, -1, -1, -1, -1] + where P is the special padding token id and -1 indicates that the token is valid for prediction. If we include + a prompt (decoder input ids), the -1 positions indicate where new tokens should be predicted. Otherwise, the + mask is set to the value in the prompt: + - [P, a, b, -1, -1, P, P, P] + - [P, P, c, d, -1, -1, P, P] + - [P, P, P, e, f, -1, -1, P] + - [P, P, P, P, g, h, -1, -1] + where a-h indicate the input prompt (decoder input ids) that are offset by 1. Now, we only override the -1 + tokens in our prediction. + """ + # (bsz * num_codebooks, seq_len) -> (bsz, num_codebooks, seq_len) + input_ids = input_ids.reshape(-1, self.num_codebooks, input_ids.shape[-1]) + bsz, num_codebooks, seq_len = input_ids.shape + + max_length = max_length if max_length is not None else self.generation_config.max_length + input_ids_shifted = ( + torch.ones((bsz, num_codebooks, max_length), dtype=torch.long, device=input_ids.device) * -1 + ) + + channel_codebooks = num_codebooks // 2 if self.config.audio_channels == 2 else num_codebooks + # we only apply the mask if we have a large enough seq len - otherwise we return as is + if max_length < 2 * channel_codebooks - 1: + return input_ids.reshape(bsz * num_codebooks, -1), input_ids_shifted.reshape(bsz * num_codebooks, -1) + + # fill the shifted ids with the prompt entries, offset by the codebook idx + for codebook in range(channel_codebooks): + if self.config.audio_channels == 1: + # mono channel - loop over the codebooks one-by-one + input_ids_shifted[:, codebook, codebook : seq_len + codebook] = input_ids[:, codebook] + else: + # left/right channels are interleaved in the generated codebooks, so handle one then the other + input_ids_shifted[:, 2 * codebook, codebook : seq_len + codebook] = input_ids[:, 2 * codebook] + input_ids_shifted[:, 2 * codebook + 1, codebook : seq_len + codebook] = input_ids[:, 2 * codebook + 1] + + # construct a pattern mask that indicates the positions of padding tokens for each codebook + # first fill the upper triangular part (the EOS padding) + delay_pattern = torch.triu( + torch.ones((channel_codebooks, max_length), dtype=torch.bool), diagonal=max_length - channel_codebooks + 1 + ) + # then fill the lower triangular part (the BOS padding) + delay_pattern = delay_pattern + torch.tril(torch.ones((channel_codebooks, max_length), dtype=torch.bool)) + + if self.config.audio_channels == 2: + # for left/right channel we need to duplicate every row of the pattern mask in an interleaved fashion + delay_pattern = delay_pattern.repeat_interleave(2, dim=0) + + mask = ~delay_pattern.to(input_ids.device) + input_ids = mask * input_ids_shifted + ~mask * pad_token_id + + # find the first position to start generating - this is the first place we have the -1 token + # and will always be in the first codebook (since it has no codebook offset) + first_codebook_ids = input_ids[:, 0, :] + start_ids = (first_codebook_ids == -1).nonzero()[:, 1] + if len(start_ids) > 0: + first_start_id = min(start_ids) + else: + # we have no tokens that need to be filled - return entire matrix of input ids + first_start_id = seq_len + + # (bsz * num_codebooks, seq_len) -> (bsz, num_codebooks, seq_len) + pattern_mask = input_ids.reshape(bsz * num_codebooks, -1) + input_ids = input_ids[..., :first_start_id].reshape(bsz * num_codebooks, -1) + return input_ids, pattern_mask + + @staticmethod + def apply_delay_pattern_mask(input_ids, decoder_pad_token_mask): + """Apply a delay pattern mask to the decoder input ids, only preserving predictions where + the mask is set to -1, and otherwise setting to the value detailed in the mask.""" + seq_len = input_ids.shape[-1] + decoder_pad_token_mask = decoder_pad_token_mask[..., :seq_len] + input_ids = torch.where(decoder_pad_token_mask == -1, input_ids, decoder_pad_token_mask) + return input_ids + + @torch.no_grad() + # Ignore copy + def generate( + self, + inputs: torch.Tensor | None = None, + generation_config: GenerationConfig | None = None, + logits_processor: LogitsProcessorList | None = None, + stopping_criteria: StoppingCriteriaList | None = None, + synced_gpus: bool | None = None, + streamer: Optional["BaseStreamer"] = None, + **kwargs, + ): + """ + + Generates sequences of token ids for models with a language modeling head. + + + + Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the + model's default generation configuration. You can override any `generation_config` by passing the corresponding + parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. + + For an overview of generation strategies and code examples, check out the [following + guide](./generation_strategies). + + + + Parameters: + inputs (`torch.Tensor` of varying shape depending on the modality, *optional*): + The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the + method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs` + should be in the format `input_ids`. For encoder-decoder models *inputs* can represent any of + `input_ids`, `input_values`, `input_features`, or `pixel_values`. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, the default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + logits_processor (`LogitsProcessorList`, *optional*): + Custom logits processors that complement the default logits processors built from arguments and + generation config. If a logit processor is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + stopping_criteria (`StoppingCriteriaList`, *optional*): + Custom stopping criteria that complement the default stopping criteria built from arguments and a + generation config. If a stopping criteria is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + synced_gpus (`bool`, *optional*, defaults to `False`): + Whether to continue running the while loop until max_length (needed to avoid deadlocking with + `FullyShardedDataParallel` and DeepSpeed ZeRO Stage 3). + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. + kwargs (`dict[str, Any]`, *optional*): + Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be + forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder + specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*. + + Return: + [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` + or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`. + + If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible + [`~utils.ModelOutput`] types are: + + - [`~generation.GenerateDecoderOnlyOutput`], + - [`~generation.GenerateBeamDecoderOnlyOutput`] + + If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible + [`~utils.ModelOutput`] types are: + + - [`~generation.GenerateEncoderDecoderOutput`], + - [`~generation.GenerateBeamEncoderDecoderOutput`] + """ + # 1. Handle `generation_config` and kwargs that might update it, and validate the resulting objects + generation_config, model_kwargs = self._prepare_generation_config(generation_config, **kwargs) + self._validate_model_kwargs(model_kwargs.copy()) + + # 2. Set generation parameters if not already defined + logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList() + stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList() + + requires_attention_mask = "encoder_outputs" not in model_kwargs + kwargs_has_attention_mask = model_kwargs.get("attention_mask", None) is not None + + # 3. Define model inputs` + input_ids, model_input_name, model_kwargs = self._prepare_model_inputs( + inputs, generation_config.bos_token_id, model_kwargs + ) + batch_size = input_ids.shape[0] // self.num_codebooks + self._prepare_special_tokens(generation_config, kwargs_has_attention_mask, device=input_ids.device) + + # 4. Define other model kwargs + model_kwargs["use_cache"] = generation_config.use_cache + model_kwargs["guidance_scale"] = generation_config.guidance_scale + + if model_kwargs.get("attention_mask", None) is None and requires_attention_mask: + model_kwargs["attention_mask"] = self._prepare_attention_mask_for_generation( + input_ids, generation_config, model_kwargs + ) + + # 5. Prepare `max_length` depending on other stopping criteria. + input_ids_length = input_ids.shape[-1] + has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None + has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None + generation_config = self._prepare_generated_length( + generation_config=generation_config, + has_default_max_length=has_default_max_length, + has_default_min_length=has_default_min_length, + model_input_name=model_input_name, + inputs_tensor=input_ids, + input_ids_length=input_ids_length, + ) + + # 6. Prepare `input_ids` which will be used for auto-regressive generation + # Build the delay pattern mask for offsetting each codebook prediction by 1 (this behaviour is specific to Musicgen) + input_ids, delay_pattern_mask = self.build_delay_pattern_mask( + input_ids, + pad_token_id=generation_config._decoder_start_token_tensor, + max_length=generation_config.max_length, + ) + + if streamer is not None: + streamer.put(input_ids.cpu()) + + # stash the delay mask so that we don't have to recompute it in each forward pass + model_kwargs["delay_pattern_mask"] = delay_pattern_mask + + # 7. determine generation mode + generation_mode = generation_config.get_generation_mode() + + # 8. prepare batched CFG externally (to enable coexistence with the unbatched CFG) + if generation_config.guidance_scale is not None and generation_config.guidance_scale > 1: + logits_processor.append(ClassifierFreeGuidanceLogitsProcessor(generation_config.guidance_scale)) + generation_config.guidance_scale = None + + # 9. prepare distribution pre_processing samplers + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_length, + encoder_input_ids=input_ids, + prefix_allowed_tokens_fn=None, + logits_processor=logits_processor, + device=input_ids.device, + ) + + # 10. prepare stopping criteria + stopping_criteria = self._get_stopping_criteria( + generation_config=generation_config, stopping_criteria=stopping_criteria + ) + + if generation_mode in (GenerationMode.SAMPLE, GenerationMode.GREEDY_SEARCH): + # expand input_ids with `num_return_sequences` additional sequences per batch + input_ids, model_kwargs = self._expand_inputs_for_generation( + input_ids=input_ids, + expand_size=generation_config.num_return_sequences, + **model_kwargs, + ) + + # 11. run sample + outputs = self._sample( + input_ids, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + generation_config=generation_config, + synced_gpus=synced_gpus, + streamer=streamer, + **model_kwargs, + ) + + else: + raise ValueError( + "Got incompatible mode for generation, should be one of greedy or sampling. " + "Ensure that beam search is de-activated by setting `num_beams=1`." + ) + + if generation_config.return_dict_in_generate: + output_ids = outputs.sequences + else: + output_ids = outputs + + # apply the pattern mask to the final ids + output_ids = self.apply_delay_pattern_mask(output_ids, model_kwargs["delay_pattern_mask"]) + + # revert the pattern delay mask by filtering the pad token id + output_ids = output_ids[output_ids != generation_config._pad_token_tensor].reshape( + batch_size, self.num_codebooks, -1 + ) + + if generation_config.return_dict_in_generate: + outputs.sequences = output_ids + return outputs + else: + return output_ids + + +@auto_docstring +class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): + config: MusicgenMelodyConfig + main_input_name = "input_ids" + output_modalities = ("audio",) + supports_gradient_checkpointing = True + _supports_flash_attn = True + _supports_sdpa = True + _supports_flex_attn = True + + def __init__( + self, + config: MusicgenMelodyConfig = None, + text_encoder: PreTrainedModel | None = None, + audio_encoder: PreTrainedModel | None = None, + decoder: MusicgenMelodyForCausalLM | None = None, + ): + r""" + text_encoder (`PreTrainedModel`, *optional*): + The text encoder model that encodes text into hidden states for conditioning. + audio_encoder (`PreTrainedModel`, *optional*): + The audio encoder model that encodes audio into hidden states for conditioning. + decoder (`MusicgenMelodyForCausalLM`, *optional*): + The decoder model that generates audio tokens based on conditioning signals. + """ + if config is None and None in (text_encoder, audio_encoder, decoder): + raise ValueError( + "Either a configuration has to be provided, or all three of text encoder, audio encoder and Musicgen Melody decoder." + ) + if config is None: + config = MusicgenMelodyConfig( + text_encoder=text_encoder.config, audio_encoder=audio_encoder.config, decoder=decoder.config + ) + else: + if not isinstance(config, self.config_class): + raise ValueError(f"Config: {config} has to be of type {self.config_class}") + + # initialize with config + super().__init__(config) + + if text_encoder is None: + text_encoder = AutoModelForTextEncoding.from_config(config.text_encoder) + + if audio_encoder is None: + audio_encoder = AutoModel.from_config(config.audio_encoder) + + if decoder is None: + decoder = MusicgenMelodyForCausalLM._from_config(config.decoder) + + self.text_encoder = text_encoder + self.audio_encoder = audio_encoder + self.decoder = decoder + + # make sure that the individual model's config refers to the shared config + # so that the updates to the config will be synced + self.config.text_encoder._attn_implementation = self.text_encoder.config._attn_implementation + self.config.audio_encoder._attn_implementation = self.audio_encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation + self.text_encoder.config = self.config.text_encoder + self.audio_encoder.config = self.config.audio_encoder + self.decoder.config = self.config.decoder + + # text encoder outputs might need to be projected to different dimension for decoder + if self.text_encoder.config.hidden_size != self.decoder.config.hidden_size: + self.enc_to_dec_proj = nn.Linear(self.text_encoder.config.hidden_size, self.decoder.config.hidden_size) + + # audio encoder outputs after chroma extraction might need to be projected to different dimension for decoder + if self.config.num_chroma != self.decoder.config.hidden_size: + self.audio_enc_to_dec_proj = nn.Linear(self.config.num_chroma, self.decoder.config.hidden_size) + + if self.text_encoder.get_output_embeddings() is not None: + raise ValueError( + f"The encoder {self.text_encoder} should not have a LM Head. Please use a model without and LM Head" + ) + + # Initialize projection layers weights and tie text encoder and decoder weights if set accordingly + self.post_init() + + @torch.no_grad() + def _init_weights(self, module): + # MusicgenMelodyForConditionalGeneration is made of PreTrainedModels that have already been initialized + # Projection layers still need to be initialized. + std = self.decoder.config.initializer_factor + if isinstance(module, nn.Linear): + init.normal_(module.weight, mean=0.0, std=std) + if module.bias is not None: + init.zeros_(module.bias) + + def get_input_embeddings(self): + return self.text_encoder.get_input_embeddings() + + def get_output_embeddings(self): + return self.decoder.get_output_embeddings() + + def set_output_embeddings(self, new_embeddings): + return self.decoder.set_output_embeddings(new_embeddings) + + @classmethod + # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenForConditionalGeneration.from_sub_models_pretrained with Musicgen->MusicgenMelody, musicgen-small->musicgen-melody + def from_sub_models_pretrained( + cls, + text_encoder_pretrained_model_name_or_path: str | None = None, + audio_encoder_pretrained_model_name_or_path: str | None = None, + decoder_pretrained_model_name_or_path: str | None = None, + *model_args, + **kwargs, + ) -> PreTrainedModel: + r""" + Instantiate a text encoder, an audio encoder, and a MusicGen decoder from one, two or three base classes of the + library from pretrained model checkpoints. + + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you need to first set it back in training mode with `model.train()`. + + Params: + text_encoder_pretrained_model_name_or_path (`str`, *optional*): + Information necessary to initiate the text encoder. Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + + audio_encoder_pretrained_model_name_or_path (`str`, *optional*): + Information necessary to initiate the audio encoder. Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + + decoder_pretrained_model_name_or_path (`str`, *optional*, defaults to `None`): + Information necessary to initiate the decoder. Can be either: + + - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + - A path to a *directory* containing model weights saved using + [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. + + model_args (remaining positional arguments, *optional*): + All remaining positional arguments will be passed to the underlying model's `__init__` method. + + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., + `output_attentions=True`). + + - To update the text encoder configuration, use the prefix *text_encoder_* for each configuration + parameter. + - To update the audio encoder configuration, use the prefix *audio_encoder_* for each configuration + parameter. + - To update the decoder configuration, use the prefix *decoder_* for each configuration parameter. + - To update the parent model configuration, do not use a prefix for each configuration parameter. + + Behaves differently depending on whether a `config` is provided or automatically loaded. + + Example: + + ```python + >>> from transformers import MusicgenMelodyForConditionalGeneration + + >>> # initialize a musicgen model from a t5 text encoder, encodec audio encoder, and musicgen decoder + >>> model = MusicgenMelodyForConditionalGeneration.from_sub_models_pretrained( + ... text_encoder_pretrained_model_name_or_path="google-t5/t5-base", + ... audio_encoder_pretrained_model_name_or_path="facebook/encodec_24khz", + ... decoder_pretrained_model_name_or_path="facebook/musicgen-melody", + ... ) + >>> # saving model after fine-tuning + >>> model.save_pretrained("./musicgen-ft") + >>> # load fine-tuned model + >>> model = MusicgenMelodyForConditionalGeneration.from_pretrained("./musicgen-ft") + ```""" + + kwargs_text_encoder = { + argument[len("text_encoder_") :]: value + for argument, value in kwargs.items() + if argument.startswith("text_encoder_") + } + + kwargs_audio_encoder = { + argument[len("audio_encoder_") :]: value + for argument, value in kwargs.items() + if argument.startswith("audio_encoder_") + } + + kwargs_decoder = { + argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_") + } + + # remove text encoder, audio encoder and decoder kwargs from kwargs + for key in kwargs_text_encoder: + del kwargs["text_encoder_" + key] + for key in kwargs_audio_encoder: + del kwargs["audio_encoder_" + key] + for key in kwargs_decoder: + del kwargs["decoder_" + key] + + # Load and initialize the encoder and decoder + # The distinction between encoder and decoder at the model level is made + # by the value of the flag `is_decoder` that we need to set correctly. + text_encoder = kwargs_text_encoder.pop("model", None) + if text_encoder is None: + if text_encoder_pretrained_model_name_or_path is None: + raise ValueError( + "If `text_encoder_model` is not defined as an argument, a `text_encoder_pretrained_model_name_or_path` has " + "to be defined." + ) + + if "config" not in kwargs_text_encoder: + encoder_config, kwargs_text_encoder = AutoConfig.from_pretrained( + text_encoder_pretrained_model_name_or_path, **kwargs_text_encoder, return_unused_kwargs=True + ) + + if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True: + logger.info( + f"Initializing {text_encoder_pretrained_model_name_or_path} as a text_encoder model " + "from a decoder model. Cross-attention and causal mask are disabled." + ) + encoder_config.is_decoder = False + encoder_config.add_cross_attention = False + + kwargs_text_encoder["config"] = encoder_config + + text_encoder = AutoModel.from_pretrained( + text_encoder_pretrained_model_name_or_path, *model_args, **kwargs_text_encoder + ) + + audio_encoder = kwargs_audio_encoder.pop("model", None) + if audio_encoder is None: + if audio_encoder_pretrained_model_name_or_path is None: + raise ValueError( + "If `audio_encoder_model` is not defined as an argument, an `audio_encoder_pretrained_model_name_or_path` has " + "to be defined." + ) + + if "config" not in kwargs_audio_encoder: + encoder_config, kwargs_audio_encoder = AutoConfig.from_pretrained( + audio_encoder_pretrained_model_name_or_path, **kwargs_audio_encoder, return_unused_kwargs=True + ) + + if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True: + logger.info( + f"Initializing {audio_encoder_pretrained_model_name_or_path} as an audio_encoder model " + "from a decoder model. Cross-attention and causal mask are disabled." + ) + encoder_config.is_decoder = False + encoder_config.add_cross_attention = False + + kwargs_audio_encoder["config"] = encoder_config + + audio_encoder = AutoModel.from_pretrained( + audio_encoder_pretrained_model_name_or_path, *model_args, **kwargs_audio_encoder + ) + + decoder = kwargs_decoder.pop("model", None) + if decoder is None: + if decoder_pretrained_model_name_or_path is None: + raise ValueError( + "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has " + "to be defined." + ) + + if "config" not in kwargs_decoder: + decoder_config, kwargs_decoder = AutoConfig.from_pretrained( + decoder_pretrained_model_name_or_path, **kwargs_decoder, return_unused_kwargs=True + ) + + if isinstance(decoder_config, MusicgenMelodyConfig): + decoder_config = decoder_config.decoder + + if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False: + logger.info( + f"Initializing {decoder_pretrained_model_name_or_path} as a decoder model. Cross attention" + f" layers are added to {decoder_pretrained_model_name_or_path} and randomly initialized if" + f" {decoder_pretrained_model_name_or_path}'s architecture allows for cross attention layers." + ) + decoder_config.is_decoder = True + decoder_config.add_cross_attention = True + + kwargs_decoder["config"] = decoder_config + + if kwargs_decoder["config"].is_decoder is False or kwargs_decoder["config"].add_cross_attention is False: + logger.warning( + f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. " + f"In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, " + "make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` " + "passed to `.from_sub_models_pretrained(...)` are set to `True` or do not pass a " + "`decoder_config` to `.from_sub_models_pretrained(...)`" + ) + + decoder = MusicgenMelodyForCausalLM.from_pretrained( + decoder_pretrained_model_name_or_path, **kwargs_decoder + ) + + # instantiate config with corresponding kwargs + config = MusicgenMelodyConfig( + text_encoder=text_encoder.config, audio_encoder=audio_encoder.config, decoder=decoder.config, **kwargs + ) + return cls(text_encoder=text_encoder, audio_encoder=audio_encoder, decoder=decoder, config=config) + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.BoolTensor | None = None, + input_features: torch.FloatTensor | None = None, + decoder_input_ids: torch.LongTensor | None = None, + decoder_attention_mask: torch.BoolTensor | None = None, + past_key_values: Cache | None = None, + encoder_hidden_states: torch.FloatTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + decoder_inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> tuple | MusicgenMelodyOutputWithPast: + r""" + decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary, corresponding to the sequence of audio codes. + + Indices can be obtained by encoding an audio prompt with an audio encoder model to predict audio codes, + such as with the [`EncodecModel`]. See [`EncodecModel.encode`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + + + The `decoder_input_ids` will automatically be converted from shape `(batch_size * num_codebooks, + target_sequence_length)` to `(batch_size, num_codebooks, target_sequence_length)` in the forward pass. If + you obtain audio codes from an audio encoding model, such as [`EncodecModel`], ensure that the number of + frames is equal to 1, and that you reshape the audio codes from `(frames, batch_size, num_codebooks, + target_sequence_length)` to `(batch_size * num_codebooks, target_sequence_length)` prior to passing them as + `decoder_input_ids`. + + + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of conditional hidden-states representing the concatenation of the projected text encoder output and the projected audio encoder output. + Used as a conditional signal and will thus be concatenated to the projected `decoder_input_ids`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + + Examples: + ```python + >>> from transformers import AutoProcessor, MusicgenMelodyForConditionalGeneration + >>> import torch + + >>> processor = AutoProcessor.from_pretrained("facebook/musicgen-melody") + >>> model = MusicgenMelodyForConditionalGeneration.from_pretrained("facebook/musicgen-melody") + + >>> inputs = processor( + ... text=["80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums"], + ... padding=True, + ... return_tensors="pt", + ... ) + + >>> pad_token_id = model.generation_config.pad_token_id + >>> decoder_input_ids = ( + ... torch.ones((inputs.input_ids.shape[0] * model.decoder.num_codebooks, 1), dtype=torch.long) + ... * pad_token_id + ... ) + + >>> logits = model(**inputs, decoder_input_ids=decoder_input_ids).logits + >>> logits.shape # (bsz * num_codebooks, encoder_len + tgt_len, vocab_size) + torch.Size([8, 249, 2048]) + ```""" + kwargs_text_encoder = { + argument[len("text_encoder_")]: value + for argument, value in kwargs.items() + if argument.startswith("text_encoder_") + } + + kwargs_decoder = { + argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_") + } + + for passthrough_key in ("output_attentions", "output_hidden_states"): + if passthrough_key in kwargs: + kwargs_text_encoder[passthrough_key] = kwargs[passthrough_key] + kwargs_decoder[passthrough_key] = kwargs[passthrough_key] + + if encoder_hidden_states is None: + if inputs_embeds is not None or input_ids is not None: + encoder_outputs = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + **kwargs_text_encoder, + ) + + encoder_hidden_states = encoder_outputs[0] + + # optionally project encoder_hidden_states + if self.text_encoder.config.hidden_size != self.decoder.config.hidden_size: + encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states) + + if attention_mask is not None and encoder_hidden_states is not None: + encoder_hidden_states = encoder_hidden_states * attention_mask[..., None] + + # set a default audio conditional hidden states if text is not None + if encoder_hidden_states is not None and input_features is None: + input_features = torch.zeros( + (encoder_hidden_states.shape[0], 1, self.config.num_chroma), + device=self.device, + dtype=self.dtype, + ) + input_features[:, :, 0] = 1 + + if input_features is not None: + audio_hidden_states = input_features + + # optionally project audio_hidden_states -> + # (batch_size, seq_len, num_chroma) -> (batch_size, seq_len, hidden_size) + if self.config.num_chroma != self.decoder.config.hidden_size: + audio_hidden_states = self.audio_enc_to_dec_proj(audio_hidden_states) + + # pad or truncate to config.chroma_length + if audio_hidden_states.shape[1] < self.config.chroma_length: + n_repeat = int(math.ceil(self.config.chroma_length / audio_hidden_states.shape[1])) + audio_hidden_states = audio_hidden_states.repeat(1, n_repeat, 1) + else: + logger.warning( + f"The conditional audio signal is of length {audio_hidden_states.shape[1]}, which exceeds" + f"the maximum chroma duration of {self.config.chroma_length}." + f"The audio will be truncated to {self.config.chroma_length} frames." + ) + audio_hidden_states = audio_hidden_states[:, : self.config.chroma_length] + + if encoder_hidden_states is not None: + encoder_hidden_states = torch.cat([audio_hidden_states, encoder_hidden_states], dim=1) + else: + encoder_hidden_states = audio_hidden_states + + if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None): + decoder_input_ids = shift_tokens_right( + labels, self.config.decoder.pad_token_id, self.config.decoder.bos_token_id + ) + + # Decode + decoder_outputs: MusicgenMelodyOutputWithPast = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_hidden_states, + inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + past_key_values=past_key_values, + labels=labels, + **kwargs_decoder, + ) + + return MusicgenMelodyOutputWithPast( + loss=decoder_outputs.loss, + logits=decoder_outputs.logits, + past_key_values=decoder_outputs.past_key_values, + hidden_states=decoder_outputs.hidden_states, + attentions=decoder_outputs.attentions, + encoder_hidden_states=encoder_hidden_states, + ) + + def prepare_inputs_for_generation( + self, + decoder_input_ids, + encoder_hidden_states=None, + past_key_values=None, + attention_mask=None, + decoder_attention_mask=None, + use_cache=None, + decoder_delay_pattern_mask=None, + guidance_scale=None, + **kwargs, + ): + # Overwritten -- MusicGen has custom processing + if decoder_delay_pattern_mask is None: + decoder_input_ids, decoder_delay_pattern_mask = self.decoder.build_delay_pattern_mask( + decoder_input_ids, + self.generation_config.pad_token_id, + max_length=self.generation_config.max_length, + ) + + # apply the delay pattern mask + decoder_input_ids = self.decoder.apply_delay_pattern_mask(decoder_input_ids, decoder_delay_pattern_mask) + + if guidance_scale is not None and guidance_scale > 1: + # for classifier free guidance we need to replicate the decoder args across the batch dim (we'll split these + # before sampling) + decoder_input_ids = decoder_input_ids.repeat((2, 1)) + if decoder_attention_mask is not None: + decoder_attention_mask = decoder_attention_mask.repeat((2, 1)) + + if past_key_values is not None: + past_length = past_key_values.get_seq_length() + + # Some generation methods already pass only the last input ID + if decoder_input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = decoder_input_ids.shape[1] - 1 + + decoder_input_ids = decoder_input_ids[:, remove_prefix_length:] + + # we only want to use conditional signal in the 1st generation step but keeping the attention mask + encoder_hidden_states = None + # we also have to update the attention mask + + return { + "input_ids": None, # encoder_hidden_states is defined. input_ids not needed + "encoder_hidden_states": encoder_hidden_states, + "past_key_values": past_key_values, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "decoder_attention_mask": decoder_attention_mask, + "use_cache": use_cache, + } + + # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenForConditionalGeneration._prepare_decoder_input_ids_for_generation + def _prepare_decoder_input_ids_for_generation( + self, + batch_size: int, + model_input_name: str, + model_kwargs: dict[str, torch.Tensor], + decoder_start_token_id: int | None = None, + bos_token_id: int | None = None, + device: torch.device | None = None, + ) -> tuple[torch.LongTensor, dict[str, torch.Tensor]]: + """Prepares `decoder_input_ids` for generation with encoder-decoder models""" + + # 1. Check whether the user has defined `decoder_input_ids` manually. To facilitate in terms of input naming, + # we also allow the user to pass it under `input_ids`, if the encoder does not use it as the main input. + if model_kwargs is not None and "decoder_input_ids" in model_kwargs: + decoder_input_ids = model_kwargs.pop("decoder_input_ids") + elif "input_ids" in model_kwargs and model_input_name != "input_ids": + decoder_input_ids = model_kwargs.pop("input_ids") + else: + decoder_input_ids = None + + # 2. Encoder-decoder models expect the `decoder_input_ids` to start with a special token. Let's ensure that. + decoder_start_token_id = self._get_decoder_start_token_id(decoder_start_token_id, bos_token_id) + if device is None: + device = self.device + decoder_input_ids_start = ( + torch.ones((batch_size * self.decoder.num_codebooks, 1), dtype=torch.long, device=device) + * decoder_start_token_id + ) + + # no user input -> use decoder_start_token_id as decoder_input_ids + if decoder_input_ids is None: + decoder_input_ids = decoder_input_ids_start + + # user input but doesn't start with decoder_start_token_id -> prepend decoder_start_token_id (and adjust + # decoder_attention_mask if provided) + elif (decoder_input_ids[..., 0] != decoder_start_token_id).all().item(): + decoder_input_ids = torch.cat([decoder_input_ids_start, decoder_input_ids], dim=-1) + if "decoder_attention_mask" in model_kwargs: + decoder_attention_mask = model_kwargs["decoder_attention_mask"] + decoder_attention_mask = torch.cat( + (torch.ones_like(decoder_attention_mask)[:, :1], decoder_attention_mask), + dim=-1, + ) + model_kwargs["decoder_attention_mask"] = decoder_attention_mask + + return decoder_input_ids, model_kwargs + + def _prepare_encoder_hidden_states_kwargs_for_generation( + self, + inputs_tensor: torch.Tensor, + model_kwargs, + model_input_name: str | None, + generation_config: GenerationConfig, + ) -> dict[str, Any]: + encoder_hidden_states = None + # attention mask is consumed once to produce text conditional hidden states through the text encoder + encoder_attention_mask = model_kwargs.pop("attention_mask") + guidance_scale = generation_config.guidance_scale + + # 1. condition on text + if inputs_tensor is not None: + encoder = self.get_encoder() + # Compatibility with Accelerate big model inference: we need the encoder to outputs stuff on the same device + # as the inputs. + if hasattr(encoder, "_hf_hook"): + encoder._hf_hook.io_same_device = True + + # Prepare args and kwargs from model kwargs. + irrelevant_prefix = ["decoder_", "use_cache"] + encoder_kwargs = { + argument: value + for argument, value in model_kwargs.items() + if not any(argument.startswith(p) for p in irrelevant_prefix) + } + encoder_signature = set(inspect.signature(encoder.forward).parameters) + encoder_accepts_wildcard = "kwargs" in encoder_signature or "model_kwargs" in encoder_signature + if not encoder_accepts_wildcard: + encoder_kwargs = { + argument: value for argument, value in encoder_kwargs.items() if argument in encoder_signature + } + encoder_kwargs["output_attentions"] = generation_config.output_attentions + encoder_kwargs["output_hidden_states"] = generation_config.output_hidden_states + + # make sure that encoder returns `ModelOutput` + model_input_name = model_input_name if model_input_name is not None else self.text_encoder.main_input_name + encoder_kwargs["return_dict"] = True + encoder_kwargs[model_input_name] = inputs_tensor + if encoder_attention_mask is not None: + encoder_kwargs["attention_mask"] = encoder_attention_mask + encoder_hidden_states = encoder(**encoder_kwargs).last_hidden_state + + # optionally project encoder_hidden_states + if self.text_encoder.config.hidden_size != self.decoder.config.hidden_size: + encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states) + + # for classifier free guidance we need to add a 'null' input to our encoder hidden states + if guidance_scale is not None and guidance_scale > 1: + encoder_hidden_states = torch.concatenate( + [encoder_hidden_states, torch.zeros_like(encoder_hidden_states)], dim=0 + ) + if encoder_attention_mask is not None: + encoder_attention_mask = torch.concatenate( + [encoder_attention_mask, torch.zeros_like(encoder_attention_mask)], dim=0 + ) + if encoder_attention_mask is not None: + encoder_hidden_states = encoder_hidden_states * encoder_attention_mask[..., None] + + # 2. condition on audio + audio_hidden_states = model_kwargs.get("input_features", None) + + if inputs_tensor is not None: + if audio_hidden_states is not None: + null_audio_hidden_states = torch.zeros_like(audio_hidden_states) + else: + null_audio_hidden_states = torch.zeros( + (inputs_tensor.shape[0], 1, self.config.num_chroma), device=self.device, dtype=self.dtype + ) + null_audio_hidden_states[:, :, 0] = 1 + + if audio_hidden_states is None: + audio_hidden_states = null_audio_hidden_states + + if audio_hidden_states is not None: + # for classifier free guidance we need to add a 'null' input to our audio hidden states + if guidance_scale is not None and guidance_scale > 1: + audio_hidden_states = torch.concatenate([audio_hidden_states, null_audio_hidden_states], dim=0) + + # optionally project audio_hidden_states -> + # (batch_size, seq_len, num_chroma) -> (batch_size, seq_len, hidden_size) + if self.config.num_chroma != self.decoder.config.hidden_size: + audio_hidden_states = self.audio_enc_to_dec_proj(audio_hidden_states) + + # pad or truncate to config.chroma_length + if audio_hidden_states.shape[1] < self.config.chroma_length: + n_repeat = int(math.ceil(self.config.chroma_length / audio_hidden_states.shape[1])) + audio_hidden_states = audio_hidden_states.repeat(1, n_repeat, 1) + audio_hidden_states = audio_hidden_states[:, : self.config.chroma_length] + + if encoder_hidden_states is not None: + encoder_hidden_states = torch.cat([audio_hidden_states, encoder_hidden_states], dim=1) + else: + encoder_hidden_states = audio_hidden_states + + model_kwargs["encoder_hidden_states"] = encoder_hidden_states + + return model_kwargs + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return shift_tokens_right(labels, self.config.decoder.pad_token_id, self.config.decoder.bos_token_id) + + def resize_token_embeddings(self, *args, **kwargs): + raise NotImplementedError( + "Resizing the embedding layers via the EncoderDecoderModel directly is not supported. Please use the" + " respective methods of the wrapped objects (model.encoder.resize_token_embeddings(...) or" + " model.decoder.resize_token_embeddings(...))" + ) + + def _maybe_initialize_input_ids_for_generation( + self, + inputs: torch.Tensor | None, + bos_token_id: int | None, + model_kwargs: dict[str, torch.Tensor], + ) -> torch.LongTensor: + """Initializes input ids for generation, if necessary.""" + if inputs is not None: + return inputs + + if bos_token_id is None: + raise ValueError("`bos_token_id` has to be defined when no `input_ids` are provided.") + + # If there is some tensor in `model_kwargs`, we can infer the batch size from it. This is helpful with + # soft-prompting or in multimodal implementations built on top of decoder-only language models. + batch_size = 1 + for value in model_kwargs.values(): + if isinstance(value, torch.Tensor): + batch_size = value.shape[0] + break + return torch.ones((batch_size, 1), dtype=torch.long, device=self.device) * bos_token_id + + def freeze_audio_encoder(self): + """ + Freeze the audio encoder weights. + """ + for param in self.audio_encoder.parameters(): + param.requires_grad = False + self.audio_encoder._requires_grad = False + + def freeze_text_encoder(self): + """ + Freeze the text encoder weights. + """ + for param in self.text_encoder.parameters(): + param.requires_grad = False + self.text_encoder._requires_grad = False + + # Copied from transformers.models.musicgen.modeling_musicgen.MusicgenForConditionalGeneration._get_decoder_start_token_id + def _get_decoder_start_token_id( + self, decoder_start_token_id: int | list[int] | None = None, bos_token_id: int | None = None + ) -> int: + decoder_start_token_id = ( + decoder_start_token_id + if decoder_start_token_id is not None + else self.generation_config.decoder_start_token_id + ) + bos_token_id = bos_token_id if bos_token_id is not None else self.generation_config.bos_token_id + + if decoder_start_token_id is not None: + return decoder_start_token_id + elif bos_token_id is not None: + return bos_token_id + raise ValueError( + "`decoder_start_token_id` or `bos_token_id` has to be defined for encoder-decoder generation." + ) + + @torch.no_grad() + def generate( + self, + inputs: torch.Tensor | None = None, + generation_config: GenerationConfig | None = None, + logits_processor: LogitsProcessorList | None = None, + stopping_criteria: StoppingCriteriaList | None = None, + synced_gpus: bool | None = None, + streamer: Optional["BaseStreamer"] = None, + **kwargs, + ): + """ + + Generates sequences of token ids for models with a language modeling head. + + + + Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the + model's default generation configuration. You can override any `generation_config` by passing the corresponding + parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. + + For an overview of generation strategies and code examples, check out the [following + guide](./generation_strategies). + + + + Parameters: + inputs (`torch.Tensor` of varying shape depending on the modality, *optional*): + The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the + method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs` + should be in the format `input_ids`. For encoder-decoder models *inputs* can represent any of + `input_ids`, `input_values`, `input_features`, or `pixel_values`. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, the default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + logits_processor (`LogitsProcessorList`, *optional*): + Custom logits processors that complement the default logits processors built from arguments and + generation config. If a logit processor is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + stopping_criteria (`StoppingCriteriaList`, *optional*): + Custom stopping criteria that complement the default stopping criteria built from arguments and a + generation config. If a stopping criteria is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + synced_gpus (`bool`, *optional*, defaults to `False`): + Whether to continue running the while loop until max_length (needed to avoid deadlocking with + `FullyShardedDataParallel` and DeepSpeed ZeRO Stage 3). + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. + kwargs (`dict[str, Any]`, *optional*): + Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be + forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder + specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*. + + Return: + [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` + or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`. + + If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible + [`~utils.ModelOutput`] types are: + + - [`~generation.GenerateDecoderOnlyOutput`], + - [`~generation.GenerateBeamDecoderOnlyOutput`] + + If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible + [`~utils.ModelOutput`] types are: + + - [`~generation.GenerateEncoderDecoderOutput`], + - [`~generation.GenerateBeamEncoderDecoderOutput`] + """ + # 1. Handle `generation_config` and kwargs that might update it, and validate the resulting objects + generation_config, model_kwargs = self._prepare_generation_config(generation_config, **kwargs) + self._validate_model_kwargs(model_kwargs.copy()) + + # 2. Set generation parameters if not already defined + logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList() + stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList() + + requires_attention_mask = "encoder_outputs" not in model_kwargs + kwargs_has_attention_mask = model_kwargs.get("attention_mask", None) is not None + + # 3. Define model inputs + inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs( + inputs, generation_config.bos_token_id, model_kwargs + ) + batch_size = inputs_tensor.shape[0] + self._prepare_special_tokens(generation_config, kwargs_has_attention_mask, device=inputs_tensor.device) + + # 4. Define other model kwargs + model_kwargs["use_cache"] = generation_config.use_cache + model_kwargs["guidance_scale"] = generation_config.guidance_scale + + if model_kwargs.get("attention_mask", None) is None and requires_attention_mask: + model_kwargs["attention_mask"] = self._prepare_attention_mask_for_generation( + inputs_tensor, generation_config, model_kwargs + ) + + if "encoder_hidden_states" not in model_kwargs: + # encoder_hidden_states are created and added to `model_kwargs` + model_kwargs = self._prepare_encoder_hidden_states_kwargs_for_generation( + inputs_tensor, model_kwargs, model_input_name, generation_config + ) + + # 5. Prepare `input_ids` which will be used for auto-regressive generation + input_ids, model_kwargs = self._prepare_decoder_input_ids_for_generation( + batch_size=batch_size, + model_input_name=model_input_name, + model_kwargs=model_kwargs, + decoder_start_token_id=generation_config._decoder_start_token_tensor, + bos_token_id=generation_config._bos_token_tensor, + device=inputs_tensor.device, + ) + + # 6. Prepare `max_length` depending on other stopping criteria. + input_ids_length = input_ids.shape[-1] + has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None + has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None + generation_config = self._prepare_generated_length( + generation_config=generation_config, + has_default_max_length=has_default_max_length, + has_default_min_length=has_default_min_length, + model_input_name=model_input_name, + inputs_tensor=inputs_tensor, + input_ids_length=input_ids_length, + ) + + self._validate_generated_length(generation_config, input_ids_length, has_default_max_length) + + # 7. Prepare the cache. + # - `model_kwargs` may be updated in place with a cache as defined by the parameters in `generation_config`. + # - different models have a different cache name expected by the model (default = "past_key_values") + # - `max_length`, prepared above, is used to determine the maximum cache length + max_cache_length = generation_config.max_length - 1 + if ( + inputs_tensor.shape[1] != input_ids_length + and model_input_name == "inputs_embeds" + and not self.config.is_encoder_decoder + ): + max_cache_length += inputs_tensor.shape[1] + self._prepare_cache_for_generation( + generation_config, + model_kwargs, + generation_mode=None, + batch_size=batch_size, + max_cache_length=max_cache_length, + ) + + # build the delay pattern mask for offsetting each codebook prediction by 1 (this behaviour is specific to MusicGen) + input_ids, decoder_delay_pattern_mask = self.decoder.build_delay_pattern_mask( + input_ids, + pad_token_id=generation_config._decoder_start_token_tensor, + max_length=generation_config.max_length, + ) + # stash the delay mask so that we don't have to recompute in each forward pass + model_kwargs["decoder_delay_pattern_mask"] = decoder_delay_pattern_mask + + # input_ids are ready to be placed on the streamer (if used) + if streamer is not None: + streamer.put(input_ids.cpu()) + + # 8. determine generation mode + generation_mode = generation_config.get_generation_mode() + + # 9. prepare batched CFG externally (to enable coexistence with the unbatched CFG) + if generation_config.guidance_scale is not None and generation_config.guidance_scale > 1: + logits_processor.append(ClassifierFreeGuidanceLogitsProcessor(generation_config.guidance_scale)) + generation_config.guidance_scale = None + + # 10. prepare distribution pre_processing samplers + logits_processor = self._get_logits_processor( + generation_config=generation_config, + input_ids_seq_length=input_ids_length, + encoder_input_ids=inputs_tensor, + prefix_allowed_tokens_fn=None, + logits_processor=logits_processor, + device=input_ids.device, + ) + + # 10. prepare stopping criteria + stopping_criteria = self._get_stopping_criteria( + generation_config=generation_config, stopping_criteria=stopping_criteria + ) + + if generation_mode in (GenerationMode.SAMPLE, GenerationMode.GREEDY_SEARCH): + # expand input_ids with `num_return_sequences` additional sequences per batch + input_ids, model_kwargs = self._expand_inputs_for_generation( + input_ids=input_ids, + expand_size=generation_config.num_return_sequences, + is_encoder_decoder=self.config.is_encoder_decoder, + **model_kwargs, + ) + + # 11. run sample + outputs = self._sample( + input_ids, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + generation_config=generation_config, + synced_gpus=synced_gpus, + streamer=streamer, + **model_kwargs, + ) + + else: + raise ValueError( + "Got incompatible mode for generation, should be one of greedy or sampling. " + "Ensure that beam search is de-activated by setting `num_beams=1`." + ) + + if generation_config.return_dict_in_generate: + output_ids = outputs.sequences + else: + output_ids = outputs + + # apply the pattern mask to the final ids + output_ids = self.decoder.apply_delay_pattern_mask(output_ids, model_kwargs["decoder_delay_pattern_mask"]) + + # revert the pattern delay mask by filtering the pad token id + output_ids = output_ids[output_ids != generation_config._pad_token_tensor].reshape( + batch_size, self.decoder.num_codebooks, -1 + ) + + # append the frame dimension back to the audio codes + output_ids = output_ids[None, ...] + + audio_scales = model_kwargs.get("audio_scales") + if audio_scales is None: + audio_scales = [None] * batch_size + + if self.decoder.config.audio_channels == 1: + output_values = self.audio_encoder.decode( + output_ids, + audio_scales=audio_scales, + ).audio_values + else: + codec_outputs_left = self.audio_encoder.decode(output_ids[:, :, ::2, :], audio_scales=audio_scales) + output_values_left = codec_outputs_left.audio_values + + codec_outputs_right = self.audio_encoder.decode(output_ids[:, :, 1::2, :], audio_scales=audio_scales) + output_values_right = codec_outputs_right.audio_values + + output_values = torch.cat([output_values_left, output_values_right], dim=1) + + if generation_config.return_dict_in_generate: + outputs.sequences = output_values + return outputs + else: + return output_values + + +__all__ = [ + "MusicgenMelodyForConditionalGeneration", + "MusicgenMelodyForCausalLM", + "MusicgenMelodyModel", + "MusicgenMelodyPreTrainedModel", +] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cbddfb4fe92d2c48a3c8f5ee7f8340650616e691 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .tokenization_wav2vec2_phoneme import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py new file mode 100644 index 0000000000000000000000000000000000000000..90fcf51fe787800e6c4d1106f80e8d688d06cbcf --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py @@ -0,0 +1,581 @@ +# Copyright 2021 The Facebook Inc. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization class for Wav2Vec2Phoneme.""" + +import json +import os +from dataclasses import dataclass +from itertools import groupby +from typing import TYPE_CHECKING, Any, Union + +import numpy as np + +from ...tokenization_python import PreTrainedTokenizer +from ...tokenization_utils_base import AddedToken +from ...utils import ( + ModelOutput, + logging, + requires_backends, + to_py_obj, +) + + +logger = logging.get_logger(__name__) + + +if TYPE_CHECKING: + import torch + + +VOCAB_FILES_NAMES = { + "vocab_file": "vocab.json", + "tokenizer_config_file": "tokenizer_config.json", +} + + +# Wav2Vec2Phoneme has no max input length + + +ListOfDict = list[dict[str, int | str]] + + +@dataclass +class Wav2Vec2PhonemeCTCTokenizerOutput(ModelOutput): + """ + Output type of [` Wav2Vec2PhonemeCTCTokenizer`], with transcription. + + Args: + text (list of `str` or `str`): + Decoded logits in text from. Usually the speech transcription. + char_offsets (list of `list[dict[str, Union[int, str]]]` or `list[dict[str, Union[int, str]]]`): + Offsets of the decoded characters. In combination with sampling rate and model downsampling rate char + offsets can be used to compute time stamps for each character. Total logit score of the beam associated with + produced text. + """ + + text: list[str] | str + char_offsets: list[ListOfDict] | ListOfDict = None + + +class Wav2Vec2PhonemeCTCTokenizer(PreTrainedTokenizer): + """ + Constructs a Wav2Vec2PhonemeCTC tokenizer. + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to + the superclass for more information regarding such methods. + + Args: + vocab_file (`str`): + File containing the vocabulary. + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sentence token. + eos_token (`str`, *optional*, defaults to `""`): + The end of sentence token. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (`str`, *optional*, defaults to `""`): + The token used for padding, for example when batching sequences of different lengths. + do_phonemize (`bool`, *optional*, defaults to `True`): + Whether the tokenizer should phonetize the input or not. Only if a sequence of phonemes is passed to the + tokenizer, `do_phonemize` should be set to `False`. + phonemizer_lang (`str`, *optional*, defaults to `"en-us"`): + The language of the phoneme set to which the tokenizer should phonetize the input text to. + phonemizer_backend (`str`, *optional*. defaults to `"espeak"`): + The backend phonetization library that shall be used by the phonemizer library. Defaults to `espeak-ng`. + See the [phonemizer package](https://github.com/bootphon/phonemizer#readme). for more information. + + **kwargs + Additional keyword arguments passed along to [`PreTrainedTokenizer`] + """ + + vocab_files_names = VOCAB_FILES_NAMES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + bos_token="", + eos_token="", + unk_token="", + pad_token="", + phone_delimiter_token=" ", + word_delimiter_token=None, + do_phonemize=True, + phonemizer_lang="en-us", + phonemizer_backend="espeak", + **kwargs, + ): + # Recover delimiters from V5 `*_token` auto-promotion; they aren't vocab tokens. + model_specific = kwargs.get("model_specific_special_tokens") or {} + if "word_delimiter_token" in model_specific: + word_delimiter_token = model_specific.pop("word_delimiter_token") + if "phone_delimiter_token" in model_specific: + phone_delimiter_token = model_specific.pop("phone_delimiter_token") + if not model_specific: + kwargs.pop("model_specific_special_tokens", None) + + self._word_delimiter_token = word_delimiter_token + self._phone_delimiter_token = phone_delimiter_token + self.do_phonemize = do_phonemize + self.phonemizer_lang = phonemizer_lang + self.phonemizer_backend = phonemizer_backend + + if do_phonemize: + self.init_backend(self.phonemizer_lang) + + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) + self.decoder = {v: k for k, v in self.encoder.items()} + + super().__init__( + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + do_phonemize=do_phonemize, + phonemizer_lang=phonemizer_lang, + phonemizer_backend=phonemizer_backend, + **kwargs, + ) + self.init_kwargs["word_delimiter_token"] = word_delimiter_token + self.init_kwargs["phone_delimiter_token"] = phone_delimiter_token + + @property + def vocab_size(self) -> int: + return len(self.decoder) + + def get_vocab(self) -> dict: + vocab = dict(self.encoder.copy()) + vocab.update(self.added_tokens_encoder) + return vocab + + def _add_tokens(self, new_tokens: list[str] | list[AddedToken], special_tokens: bool = False) -> int: + # Overwritten to never strip! + to_add = [] + for token in new_tokens: + if isinstance(token, str): + to_add.append(AddedToken(token, rstrip=False, lstrip=False, normalized=True, special=special_tokens)) + else: + to_add.append(token) + + return super()._add_tokens(to_add, special_tokens) + + def init_backend(self, phonemizer_lang: str): + """ + Initializes the backend. + + Args: + phonemizer_lang (`str`): The language to be used. + """ + requires_backends(self, "phonemizer") + from phonemizer.backend import BACKENDS + + self._phonemizer_backend = BACKENDS[self.phonemizer_backend](phonemizer_lang, language_switch="remove-flags") + + def prepare_for_tokenization( + self, + text: str, + is_split_into_words: bool = False, + phonemizer_lang: str | None = None, + do_phonemize: bool | None = None, + **kwargs, + ) -> tuple[str, dict[str, Any]]: + """ + Performs any necessary transformations before tokenization. + + This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the + `kwargs` at the end of the encoding process to be sure all the arguments have been used. + + Args: + text (`str`): + The text to prepare. + is_split_into_words (`bool`, *optional*, defaults to `False`): + Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the + tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) + which it will tokenize. This is useful for NER or token classification. + phonemizer_lang (`str`, *optional*): + The language of the phoneme set to which the tokenizer should phonetize the input text to. + do_phonemize (`bool`, *optional*): + Whether the tokenizer should phonetize the input text or not. Only if a sequence of phonemes is passed + to the tokenizer, `do_phonemize` should be set to `False`. + + + Returns: + `tuple[str, dict[str, Any]]`: The prepared text and the unused kwargs. + """ + if is_split_into_words: + text = " " + text + + # set whether tokenizer should phonemize or not + if do_phonemize is not None: + self.do_phonemize = do_phonemize + + # set the correct phonemizer language + if phonemizer_lang is not None: + self.phonemizer_lang = phonemizer_lang + self.init_backend(phonemizer_lang) + + return (text, {}) + + def _tokenize(self, text, **kwargs): + """ + Converts a string into a sequence of tokens (string), using the tokenizer. + """ + + # make sure whitespace is stripped to prevent + text = text.strip() + + # phonemize + if self.do_phonemize: + text = text.lower() + + # create list of phonemes + text = self.phonemize(text, self.phonemizer_lang) + + # make sure ' ' is between phonemes + tokens = text.split(" ") + + tokens = list(filter(lambda p: p.strip() != "", tokens)) + return tokens + + def phonemize(self, text: str, phonemizer_lang: str | None = None) -> str: + from phonemizer.separator import Separator + + word_delimiter = self.word_delimiter_token + " " if self.word_delimiter_token is not None else "" + if phonemizer_lang is not None and phonemizer_lang != self.phonemizer_lang: + self.init_backend(phonemizer_lang) + else: + phonemizer_lang = self.phonemizer_lang + + separator = Separator(phone=self.phone_delimiter_token, word=word_delimiter, syllable="") + phonemes = self._phonemizer_backend.phonemize( + [text], + separator=separator, + ) + phonemes = phonemes[0].strip() + + return phonemes + + @property + def word_delimiter_token(self) -> str: + """ + `str`: Word delimiter token. Log an error if used while not having been set. + """ + if self._word_delimiter_token is None: + if self.verbose: + logger.error("Using word_delimiter_token, but it is not set yet.") + return None + return str(self._word_delimiter_token) + + @property + def word_delimiter_token_id(self) -> int | None: + """ + `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been + set. + """ + if self._word_delimiter_token is None: + return None + return self.convert_tokens_to_ids(self.word_delimiter_token) + + @word_delimiter_token.setter + def word_delimiter_token(self, value): + self._word_delimiter_token = value + + @word_delimiter_token_id.setter + def word_delimiter_token_id(self, value): + self._word_delimiter_token = self.convert_tokens_to_ids(value) + + @property + def phone_delimiter_token(self) -> str: + """ + `str`: Word delimiter token. Log an error if used while not having been set. + """ + if self._phone_delimiter_token is None: + if self.verbose: + logger.error("Using phone_delimiter_token, but it is not set yet.") + return None + return str(self._phone_delimiter_token) + + @property + def phone_delimiter_token_id(self) -> int | None: + """ + `Optional[int]`: Id of the phone_delimiter_token in the vocabulary. Returns `None` if the token has not been + set. + """ + if self._phone_delimiter_token is None: + return None + return self.convert_tokens_to_ids(self.phone_delimiter_token) + + @phone_delimiter_token.setter + def phone_delimiter_token(self, value): + self._phone_delimiter_token = value + + @phone_delimiter_token_id.setter + def phone_delimiter_token_id(self, value): + self._phone_delimiter_token = self.convert_tokens_to_ids(value) + + def _convert_token_to_id(self, token: str) -> int: + """Converts a token (str) in an index (integer) using the vocab.""" + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index: int) -> str: + """Converts an index (integer) in a token (str) using the vocab.""" + result = self.decoder.get(index, self.unk_token) + return result + + def convert_tokens_to_string( + self, + tokens: list[str], + group_tokens: bool = True, + spaces_between_special_tokens: bool = False, + filter_word_delimiter_token: bool = True, + output_char_offsets: bool = False, + ) -> str: + """ + Converts a connectionist-temporal-classification (CTC) output tokens into a single string. + """ + # group same tokens into non-repeating tokens in CTC style decoding + if group_tokens: + chars, char_repetitions = zip(*((token, len(list(group_iter))) for token, group_iter in groupby(tokens))) + else: + chars = tokens + char_repetitions = len(tokens) * [1] + + # filter self.pad_token which is used as CTC-blank token + processed_chars = list(filter(lambda char: char != self.pad_token, chars)) + + # also filter self.word_delimiter_token if not not + if filter_word_delimiter_token and self.word_delimiter_token is not None: + processed_chars = list(filter(lambda token: token != self.word_delimiter_token, processed_chars)) + + # retrieve offsets + char_offsets = None + if output_char_offsets: + word_delimiter_token_for_offsets = ( + self.word_delimiter_token if filter_word_delimiter_token is True else None + ) + char_offsets = self._compute_offsets( + char_repetitions, chars, self.pad_token, word_delimiter_token=word_delimiter_token_for_offsets + ) + + if len(char_offsets) != len(processed_chars): + raise ValueError( + f"`char_offsets`: {char_offsets} and `processed_tokens`: {processed_chars}" + " have to be of the same length, but are: `len(offsets)`: " + f"{len(char_offsets)} and `len(processed_tokens)`: {len(processed_chars)}" + ) + + # set tokens to correct processed token + for i, char in enumerate(processed_chars): + char_offsets[i]["char"] = char + + string = " ".join(processed_chars).strip() + + return {"text": string, "char_offsets": char_offsets} + + @staticmethod + def _compute_offsets( + char_repetitions: list[int], chars: list[str], ctc_token: int, word_delimiter_token: int | None = None + ) -> list[dict[str, str | int]]: + end_indices = np.asarray(char_repetitions).cumsum() + start_indices = np.concatenate(([0], end_indices[:-1])) + + offsets = [ + {"char": t, "start_offset": s, "end_offset": e} for t, s, e in zip(chars, start_indices, end_indices) + ] + + # filter out CTC token + offsets = list(filter(lambda offsets: offsets["char"] != ctc_token, offsets)) + + # filter out word delimiter token if necessary + if word_delimiter_token is not None: + offsets = list(filter(lambda offsets: offsets["char"] != word_delimiter_token, offsets)) + + return offsets + + def _decode( + self, + token_ids: list[int], + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool | None = None, + group_tokens: bool = True, + filter_word_delimiter_token: bool = True, + spaces_between_special_tokens: bool = False, + output_char_offsets: bool = False, + ) -> str: + """ + special _decode function is needed for Wav2Vec2PhonemeTokenizer because added tokens should be treated exactly + the same as tokens of the base vocabulary and therefore the function `convert_tokens_to_string` has to be + called on the whole token list and not individually on added tokens + """ + filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) + + result = [] + for token in filtered_tokens: + if skip_special_tokens and token in self.all_special_ids: + continue + result.append(token) + + string_output = self.convert_tokens_to_string( + result, + group_tokens=group_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + filter_word_delimiter_token=filter_word_delimiter_token, + output_char_offsets=output_char_offsets, + ) + + text = string_output["text"] + + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) + if clean_up_tokenization_spaces: + text = self.clean_up_tokenization(text) + + if output_char_offsets: + return Wav2Vec2PhonemeCTCTokenizerOutput(text=text, char_offsets=string_output["char_offsets"]) + else: + return text + + # overwritten from `tokenization_utils_base.py` because we need docs for `output_char_offsets` here + def decode( + self, + token_ids: Union[int, list[int], np.ndarray, "torch.Tensor"], + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool | None = None, + output_char_offsets: bool = False, + **kwargs, + ) -> str: + """ + Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special + tokens and clean up tokenization spaces. + + Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. + + Args: + token_ids (`Union[int, list[int], np.ndarray, torch.Tensor]`): + List of tokenized input ids. Can be obtained using the `__call__` method. + skip_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to remove special tokens in the decoding. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. + output_char_offsets (`bool`, *optional*, defaults to `False`): + Whether or not to output character offsets. Character offsets can be used in combination with the + sampling rate and model downsampling rate to compute the time-stamps of transcribed characters. + + + + Please take a look at the Example of [`~models.wav2vec2.tokenization_wav2vec2.decode`] to better + understand how to make use of `output_word_offsets`. + [`~model.wav2vec2_phoneme.tokenization_wav2vec2_phoneme.batch_decode`] works the same way with + phonemes. + + + + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific decode method. + + Returns: + `str` or [`~models.wav2vec2.tokenization_wav2vec2_phoneme.Wav2Vec2PhonemeCTCTokenizerOutput`]: The decoded + sentence. Will be a [`~models.wav2vec2.tokenization_wav2vec2_phoneme.Wav2Vec2PhonemeCTCTokenizerOutput`] + when `output_char_offsets == True`. + """ + # Convert inputs to python lists + token_ids = to_py_obj(token_ids) + + return self._decode( + token_ids=token_ids, + skip_special_tokens=skip_special_tokens, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + output_char_offsets=output_char_offsets, + **kwargs, + ) + + # overwritten from `tokenization_utils_base.py` because tokenizer can output + # `ModelOutput` which should not be a list for batched output and because + # we need docs for `output_char_offsets` here + def batch_decode( + self, + sequences: Union[list[int], list[list[int]], np.ndarray, "torch.Tensor"], + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool | None = None, + output_char_offsets: bool = False, + **kwargs, + ) -> list[str]: + """ + Convert a list of lists of token ids into a list of strings by calling decode. + + Args: + sequences (`Union[list[int], list[list[int]], np.ndarray, torch.Tensor]`): + List of tokenized input ids. Can be obtained using the `__call__` method. + skip_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to remove special tokens in the decoding. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. + output_char_offsets (`bool`, *optional*, defaults to `False`): + Whether or not to output character offsets. Character offsets can be used in combination with the + sampling rate and model downsampling rate to compute the time-stamps of transcribed characters. + + + + Please take a look at the Example of [`~models.wav2vec2.tokenization_wav2vec2.decode`] to better + understand how to make use of `output_word_offsets`. + [`~model.wav2vec2_phoneme.tokenization_wav2vec2_phoneme.batch_decode`] works analogous with phonemes + and batched output. + + + + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific decode method. + + Returns: + `list[str]` or [`~models.wav2vec2.tokenization_wav2vec2_phoneme.Wav2Vec2PhonemeCTCTokenizerOutput`]: The + decoded sentence. Will be a + [`~models.wav2vec2.tokenization_wav2vec2_phoneme.Wav2Vec2PhonemeCTCTokenizerOutput`] when + `output_char_offsets == True`. + """ + batch_decoded = [ + self.decode( + seq, + skip_special_tokens=skip_special_tokens, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + output_char_offsets=output_char_offsets, + **kwargs, + ) + for seq in sequences + ] + if output_char_offsets: + # transform list of dicts to dict of lists + return Wav2Vec2PhonemeCTCTokenizerOutput({k: [d[k] for d in batch_decoded] for k in batch_decoded[0]}) + + return batch_decoded + + def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str]: + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + return (vocab_file,) + + +__all__ = ["Wav2Vec2PhonemeCTCTokenizer"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_articlefull_elfopt_shared_wheel.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_articlefull_elfopt_shared_wheel.log new file mode 100644 index 0000000000000000000000000000000000000000..e157eee1addf715d0019ccb834af26e55b865902 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_articlefull_elfopt_shared_wheel.log @@ -0,0 +1,49 @@ +Muon: 54 2D params; Nesterov-AdamW: 76 other params +{ + "data_mode": "cache", + "cache_path": "cache/owt_t5_llmclean_qwen36_35b_articlefull_pack1023_10k_appendeos1.pt", + "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext", + "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json", + "text_column": "text", + "pack_len": 1023, + "append_eos": 1, + "num_workers": 0, + "shuffle_buffer": 8192, + "reject_txt": "cache/online_rejected.txt", + "out_dir": "runs/debug_articlefull_elfopt_shared_wheel", + "subset_size": 10000, + "resume": "", + "steps": 1, + "batch_size": 2, + "grad_accum": 1, + "lr": 7.8125e-06, + "blr": 0.001, + "min_lr": 0.0, + "lr_schedule": "constant", + "warmup_steps": 2500, + "warmup_epochs": 0.5, + "optimizer": "muon", + "weight_decay": 0.0, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "grad_clip": 1.0, + "log_every": 1, + "save_every": 999999, + "dim": 768, + "layers": 12, + "heads": 12, + "mlp_dim": 3072, + "time_tokens": 4, + "c_min": 1.0, + "c_max": 1024.0, + "c_schedule": "sqrt", + "seed": 1234, + "loader_batches_per_rank": 5000, + "optimizer_steps_per_epoch": 5000, + "steps_per_epoch": 5000, + "effective_batch_size": 2 +} +[data] mode=cache rows=10000 length=1024 vocab=32100 seen=24862 dropped=2100 bos=1: eos=1: +[optim] optimizer=muon lr=7.812500e-06 blr=1.000000e-03 effective_batch=2 warmup_steps=2500 lr_schedule=constant wd=0.0 loader_batches_per_rank=5000 optimizer_steps_per_epoch=5000 +step=1 lr=3.125000e-09 loss=10.5609 {'pos0_bos_p': 2.3223959942697547e-05, 'pos0_bos_top1': 0, 'last_eos_p': 2.623751242936123e-05, 'last_eos_top1': 0} diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_owt_lr3e3_bottleneck16_step552k_decode64_ema_20260615_084145.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_owt_lr3e3_bottleneck16_step552k_decode64_ema_20260615_084145.log new file mode 100644 index 0000000000000000000000000000000000000000..31daaad96302837427a7af4e149318568a227173 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/infer_owt_lr3e3_bottleneck16_step552k_decode64_ema_20260615_084145.log @@ -0,0 +1,36 @@ +[2026-06-15T08:41:45+00:00] start bottleneck16 latest step552k infer +-rw-r--r-- 1 root root 856M Jun 15 08:03 runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_548000.pt +-rw-r--r-- 1 root root 856M Jun 15 08:12 runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_549000.pt +-rw-r--r-- 1 root root 856M Jun 15 08:20 runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_550000.pt +-rw-r--r-- 1 root root 856M Jun 15 08:29 runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_551000.pt +-rw-r--r-- 1 root root 856M Jun 15 08:37 runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_552000.pt +checkpoint=runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_2x8gpu_50ep_lr3e3_ema0p9999_elfopt_not5_bottleneck16_unfixed_norm_stateprobadd_selfcond_ce_fast_20260612_030202/step_552000.pt +use_ema=1 +step=552000 +decode_steps=64 +n=64 chunk_n=8 gpu=0 +out_base=/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260615 +[2026-06-15T08:41:45+00:00] infer step=552000 decode=64 -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260615/owt_lr3e3_not5_bottleneck16_latest_step552000_ema_sc1p0_decode64_n64 +[2026-06-15T08:41:45+00:00] run decode=64 chunk=0 n=8 seed=123 +[2026-06-15T08:42:02+00:00] done decode=64 chunk=0 +[2026-06-15T08:42:02+00:00] run decode=64 chunk=1 n=8 seed=124 +[2026-06-15T08:42:19+00:00] done decode=64 chunk=1 +[2026-06-15T08:42:19+00:00] run decode=64 chunk=2 n=8 seed=125 +[2026-06-15T08:42:35+00:00] done decode=64 chunk=2 +[2026-06-15T08:42:35+00:00] run decode=64 chunk=3 n=8 seed=126 +[2026-06-15T08:42:52+00:00] done decode=64 chunk=3 +[2026-06-15T08:42:52+00:00] run decode=64 chunk=4 n=8 seed=127 +[2026-06-15T08:43:09+00:00] done decode=64 chunk=4 +[2026-06-15T08:43:09+00:00] run decode=64 chunk=5 n=8 seed=128 +[2026-06-15T08:43:26+00:00] done decode=64 chunk=5 +[2026-06-15T08:43:26+00:00] run decode=64 chunk=6 n=8 seed=129 +[2026-06-15T08:43:42+00:00] done decode=64 chunk=6 +[2026-06-15T08:43:42+00:00] run decode=64 chunk=7 n=8 seed=130 +[2026-06-15T08:43:59+00:00] done decode=64 chunk=7 +merged 64 samples -> /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260615/owt_lr3e3_not5_bottleneck16_latest_step552000_ema_sc1p0_decode64_n64/sc1p0/samples64.txt +loading scorer /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard dtype=fp32 device=cuda +run kind ppl mean_entropy distinct_1 distinct_2 top_token_mass eos_rows eos_total ppl_tokens t5_tokens path +sc1p0 raw_full 32.18946562436723 4.794616828572085 0.06929465724556837 0.3957697373518246 0.04843808107103012 61 61 61390 64536 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260615/owt_lr3e3_not5_bottleneck16_latest_step552000_ema_sc1p0_decode64_n64/sc1p0 +sc1p0 pre_eos 37.00546922148033 4.809895122812238 0.07104502271788517 0.40573516562078005 0.0496616147173768 0 0 58182 62946 /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260615/owt_lr3e3_not5_bottleneck16_latest_step552000_ema_sc1p0_decode64_n64/sc1p0 +[2026-06-15T08:44:23+00:00] done +[2026-06-15T08:44:23+00:00] all done diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_llmclean_qwen36_35b_articlefull_full_rev8_4gpu_resume_20260531_013159.outer.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_llmclean_qwen36_35b_articlefull_full_rev8_4gpu_resume_20260531_013159.outer.log new file mode 100644 index 0000000000000000000000000000000000000000..a777208a5896a93eed897ddc7309b4fb2b00a081 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/owt_llmclean_qwen36_35b_articlefull_full_rev8_4gpu_resume_20260531_013159.outer.log @@ -0,0 +1,1867 @@ +[llmclean 8gpu reverse] GPU_PORTS=0:8008 1:8009 2:8010 3:8011 +[llmclean 8gpu reverse] API_BASES=http://127.0.0.1:8008/v1,http://127.0.0.1:8009/v1,http://127.0.0.1:8010/v1,http://127.0.0.1:8011/v1 +[llmclean 8gpu reverse] workers=48 max_inflight=96 +[llmclean 8gpu reverse] reverse=1 +[llmclean 8gpu reverse] min_doc_idx=0 max_doc_idx=0 +[llmclean 8gpu reverse] out=cache/owt_llmclean_qwen36_35b_articlefull_full_rev8 +[vllm replicas] starting gpu=0 port=8008 log=logs/vllm_qwen36_35b_a3b_gpu0_port8008.log +[vllm replicas] starting gpu=1 port=8009 log=logs/vllm_qwen36_35b_a3b_gpu1_port8009.log +[vllm replicas] starting gpu=2 port=8010 log=logs/vllm_qwen36_35b_a3b_gpu2_port8010.log +[vllm replicas] starting gpu=3 port=8011 log=logs/vllm_qwen36_35b_a3b_gpu3_port8011.log +[vllm replicas] endpoints: + http://127.0.0.1:8008/v1 + http://127.0.0.1:8009/v1 + http://127.0.0.1:8010/v1 + http://127.0.0.1:8011/v1 +[llmclean 8gpu reverse] waiting for vLLM endpoints... +[llm clean] data=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext input_jsonl= out=cache/owt_llmclean_qwen36_35b_articlefull_full_rev8 model=qwen36-35b-a3b workers=48 api_bases=http://127.0.0.1:8008/v1,http://127.0.0.1:8009/v1,http://127.0.0.1:8010/v1,http://127.0.0.1:8011/v1 prompt_style=shortessay max_docs=0 min_doc_idx=0 max_doc_idx=0 max_chunks=0 max_keep_chunks=0 resume=1 reverse=1 existing_keys=1668065 existing_kept=2202897 existing_rejected=77844 +[llm clean] completed=100 submitted=195 kept=2202996 rejected=77845 last=keep reason=rewritten +[llm clean] completed=200 submitted=295 kept=2203091 rejected=77850 last=keep reason=rewritten +[llm clean] completed=300 submitted=395 kept=2203187 rejected=77854 last=reject reason=The text contains dangerous, nonsensical, and incoherent advice (e.g., smearing feces on oneself to tame a bobcat, refus +[llm clean] completed=400 submitted=495 kept=2203284 rejected=77857 last=keep reason=rewritten +[llm clean] completed=500 submitted=595 kept=2203382 rejected=77859 last=keep reason=rewritten +[llm clean] completed=600 submitted=695 kept=2203477 rejected=77864 last=keep reason=rewritten +[llm clean] completed=700 submitted=795 kept=2203575 rejected=77866 last=keep reason=rewritten +[llm clean] completed=800 submitted=895 kept=2203638 rejected=77903 last=keep reason=rewritten +[llm clean] completed=900 submitted=995 kept=2203737 rejected=77904 last=keep reason=rewritten +[llm clean] completed=1000 submitted=1095 kept=2203835 rejected=77906 last=keep reason=rewritten +[llm clean] completed=1100 submitted=1195 kept=2203934 rejected=77907 last=keep reason=rewritten +[llm clean] completed=1200 submitted=1295 kept=2204032 rejected=77909 last=keep reason=rewritten +[llm clean] completed=1300 submitted=1395 kept=2204112 rejected=77929 last=keep reason=clean_no_rewrite +[llm clean] completed=1400 submitted=1495 kept=2204204 rejected=77937 last=keep reason=rewritten +[llm clean] completed=1500 submitted=1595 kept=2204302 rejected=77939 last=keep reason=rewritten +[llm clean] completed=1600 submitted=1695 kept=2204399 rejected=77942 last=keep reason=rewritten +[llm clean] completed=1700 submitted=1795 kept=2204498 rejected=77943 last=keep reason=rewritten +[llm clean] completed=1800 submitted=1895 kept=2204596 rejected=77945 last=keep reason=rewritten +[llm clean] completed=1900 submitted=1995 kept=2204693 rejected=77948 last=keep reason=rewritten +[llm clean] completed=2000 submitted=2095 kept=2204789 rejected=77952 last=keep reason=rewritten +[llm clean] completed=2100 submitted=2195 kept=2204887 rejected=77954 last=keep reason=rewritten +[llm clean] completed=2200 submitted=2295 kept=2204985 rejected=77956 last=keep reason=rewritten +[llm clean] completed=2300 submitted=2395 kept=2205078 rejected=77963 last=keep reason=rewritten +[llm clean] completed=2400 submitted=2495 kept=2205176 rejected=77965 last=keep reason=rewritten +[llm clean] completed=2500 submitted=2595 kept=2205275 rejected=77966 last=keep reason=rewritten +[llm clean] completed=2600 submitted=2695 kept=2205373 rejected=77968 last=keep reason=rewritten +[llm clean] completed=2700 submitted=2795 kept=2205465 rejected=77976 last=reject reason=The text is a series of photo captions and navigation labels from a photo gallery, lacking coherent article-style prose +[llm clean] completed=2800 submitted=2895 kept=2205562 rejected=77979 last=keep reason=rewritten +[llm clean] completed=2900 submitted=2995 kept=2205662 rejected=77979 last=keep reason=rewritten +[llm clean] completed=3000 submitted=3095 kept=2205757 rejected=77984 last=keep reason=rewritten +[llm clean] completed=3100 submitted=3195 kept=2205853 rejected=77988 last=keep reason=rewritten +[llm clean] completed=3200 submitted=3295 kept=2205951 rejected=77990 last=keep reason=rewritten +[llm clean] completed=3300 submitted=3395 kept=2206046 rejected=77995 last=keep reason=rewritten +[llm clean] completed=3400 submitted=3495 kept=2206141 rejected=78000 last=keep reason=rewritten +[llm clean] completed=3500 submitted=3595 kept=2206240 rejected=78001 last=keep reason=rewritten +[llm clean] completed=3600 submitted=3695 kept=2206325 rejected=78016 last=reject reason= +[llm clean] completed=3700 submitted=3795 kept=2206395 rejected=78046 last=keep reason=rewritten +[llm clean] completed=3800 submitted=3895 kept=2206493 rejected=78048 last=keep reason=rewritten +[llm clean] completed=3900 submitted=3995 kept=2206589 rejected=78052 last=keep reason=rewritten +[llm clean] completed=4000 submitted=4095 kept=2206684 rejected=78057 last=keep reason=rewritten +[llm clean] completed=4100 submitted=4195 kept=2206778 rejected=78063 last=keep reason=rewritten +[llm clean] completed=4200 submitted=4295 kept=2206878 rejected=78063 last=keep reason=rewritten +[llm clean] completed=4300 submitted=4395 kept=2206974 rejected=78067 last=keep reason=rewritten +[llm clean] completed=4400 submitted=4495 kept=2207073 rejected=78068 last=keep reason=rewritten +[llm clean] completed=4500 submitted=4595 kept=2207173 rejected=78068 last=keep reason=rewritten +[llm clean] completed=4600 submitted=4695 kept=2207272 rejected=78069 last=keep reason=rewritten +[llm clean] completed=4700 submitted=4795 kept=2207349 rejected=78092 last=keep reason=rewritten +[llm clean] completed=4800 submitted=4895 kept=2207449 rejected=78092 last=keep reason=rewritten +[llm clean] completed=4900 submitted=4995 kept=2207547 rejected=78094 last=keep reason=rewritten +[llm clean] completed=5000 submitted=5095 kept=2207645 rejected=78096 last=keep reason=rewritten +[llm clean] completed=5100 submitted=5195 kept=2207743 rejected=78098 last=keep reason=rewritten +[llm clean] completed=5200 submitted=5295 kept=2207840 rejected=78101 last=keep reason=rewritten +[llm clean] completed=5300 submitted=5395 kept=2207938 rejected=78103 last=keep reason=rewritten +[llm clean] completed=5400 submitted=5495 kept=2208033 rejected=78108 last=keep reason=rewritten +[llm clean] completed=5500 submitted=5595 kept=2208133 rejected=78108 last=keep reason=rewritten +[llm clean] completed=5600 submitted=5695 kept=2208232 rejected=78109 last=keep reason=rewritten +[llm clean] completed=5700 submitted=5795 kept=2208330 rejected=78111 last=keep reason=rewritten +[llm clean] completed=5800 submitted=5895 kept=2208425 rejected=78116 last=keep reason=rewritten +[llm clean] completed=5900 submitted=5995 kept=2208524 rejected=78117 last=keep reason=rewritten +[llm clean] completed=6000 submitted=6095 kept=2208622 rejected=78119 last=keep reason=rewritten +[llm clean] completed=6100 submitted=6195 kept=2208720 rejected=78121 last=keep reason=rewritten +[llm clean] completed=6200 submitted=6295 kept=2208818 rejected=78123 last=keep reason=rewritten +[llm clean] completed=6300 submitted=6395 kept=2208915 rejected=78126 last=keep reason=rewritten +[llm clean] completed=6400 submitted=6495 kept=2209010 rejected=78131 last=keep reason=rewritten +[llm clean] completed=6500 submitted=6595 kept=2209108 rejected=78133 last=keep reason=rewritten +[llm clean] completed=6600 submitted=6695 kept=2209207 rejected=78134 last=keep reason=rewritten +[llm clean] completed=6700 submitted=6795 kept=2209301 rejected=78140 last=reject reason= +[llm clean] completed=6800 submitted=6895 kept=2209398 rejected=78143 last=keep reason=rewritten +[llm clean] completed=6900 submitted=6995 kept=2209498 rejected=78143 last=keep reason=rewritten +[llm clean] completed=7000 submitted=7095 kept=2209597 rejected=78144 last=keep reason=rewritten +[llm clean] completed=7100 submitted=7195 kept=2209695 rejected=78146 last=reject reason=The text is a raw shell script and configuration file content, not coherent article-style prose. It consists of code, co +[llm clean] completed=7200 submitted=7295 kept=2209790 rejected=78151 last=keep reason=rewritten +[llm clean] completed=7300 submitted=7395 kept=2209888 rejected=78153 last=keep reason=rewritten +[llm clean] completed=7400 submitted=7495 kept=2209987 rejected=78154 last=keep reason=rewritten +[llm clean] completed=7500 submitted=7595 kept=2210086 rejected=78155 last=keep reason=rewritten +[llm clean] completed=7600 submitted=7695 kept=2210183 rejected=78158 last=keep reason=rewritten +[llm clean] completed=7700 submitted=7795 kept=2210280 rejected=78161 last=keep reason=rewritten +[llm clean] completed=7800 submitted=7895 kept=2210377 rejected=78164 last=keep reason=rewritten +[llm clean] completed=7900 submitted=7995 kept=2210476 rejected=78165 last=keep reason=rewritten +[llm clean] completed=8000 submitted=8095 kept=2210574 rejected=78167 last=keep reason=rewritten +[llm clean] completed=8100 submitted=8195 kept=2210670 rejected=78171 last=keep reason=rewritten +[llm clean] completed=8200 submitted=8295 kept=2210767 rejected=78174 last=keep reason=rewritten +[llm clean] completed=8300 submitted=8395 kept=2210860 rejected=78181 last=keep reason=rewritten +[llm clean] completed=8400 submitted=8495 kept=2210958 rejected=78183 last=keep reason=rewritten +[llm clean] completed=8500 submitted=8595 kept=2211057 rejected=78184 last=keep reason=rewritten +[llm clean] completed=8600 submitted=8695 kept=2211146 rejected=78195 last=keep reason=rewritten +[llm clean] completed=8700 submitted=8795 kept=2211244 rejected=78197 last=keep reason=rewritten +[llm clean] completed=8800 submitted=8895 kept=2211343 rejected=78198 last=keep reason=rewritten +[llm clean] completed=8900 submitted=8995 kept=2211440 rejected=78201 last=keep reason=rewritten +[llm clean] completed=9000 submitted=9095 kept=2211536 rejected=78205 last=keep reason=rewritten +[llm clean] completed=9100 submitted=9195 kept=2211634 rejected=78207 last=keep reason=rewritten +[llm clean] completed=9200 submitted=9295 kept=2211729 rejected=78212 last=reject reason= +[llm clean] completed=9300 submitted=9395 kept=2211828 rejected=78213 last=keep reason=rewritten +[llm clean] completed=9400 submitted=9495 kept=2211924 rejected=78217 last=reject reason=The input is primarily a list of song lyrics and metadata without coherent article-style prose structure. It lacks narra +[llm clean] completed=9500 submitted=9595 kept=2212023 rejected=78218 last=keep reason=rewritten +[llm clean] completed=9600 submitted=9695 kept=2212120 rejected=78221 last=keep reason=rewritten +[llm clean] completed=9700 submitted=9795 kept=2212220 rejected=78221 last=keep reason=rewritten +[llm clean] completed=9800 submitted=9895 kept=2212317 rejected=78224 last=keep reason=rewritten +[llm clean] completed=9900 submitted=9995 kept=2212406 rejected=78235 last=reject reason= +[llm clean] completed=10000 submitted=10095 kept=2212484 rejected=78257 last=keep reason=rewritten +[llm clean] completed=10100 submitted=10195 kept=2212581 rejected=78260 last=keep reason=rewritten +[llm clean] completed=10200 submitted=10295 kept=2212669 rejected=78272 last=keep reason=rewritten +[llm clean] completed=10300 submitted=10395 kept=2212761 rejected=78280 last=keep reason=rewritten +[llm clean] completed=10400 submitted=10495 kept=2212860 rejected=78281 last=keep reason=rewritten +[llm clean] completed=10500 submitted=10595 kept=2212959 rejected=78282 last=keep reason=rewritten +[llm clean] completed=10600 submitted=10695 kept=2213053 rejected=78288 last=keep reason=rewritten +[llm clean] completed=10700 submitted=10795 kept=2213150 rejected=78291 last=keep reason=rewritten +[llm clean] completed=10800 submitted=10895 kept=2213250 rejected=78291 last=keep reason=rewritten +[llm clean] completed=10900 submitted=10995 kept=2213348 rejected=78293 last=keep reason=rewritten +[llm clean] completed=11000 submitted=11095 kept=2213445 rejected=78296 last=keep reason=rewritten +[llm clean] completed=11100 submitted=11195 kept=2213537 rejected=78304 last=keep reason=rewritten +[llm clean] completed=11200 submitted=11295 kept=2213634 rejected=78307 last=keep reason=rewritten +[llm clean] completed=11300 submitted=11395 kept=2213731 rejected=78310 last=keep reason=rewritten +[llm clean] completed=11400 submitted=11495 kept=2213829 rejected=78312 last=keep reason=rewritten +[llm clean] completed=11500 submitted=11595 kept=2213925 rejected=78316 last=keep reason=rewritten +[llm clean] completed=11600 submitted=11695 kept=2214023 rejected=78318 last=keep reason=rewritten +[llm clean] completed=11700 submitted=11795 kept=2214123 rejected=78318 last=keep reason=rewritten +[llm clean] completed=11800 submitted=11895 kept=2214214 rejected=78327 last=keep reason=rewritten +[llm clean] completed=11900 submitted=11995 kept=2214312 rejected=78329 last=keep reason=rewritten +[llm clean] completed=12000 submitted=12095 kept=2214411 rejected=78330 last=keep reason=rewritten +[llm clean] completed=12100 submitted=12195 kept=2214509 rejected=78332 last=keep reason=rewritten +[llm clean] completed=12200 submitted=12295 kept=2214605 rejected=78336 last=keep reason=rewritten +[llm clean] completed=12300 submitted=12395 kept=2214692 rejected=78349 last=keep reason=rewritten +[llm clean] completed=12400 submitted=12495 kept=2214789 rejected=78352 last=keep reason=rewritten +[llm clean] completed=12500 submitted=12595 kept=2214885 rejected=78356 last=keep reason=rewritten +[llm clean] completed=12600 submitted=12695 kept=2214980 rejected=78361 last=keep reason=rewritten +[llm clean] completed=12700 submitted=12795 kept=2215079 rejected=78362 last=keep reason=rewritten +[llm clean] completed=12800 submitted=12895 kept=2215176 rejected=78365 last=reject reason=The text is a form with dropdown lists of states, countries, and contact fields, lacking coherent article-style prose. +[llm clean] completed=12900 submitted=12995 kept=2215274 rejected=78367 last=keep reason=rewritten +[llm clean] completed=13000 submitted=13095 kept=2215371 rejected=78370 last=keep reason=rewritten +[llm clean] completed=13100 submitted=13195 kept=2215464 rejected=78377 last=keep reason=rewritten +[llm clean] completed=13200 submitted=13295 kept=2215563 rejected=78378 last=keep reason=rewritten +[llm clean] completed=13300 submitted=13395 kept=2215651 rejected=78390 last=keep reason=rewritten +[llm clean] completed=13400 submitted=13495 kept=2215747 rejected=78394 last=keep reason=rewritten +[llm clean] completed=13500 submitted=13595 kept=2215844 rejected=78397 last=keep reason=rewritten +[llm clean] completed=13600 submitted=13695 kept=2215940 rejected=78401 last=keep reason=rewritten +[llm clean] completed=13700 submitted=13795 kept=2216037 rejected=78404 last=keep reason=rewritten +[llm clean] completed=13800 submitted=13895 kept=2216137 rejected=78404 last=keep reason=rewritten +[llm clean] completed=13900 submitted=13995 kept=2216236 rejected=78405 last=keep reason=rewritten +[llm clean] completed=14000 submitted=14095 kept=2216332 rejected=78409 last=keep reason=rewritten +[llm clean] completed=14100 submitted=14195 kept=2216410 rejected=78431 last=keep reason=rewritten +[llm clean] completed=14200 submitted=14295 kept=2216510 rejected=78431 last=keep reason=rewritten +[llm clean] completed=14300 submitted=14395 kept=2216608 rejected=78433 last=keep reason=rewritten +[llm clean] completed=14400 submitted=14495 kept=2216707 rejected=78434 last=keep reason=rewritten +[llm clean] completed=14500 submitted=14595 kept=2216806 rejected=78435 last=keep reason=rewritten +[llm clean] completed=14600 submitted=14695 kept=2216895 rejected=78446 last=keep reason=rewritten +[llm clean] completed=14700 submitted=14795 kept=2216992 rejected=78449 last=keep reason=rewritten +[llm clean] completed=14800 submitted=14895 kept=2217089 rejected=78452 last=keep reason=rewritten +[llm clean] completed=14900 submitted=14995 kept=2217188 rejected=78453 last=keep reason=rewritten +[llm clean] completed=15000 submitted=15095 kept=2217286 rejected=78455 last=keep reason=rewritten +[llm clean] completed=15100 submitted=15195 kept=2217385 rejected=78456 last=keep reason=rewritten +[llm clean] completed=15200 submitted=15295 kept=2217483 rejected=78458 last=keep reason=rewritten +[llm clean] completed=15300 submitted=15395 kept=2217582 rejected=78459 last=keep reason=rewritten +[llm clean] completed=15400 submitted=15495 kept=2217680 rejected=78461 last=keep reason=rewritten +[llm clean] completed=15500 submitted=15595 kept=2217778 rejected=78463 last=keep reason=rewritten +[llm clean] completed=15600 submitted=15695 kept=2217878 rejected=78463 last=keep reason=rewritten +[llm clean] completed=15700 submitted=15795 kept=2217977 rejected=78464 last=keep reason=rewritten +[llm clean] completed=15800 submitted=15895 kept=2218069 rejected=78472 last=keep reason=rewritten +[llm clean] completed=15900 submitted=15995 kept=2218169 rejected=78472 last=keep reason=rewritten +[llm clean] completed=16000 submitted=16095 kept=2218265 rejected=78476 last=keep reason=rewritten +[llm clean] completed=16100 submitted=16195 kept=2218362 rejected=78479 last=keep reason=rewritten +[llm clean] completed=16200 submitted=16295 kept=2218459 rejected=78482 last=keep reason=rewritten +[llm clean] completed=16300 submitted=16395 kept=2218554 rejected=78487 last=keep reason=rewritten +[llm clean] completed=16400 submitted=16495 kept=2218652 rejected=78489 last=keep reason=rewritten +[llm clean] completed=16500 submitted=16595 kept=2218752 rejected=78489 last=keep reason=rewritten +[llm clean] completed=16600 submitted=16695 kept=2218846 rejected=78495 last=keep reason=rewritten +[llm clean] completed=16700 submitted=16795 kept=2218943 rejected=78498 last=keep reason=rewritten +[llm clean] completed=16800 submitted=16895 kept=2219039 rejected=78502 last=keep reason=rewritten +[llm clean] completed=16900 submitted=16995 kept=2219139 rejected=78502 last=keep reason=rewritten +[llm clean] completed=17000 submitted=17095 kept=2219237 rejected=78504 last=keep reason=rewritten +[llm clean] completed=17100 submitted=17195 kept=2219335 rejected=78506 last=keep reason=rewritten +[llm clean] completed=17200 submitted=17295 kept=2219432 rejected=78509 last=keep reason=rewritten +[llm clean] completed=17300 submitted=17395 kept=2219530 rejected=78511 last=keep reason=rewritten +[llm clean] completed=17400 submitted=17495 kept=2219627 rejected=78514 last=keep reason=rewritten +[llm clean] completed=17500 submitted=17595 kept=2219726 rejected=78515 last=keep reason=rewritten +[llm clean] completed=17600 submitted=17695 kept=2219826 rejected=78515 last=keep reason=rewritten +[llm clean] completed=17700 submitted=17795 kept=2219923 rejected=78518 last=keep reason=rewritten +[llm clean] completed=17800 submitted=17895 kept=2220022 rejected=78519 last=keep reason=rewritten +[llm clean] completed=17900 submitted=17995 kept=2220118 rejected=78523 last=keep reason=rewritten +[llm clean] completed=18000 submitted=18095 kept=2220215 rejected=78526 last=keep reason=rewritten +[llm clean] completed=18100 submitted=18195 kept=2220310 rejected=78531 last=keep reason=rewritten +[llm clean] completed=18200 submitted=18295 kept=2220407 rejected=78534 last=keep reason=rewritten +[llm clean] completed=18300 submitted=18395 kept=2220506 rejected=78535 last=keep reason=rewritten +[llm clean] completed=18400 submitted=18495 kept=2220603 rejected=78538 last=keep reason=rewritten +[llm clean] completed=18500 submitted=18595 kept=2220703 rejected=78538 last=keep reason=rewritten +[llm clean] completed=18600 submitted=18695 kept=2220798 rejected=78543 last=keep reason=rewritten +[llm clean] completed=18700 submitted=18795 kept=2220895 rejected=78546 last=keep reason=rewritten +[llm clean] completed=18800 submitted=18895 kept=2220992 rejected=78549 last=keep reason=rewritten +[llm clean] completed=18900 submitted=18995 kept=2221088 rejected=78553 last=keep reason=rewritten +[llm clean] completed=19000 submitted=19095 kept=2221180 rejected=78561 last=keep reason=rewritten +[llm clean] completed=19100 submitted=19195 kept=2221277 rejected=78564 last=keep reason=rewritten +[llm clean] completed=19200 submitted=19295 kept=2221376 rejected=78565 last=keep reason=rewritten +[llm clean] completed=19300 submitted=19395 kept=2221474 rejected=78567 last=keep reason=rewritten +[llm clean] completed=19400 submitted=19495 kept=2221572 rejected=78569 last=keep reason=rewritten +[llm clean] completed=19500 submitted=19595 kept=2221669 rejected=78572 last=keep reason=rewritten +[llm clean] completed=19600 submitted=19695 kept=2221768 rejected=78573 last=keep reason=rewritten +[llm clean] completed=19700 submitted=19795 kept=2221861 rejected=78580 last=reject reason=The text is a list of song titles, years, and album names without any narrative, context, or article-style prose structu +[llm clean] completed=19800 submitted=19895 kept=2221954 rejected=78587 last=keep reason=rewritten +[llm clean] completed=19900 submitted=19995 kept=2222049 rejected=78592 last=keep reason=rewritten +[llm clean] completed=20000 submitted=20095 kept=2222145 rejected=78596 last=keep reason=rewritten +[llm clean] completed=20100 submitted=20195 kept=2222243 rejected=78598 last=keep reason=rewritten +[llm clean] completed=20200 submitted=20295 kept=2222342 rejected=78599 last=keep reason=rewritten +[llm clean] completed=20300 submitted=20395 kept=2222441 rejected=78600 last=keep reason=rewritten +[llm clean] completed=20400 submitted=20495 kept=2222538 rejected=78603 last=keep reason=rewritten +[llm clean] completed=20500 submitted=20595 kept=2222637 rejected=78604 last=keep reason=rewritten +[llm clean] completed=20600 submitted=20695 kept=2222734 rejected=78607 last=keep reason=rewritten +[llm clean] completed=20700 submitted=20795 kept=2222834 rejected=78607 last=keep reason=rewritten +[llm clean] completed=20800 submitted=20895 kept=2222933 rejected=78608 last=keep reason=rewritten +[llm clean] completed=20900 submitted=20995 kept=2223033 rejected=78608 last=keep reason=rewritten +[llm clean] completed=21000 submitted=21095 kept=2223127 rejected=78614 last=keep reason=rewritten +[llm clean] completed=21100 submitted=21195 kept=2223224 rejected=78617 last=keep reason=rewritten +[llm clean] completed=21200 submitted=21295 kept=2223321 rejected=78620 last=keep reason=rewritten +[llm clean] completed=21300 submitted=21395 kept=2223418 rejected=78623 last=keep reason=rewritten +[llm clean] completed=21400 submitted=21495 kept=2223516 rejected=78625 last=keep reason=rewritten +[llm clean] completed=21500 submitted=21595 kept=2223615 rejected=78626 last=keep reason=rewritten +[llm clean] completed=21600 submitted=21695 kept=2223711 rejected=78630 last=keep reason=rewritten +[llm clean] completed=21700 submitted=21795 kept=2223811 rejected=78630 last=keep reason=rewritten +[llm clean] completed=21800 submitted=21895 kept=2223906 rejected=78635 last=keep reason=rewritten +[llm clean] completed=21900 submitted=21995 kept=2224003 rejected=78638 last=keep reason=rewritten +[llm clean] completed=22000 submitted=22095 kept=2224100 rejected=78641 last=keep reason=rewritten +[llm clean] completed=22100 submitted=22195 kept=2224191 rejected=78650 last=keep reason=rewritten +[llm clean] completed=22200 submitted=22295 kept=2224287 rejected=78654 last=keep reason=rewritten +[llm clean] completed=22300 submitted=22395 kept=2224382 rejected=78659 last=keep reason=rewritten +[llm clean] completed=22400 submitted=22495 kept=2224479 rejected=78662 last=keep reason=rewritten +[llm clean] completed=22500 submitted=22595 kept=2224564 rejected=78677 last=keep reason=rewritten +[llm clean] completed=22600 submitted=22695 kept=2224662 rejected=78679 last=keep reason=rewritten +[llm clean] completed=22700 submitted=22795 kept=2224759 rejected=78682 last=keep reason=rewritten +[llm clean] completed=22800 submitted=22895 kept=2224857 rejected=78684 last=keep reason=rewritten +[llm clean] completed=22900 submitted=22995 kept=2224956 rejected=78685 last=keep reason=rewritten +[llm clean] completed=23000 submitted=23095 kept=2225053 rejected=78688 last=keep reason=rewritten +[llm clean] completed=23100 submitted=23195 kept=2225151 rejected=78690 last=keep reason=rewritten +[llm clean] completed=23200 submitted=23295 kept=2225249 rejected=78692 last=keep reason=rewritten +[llm clean] completed=23300 submitted=23395 kept=2225349 rejected=78692 last=keep reason=rewritten +[llm clean] completed=23400 submitted=23495 kept=2225443 rejected=78698 last=keep reason=rewritten +[llm clean] completed=23500 submitted=23595 kept=2225542 rejected=78699 last=keep reason=rewritten +[llm clean] completed=23600 submitted=23695 kept=2225641 rejected=78700 last=keep reason=rewritten +[llm clean] completed=23700 submitted=23795 kept=2225734 rejected=78707 last=reject reason=The text is a fragmented list of names, causes of death, locations, and dates with no coherent narrative or article-styl +[llm clean] completed=23800 submitted=23895 kept=2225832 rejected=78709 last=keep reason=rewritten +[llm clean] completed=23900 submitted=23995 kept=2225931 rejected=78710 last=keep reason=rewritten +[llm clean] completed=24000 submitted=24095 kept=2226029 rejected=78712 last=keep reason=rewritten +[llm clean] completed=24100 submitted=24195 kept=2226122 rejected=78719 last=keep reason=rewritten +[llm clean] completed=24200 submitted=24295 kept=2226221 rejected=78720 last=keep reason=rewritten +[llm clean] completed=24300 submitted=24395 kept=2226316 rejected=78725 last=keep reason=rewritten +[llm clean] completed=24400 submitted=24495 kept=2226416 rejected=78725 last=keep reason=rewritten +[llm clean] completed=24500 submitted=24595 kept=2226515 rejected=78726 last=keep reason=rewritten +[llm clean] completed=24600 submitted=24694 kept=2226608 rejected=78733 last=reject reason=The text is a fragmented forum thread with heavy boilerplate, navigation artifacts, user signatures, and repetitive meta +[llm clean] completed=24700 submitted=24795 kept=2226701 rejected=78740 last=keep reason=rewritten +[llm clean] completed=24800 submitted=24895 kept=2226799 rejected=78742 last=keep reason=rewritten +[llm clean] completed=24900 submitted=24995 kept=2226897 rejected=78744 last=keep reason=rewritten +[llm clean] completed=25000 submitted=25095 kept=2226994 rejected=78747 last=keep reason=rewritten +[llm clean] completed=25100 submitted=25195 kept=2227091 rejected=78750 last=keep reason=rewritten +[llm clean] completed=25200 submitted=25295 kept=2227191 rejected=78750 last=keep reason=rewritten +[llm clean] completed=25300 submitted=25395 kept=2227288 rejected=78753 last=keep reason=rewritten +[llm clean] completed=25400 submitted=25495 kept=2227388 rejected=78753 last=keep reason=rewritten +[llm clean] completed=25500 submitted=25595 kept=2227485 rejected=78756 last=keep reason=rewritten +[llm clean] completed=25600 submitted=25695 kept=2227585 rejected=78756 last=keep reason=rewritten +[llm clean] completed=25700 submitted=25795 kept=2227683 rejected=78758 last=keep reason=rewritten +[llm clean] completed=25800 submitted=25895 kept=2227774 rejected=78767 last=keep reason=rewritten +[llm clean] completed=25900 submitted=25995 kept=2227868 rejected=78773 last=keep reason=rewritten +[llm clean] completed=26000 submitted=26095 kept=2227967 rejected=78774 last=keep reason=rewritten +[llm clean] completed=26100 submitted=26195 kept=2228064 rejected=78777 last=keep reason=rewritten +[llm clean] completed=26200 submitted=26295 kept=2228164 rejected=78777 last=keep reason=rewritten +[llm clean] completed=26300 submitted=26395 kept=2228262 rejected=78779 last=keep reason=rewritten +[llm clean] completed=26400 submitted=26495 kept=2228361 rejected=78780 last=keep reason=rewritten +[llm clean] completed=26500 submitted=26595 kept=2228460 rejected=78781 last=keep reason=rewritten +[llm clean] completed=26600 submitted=26695 kept=2228560 rejected=78781 last=keep reason=rewritten +[llm clean] completed=26700 submitted=26795 kept=2228657 rejected=78784 last=keep reason=rewritten +[llm clean] completed=26800 submitted=26895 kept=2228754 rejected=78787 last=keep reason=rewritten +[llm clean] completed=26900 submitted=26994 kept=2228847 rejected=78794 last=keep reason=rewritten +[llm clean] completed=27000 submitted=27095 kept=2228930 rejected=78811 last=keep reason=rewritten +[llm clean] completed=27100 submitted=27195 kept=2229027 rejected=78814 last=keep reason=rewritten +[llm clean] completed=27200 submitted=27295 kept=2229124 rejected=78817 last=keep reason=rewritten +[llm clean] completed=27300 submitted=27395 kept=2229221 rejected=78820 last=keep reason=rewritten +[llm clean] completed=27400 submitted=27495 kept=2229318 rejected=78823 last=keep reason=rewritten +[llm clean] completed=27500 submitted=27595 kept=2229415 rejected=78826 last=keep reason=rewritten +[llm clean] completed=27600 submitted=27695 kept=2229512 rejected=78829 last=reject reason=The text is a raw, unstructured roster of names, titles, and political affiliations (DNC members, super delegates, etc.) +[llm clean] completed=27700 submitted=27795 kept=2229602 rejected=78839 last=keep reason=rewritten +[llm clean] completed=27800 submitted=27895 kept=2229700 rejected=78841 last=keep reason=rewritten +[llm clean] completed=27900 submitted=27995 kept=2229800 rejected=78841 last=keep reason=rewritten +[llm clean] completed=28000 submitted=28095 kept=2229878 rejected=78863 last=keep reason=rewritten +[llm clean] completed=28100 submitted=28195 kept=2229977 rejected=78864 last=keep reason=rewritten +[llm clean] completed=28200 submitted=28295 kept=2230073 rejected=78868 last=keep reason=rewritten +[llm clean] completed=28300 submitted=28395 kept=2230170 rejected=78871 last=keep reason=rewritten +[llm clean] completed=28400 submitted=28495 kept=2230266 rejected=78875 last=keep reason=rewritten +[llm clean] completed=28500 submitted=28595 kept=2230365 rejected=78876 last=keep reason=rewritten +[llm clean] completed=28600 submitted=28695 kept=2230464 rejected=78877 last=keep reason=rewritten +[llm clean] completed=28700 submitted=28795 kept=2230561 rejected=78880 last=keep reason=rewritten +[llm clean] completed=28800 submitted=28895 kept=2230659 rejected=78882 last=keep reason=rewritten +[llm clean] completed=28900 submitted=28995 kept=2230752 rejected=78889 last=keep reason=rewritten +[llm clean] completed=29000 submitted=29095 kept=2230852 rejected=78889 last=keep reason=rewritten +[llm clean] completed=29100 submitted=29195 kept=2230951 rejected=78890 last=keep reason=rewritten +[llm clean] completed=29200 submitted=29295 kept=2231051 rejected=78890 last=keep reason=rewritten +[llm clean] completed=29300 submitted=29395 kept=2231147 rejected=78894 last=keep reason=rewritten +[llm clean] completed=29400 submitted=29495 kept=2231246 rejected=78895 last=keep reason=rewritten +[llm clean] completed=29500 submitted=29593 kept=2231345 rejected=78896 last=keep reason=rewritten +[llm clean] completed=29600 submitted=29695 kept=2231437 rejected=78904 last=keep reason=rewritten +[llm clean] completed=29700 submitted=29795 kept=2231525 rejected=78916 last=keep reason=rewritten +[llm clean] completed=29800 submitted=29895 kept=2231623 rejected=78918 last=keep reason=rewritten +[llm clean] completed=29900 submitted=29995 kept=2231722 rejected=78919 last=keep reason=rewritten +[llm clean] completed=30000 submitted=30095 kept=2231817 rejected=78924 last=keep reason=rewritten +[llm clean] completed=30100 submitted=30194 kept=2231915 rejected=78926 last=keep reason=rewritten +[llm clean] completed=30200 submitted=30295 kept=2232014 rejected=78927 last=keep reason=rewritten +[llm clean] completed=30300 submitted=30395 kept=2232109 rejected=78932 last=keep reason=rewritten +[llm clean] completed=30400 submitted=30495 kept=2232203 rejected=78938 last=keep reason=rewritten +[llm clean] completed=30500 submitted=30595 kept=2232300 rejected=78941 last=keep reason=rewritten +[llm clean] completed=30600 submitted=30695 kept=2232397 rejected=78944 last=keep reason=rewritten +[llm clean] completed=30700 submitted=30795 kept=2232497 rejected=78944 last=keep reason=rewritten +[llm clean] completed=30800 submitted=30895 kept=2232591 rejected=78950 last=keep reason=rewritten +[llm clean] completed=30900 submitted=30995 kept=2232689 rejected=78952 last=keep reason=rewritten +[llm clean] completed=31000 submitted=31095 kept=2232786 rejected=78955 last=keep reason=rewritten +[llm clean] completed=31100 submitted=31195 kept=2232885 rejected=78956 last=keep reason=rewritten +[llm clean] completed=31200 submitted=31295 kept=2232985 rejected=78956 last=keep reason=rewritten +[llm clean] completed=31300 submitted=31395 kept=2233083 rejected=78958 last=keep reason=rewritten +[llm clean] completed=31400 submitted=31495 kept=2233178 rejected=78963 last=keep reason=rewritten +[llm clean] completed=31500 submitted=31595 kept=2233275 rejected=78966 last=keep reason=rewritten +[llm clean] completed=31600 submitted=31695 kept=2233369 rejected=78972 last=keep reason=rewritten +[llm clean] completed=31700 submitted=31795 kept=2233469 rejected=78972 last=keep reason=rewritten +[llm clean] completed=31800 submitted=31895 kept=2233556 rejected=78985 last=keep reason=rewritten +[llm clean] completed=31900 submitted=31995 kept=2233655 rejected=78986 last=keep reason=rewritten +[llm clean] completed=32000 submitted=32095 kept=2233751 rejected=78990 last=keep reason=rewritten +[llm clean] completed=32100 submitted=32195 kept=2233840 rejected=79001 last=keep reason=rewritten +[llm clean] completed=32200 submitted=32295 kept=2233936 rejected=79005 last=keep reason=rewritten +[llm clean] completed=32300 submitted=32395 kept=2234033 rejected=79008 last=keep reason=rewritten +[llm clean] completed=32400 submitted=32495 kept=2234131 rejected=79010 last=keep reason=rewritten +[llm clean] completed=32500 submitted=32595 kept=2234226 rejected=79015 last=keep reason=rewritten +[llm clean] completed=32600 submitted=32695 kept=2234325 rejected=79016 last=keep reason=rewritten +[llm clean] completed=32700 submitted=32795 kept=2234423 rejected=79018 last=keep reason=rewritten +[llm clean] completed=32800 submitted=32895 kept=2234521 rejected=79020 last=keep reason=rewritten +[llm clean] completed=32900 submitted=32995 kept=2234619 rejected=79022 last=keep reason=rewritten +[llm clean] completed=33000 submitted=33095 kept=2234717 rejected=79024 last=keep reason=rewritten +[llm clean] completed=33100 submitted=33195 kept=2234816 rejected=79025 last=keep reason=rewritten +[llm clean] completed=33200 submitted=33295 kept=2234911 rejected=79030 last=keep reason=rewritten +[llm clean] completed=33300 submitted=33395 kept=2235002 rejected=79039 last=reject reason=The text is a disjointed collection of comic book blurbs, creator bios, and website boilerplate (like 'LOADING...' and ' +[llm clean] completed=33400 submitted=33495 kept=2235097 rejected=79044 last=keep reason=rewritten +[llm clean] completed=33500 submitted=33595 kept=2235194 rejected=79047 last=keep reason=rewritten +[llm clean] completed=33600 submitted=33695 kept=2235290 rejected=79051 last=keep reason=rewritten +[llm clean] completed=33700 submitted=33795 kept=2235390 rejected=79051 last=keep reason=rewritten +[llm clean] completed=33800 submitted=33895 kept=2235486 rejected=79055 last=keep reason=rewritten +[llm clean] completed=33900 submitted=33995 kept=2235580 rejected=79061 last=keep reason=rewritten +[llm clean] completed=34000 submitted=34095 kept=2235678 rejected=79063 last=keep reason=rewritten +[llm clean] completed=34100 submitted=34195 kept=2235777 rejected=79064 last=keep reason=rewritten +[llm clean] completed=34200 submitted=34295 kept=2235873 rejected=79068 last=keep reason=rewritten +[llm clean] completed=34300 submitted=34395 kept=2235971 rejected=79070 last=keep reason=rewritten +[llm clean] completed=34400 submitted=34495 kept=2236070 rejected=79071 last=keep reason=rewritten +[llm clean] completed=34500 submitted=34595 kept=2236165 rejected=79076 last=keep reason=rewritten +[llm clean] completed=34600 submitted=34695 kept=2236264 rejected=79077 last=keep reason=rewritten +[llm clean] completed=34700 submitted=34795 kept=2236359 rejected=79082 last=keep reason=rewritten +[llm clean] completed=34800 submitted=34895 kept=2236454 rejected=79087 last=keep reason=rewritten +[llm clean] completed=34900 submitted=34995 kept=2236554 rejected=79087 last=keep reason=rewritten +[llm clean] completed=35000 submitted=35095 kept=2236652 rejected=79089 last=keep reason=rewritten +[llm clean] completed=35100 submitted=35195 kept=2236735 rejected=79106 last=keep reason=rewritten +[llm clean] completed=35200 submitted=35295 kept=2236831 rejected=79110 last=keep reason=rewritten +[llm clean] completed=35300 submitted=35395 kept=2236929 rejected=79112 last=keep reason=rewritten +[llm clean] completed=35400 submitted=35495 kept=2237028 rejected=79113 last=keep reason=rewritten +[llm clean] completed=35500 submitted=35595 kept=2237127 rejected=79114 last=keep reason=rewritten +[llm clean] completed=35600 submitted=35695 kept=2237226 rejected=79115 last=keep reason=rewritten +[llm clean] completed=35700 submitted=35795 kept=2237325 rejected=79116 last=keep reason=rewritten +[llm clean] completed=35800 submitted=35895 kept=2237422 rejected=79119 last=reject reason=The text is primarily a mix of a short fiction snippet, author notes, meta-commentary, and unrelated blog-style updates. +[llm clean] completed=35900 submitted=35995 kept=2237517 rejected=79124 last=keep reason=rewritten +[llm clean] completed=36000 submitted=36095 kept=2237615 rejected=79126 last=keep reason=rewritten +[llm clean] completed=36100 submitted=36195 kept=2237713 rejected=79128 last=keep reason=rewritten +[llm clean] completed=36200 submitted=36295 kept=2237807 rejected=79134 last=keep reason=rewritten +[llm clean] completed=36300 submitted=36395 kept=2237886 rejected=79155 last=keep reason=rewritten +[llm clean] completed=36400 submitted=36495 kept=2237984 rejected=79157 last=keep reason=rewritten +[llm clean] completed=36500 submitted=36595 kept=2238079 rejected=79162 last=keep reason=rewritten +[llm clean] completed=36600 submitted=36695 kept=2238173 rejected=79168 last=keep reason=rewritten +[llm clean] completed=36700 submitted=36795 kept=2238273 rejected=79168 last=keep reason=rewritten +[llm clean] completed=36800 submitted=36895 kept=2238364 rejected=79177 last=keep reason=rewritten +[llm clean] completed=36900 submitted=36995 kept=2238452 rejected=79189 last=keep reason=rewritten +[llm clean] completed=37000 submitted=37095 kept=2238550 rejected=79191 last=keep reason=rewritten +[llm clean] completed=37100 submitted=37195 kept=2238648 rejected=79193 last=keep reason=rewritten +[llm clean] completed=37200 submitted=37295 kept=2238725 rejected=79216 last=keep reason=rewritten +[llm clean] completed=37300 submitted=37395 kept=2238823 rejected=79218 last=keep reason=rewritten +[llm clean] completed=37400 submitted=37495 kept=2238920 rejected=79221 last=keep reason=rewritten +[llm clean] completed=37500 submitted=37595 kept=2239020 rejected=79221 last=keep reason=rewritten +[llm clean] completed=37600 submitted=37695 kept=2239118 rejected=79223 last=keep reason=rewritten +[llm clean] completed=37700 submitted=37795 kept=2239217 rejected=79224 last=reject reason=The text contains hate speech, antisemitic tropes, and conspiracy theories, making it unsuitable for rewriting into clea +[llm clean] completed=37800 submitted=37895 kept=2239314 rejected=79227 last=keep reason=rewritten +[llm clean] completed=37900 submitted=37995 kept=2239414 rejected=79227 last=keep reason=rewritten +[llm clean] completed=38000 submitted=38095 kept=2239513 rejected=79228 last=keep reason=rewritten +[llm clean] completed=38100 submitted=38195 kept=2239613 rejected=79228 last=keep reason=rewritten +[llm clean] completed=38200 submitted=38295 kept=2239711 rejected=79230 last=keep reason=rewritten +[llm clean] completed=38300 submitted=38395 kept=2239792 rejected=79249 last=keep reason=rewritten +[llm clean] completed=38400 submitted=38495 kept=2239881 rejected=79260 last=keep reason=rewritten +[llm clean] completed=38500 submitted=38595 kept=2239981 rejected=79260 last=keep reason=rewritten +[llm clean] completed=38600 submitted=38695 kept=2240079 rejected=79262 last=keep reason=rewritten +[llm clean] completed=38700 submitted=38795 kept=2240179 rejected=79262 last=keep reason=rewritten +[llm clean] completed=38800 submitted=38895 kept=2240277 rejected=79264 last=keep reason=rewritten +[llm clean] completed=38900 submitted=38995 kept=2240375 rejected=79266 last=keep reason=rewritten +[llm clean] completed=39000 submitted=39095 kept=2240475 rejected=79266 last=keep reason=rewritten +[llm clean] completed=39100 submitted=39195 kept=2240574 rejected=79267 last=keep reason=rewritten +[llm clean] completed=39200 submitted=39295 kept=2240673 rejected=79268 last=keep reason=rewritten +[llm clean] completed=39300 submitted=39395 kept=2240771 rejected=79270 last=keep reason=rewritten +[llm clean] completed=39400 submitted=39495 kept=2240871 rejected=79270 last=keep reason=rewritten +[llm clean] completed=39500 submitted=39595 kept=2240970 rejected=79271 last=keep reason=rewritten +[llm clean] completed=39600 submitted=39695 kept=2241066 rejected=79275 last=keep reason=rewritten +[llm clean] completed=39700 submitted=39795 kept=2241165 rejected=79276 last=keep reason=rewritten +[llm clean] completed=39800 submitted=39895 kept=2241260 rejected=79281 last=keep reason=rewritten +[llm clean] completed=39900 submitted=39995 kept=2241360 rejected=79281 last=keep reason=rewritten +[llm clean] completed=40000 submitted=40095 kept=2241456 rejected=79285 last=keep reason=rewritten +[llm clean] completed=40100 submitted=40195 kept=2241555 rejected=79286 last=keep reason=rewritten +[llm clean] completed=40200 submitted=40295 kept=2241649 rejected=79292 last=keep reason=rewritten +[llm clean] completed=40300 submitted=40395 kept=2241743 rejected=79298 last=keep reason=rewritten +[llm clean] completed=40400 submitted=40495 kept=2241843 rejected=79298 last=keep reason=rewritten +[llm clean] completed=40500 submitted=40595 kept=2241940 rejected=79301 last=keep reason=rewritten +[llm clean] completed=40600 submitted=40695 kept=2242033 rejected=79308 last=keep reason=rewritten +[llm clean] completed=40700 submitted=40795 kept=2242130 rejected=79311 last=keep reason=rewritten +[llm clean] completed=40800 submitted=40895 kept=2242226 rejected=79315 last=keep reason=rewritten +[llm clean] completed=40900 submitted=40995 kept=2242323 rejected=79318 last=keep reason=rewritten +[llm clean] completed=41000 submitted=41095 kept=2242422 rejected=79319 last=keep reason=rewritten +[llm clean] completed=41100 submitted=41195 kept=2242518 rejected=79323 last=keep reason=rewritten +[llm clean] completed=41200 submitted=41295 kept=2242612 rejected=79329 last=keep reason=rewritten +[llm clean] completed=41300 submitted=41395 kept=2242710 rejected=79331 last=keep reason=rewritten +[llm clean] completed=41400 submitted=41495 kept=2242805 rejected=79336 last=reject reason=The text is a satirical or fictional piece with absurd claims (e.g., wealth disparity saving lives, 'affluenza' as a dis +[llm clean] completed=41500 submitted=41595 kept=2242904 rejected=79337 last=keep reason=rewritten +[llm clean] completed=41600 submitted=41695 kept=2243001 rejected=79340 last=keep reason=rewritten +[llm clean] completed=41700 submitted=41795 kept=2243095 rejected=79346 last=keep reason=rewritten +[llm clean] completed=41800 submitted=41895 kept=2243194 rejected=79347 last=keep reason=rewritten +[llm clean] completed=41900 submitted=41995 kept=2243292 rejected=79349 last=keep reason=rewritten +[llm clean] completed=42000 submitted=42095 kept=2243370 rejected=79371 last=reject reason=The text is a collection of user comments from a talk page (Wikipedia discussion) containing usernames, timestamps, talk +[llm clean] completed=42100 submitted=42195 kept=2243466 rejected=79375 last=keep reason=rewritten +[llm clean] completed=42200 submitted=42295 kept=2243565 rejected=79376 last=keep reason=rewritten +[llm clean] completed=42300 submitted=42395 kept=2243664 rejected=79377 last=keep reason=rewritten +[llm clean] completed=42400 submitted=42495 kept=2243762 rejected=79379 last=keep reason=rewritten +[llm clean] completed=42500 submitted=42595 kept=2243858 rejected=79383 last=keep reason=rewritten +[llm clean] completed=42600 submitted=42695 kept=2243956 rejected=79385 last=keep reason=rewritten +[llm clean] completed=42700 submitted=42795 kept=2244053 rejected=79388 last=keep reason=rewritten +[llm clean] completed=42800 submitted=42895 kept=2244151 rejected=79390 last=keep reason=rewritten +[llm clean] completed=42900 submitted=42995 kept=2244249 rejected=79392 last=keep reason=rewritten +[llm clean] completed=43000 submitted=43095 kept=2244348 rejected=79393 last=keep reason=rewritten +[llm clean] completed=43100 submitted=43195 kept=2244442 rejected=79399 last=keep reason=rewritten +[llm clean] completed=43200 submitted=43295 kept=2244540 rejected=79401 last=keep reason=rewritten +[llm clean] completed=43300 submitted=43395 kept=2244639 rejected=79402 last=keep reason=rewritten +[llm clean] completed=43400 submitted=43495 kept=2244738 rejected=79403 last=keep reason=clean_no_rewrite +[llm clean] completed=43500 submitted=43595 kept=2244834 rejected=79407 last=keep reason=rewritten +[llm clean] completed=43600 submitted=43695 kept=2244932 rejected=79409 last=keep reason=rewritten +[llm clean] completed=43700 submitted=43795 kept=2245030 rejected=79411 last=keep reason=rewritten +[llm clean] completed=43800 submitted=43895 kept=2245130 rejected=79411 last=keep reason=rewritten +[llm clean] completed=43900 submitted=43995 kept=2245228 rejected=79413 last=keep reason=rewritten +[llm clean] completed=44000 submitted=44095 kept=2245326 rejected=79415 last=keep reason=rewritten +[llm clean] completed=44100 submitted=44195 kept=2245423 rejected=79418 last=keep reason=rewritten +[llm clean] completed=44200 submitted=44295 kept=2245518 rejected=79423 last=keep reason=rewritten +[llm clean] completed=44300 submitted=44395 kept=2245617 rejected=79424 last=keep reason=rewritten +[llm clean] completed=44400 submitted=44495 kept=2245717 rejected=79424 last=keep reason=rewritten +[llm clean] completed=44500 submitted=44594 kept=2245816 rejected=79425 last=keep reason=rewritten +[llm clean] completed=44600 submitted=44695 kept=2245915 rejected=79426 last=keep reason=rewritten +[llm clean] completed=44700 submitted=44795 kept=2246011 rejected=79430 last=keep reason=rewritten +[llm clean] completed=44800 submitted=44895 kept=2246110 rejected=79431 last=keep reason=rewritten +[llm clean] completed=44900 submitted=44995 kept=2246206 rejected=79435 last=keep reason=rewritten +[llm clean] completed=45000 submitted=45095 kept=2246302 rejected=79439 last=keep reason=rewritten +[llm clean] completed=45100 submitted=45195 kept=2246400 rejected=79441 last=keep reason=rewritten +[llm clean] completed=45200 submitted=45295 kept=2246498 rejected=79443 last=keep reason=rewritten +[llm clean] completed=45300 submitted=45395 kept=2246598 rejected=79443 last=keep reason=rewritten +[llm clean] completed=45400 submitted=45495 kept=2246693 rejected=79448 last=keep reason=rewritten +[llm clean] completed=45500 submitted=45595 kept=2246791 rejected=79450 last=keep reason=rewritten +[llm clean] completed=45600 submitted=45695 kept=2246890 rejected=79451 last=keep reason=rewritten +[llm clean] completed=45700 submitted=45795 kept=2246988 rejected=79453 last=keep reason=rewritten +[llm clean] completed=45800 submitted=45895 kept=2247086 rejected=79455 last=keep reason=rewritten +[llm clean] completed=45900 submitted=45995 kept=2247186 rejected=79455 last=keep reason=rewritten +[llm clean] completed=46000 submitted=46095 kept=2247286 rejected=79455 last=keep reason=rewritten +[llm clean] completed=46100 submitted=46195 kept=2247385 rejected=79456 last=keep reason=rewritten +[llm clean] completed=46200 submitted=46295 kept=2247480 rejected=79461 last=keep reason=rewritten +[llm clean] completed=46300 submitted=46395 kept=2247579 rejected=79462 last=keep reason=rewritten +[llm clean] completed=46400 submitted=46495 kept=2247678 rejected=79463 last=keep reason=rewritten +[llm clean] completed=46500 submitted=46595 kept=2247776 rejected=79465 last=keep reason=rewritten +[llm clean] completed=46600 submitted=46695 kept=2247876 rejected=79465 last=keep reason=rewritten +[llm clean] completed=46700 submitted=46795 kept=2247969 rejected=79472 last=keep reason=rewritten +[llm clean] completed=46800 submitted=46895 kept=2248068 rejected=79473 last=keep reason=rewritten +[llm clean] completed=46900 submitted=46995 kept=2248167 rejected=79474 last=keep reason=rewritten +[llm clean] completed=47000 submitted=47095 kept=2248266 rejected=79475 last=keep reason=rewritten +[llm clean] completed=47100 submitted=47195 kept=2248365 rejected=79476 last=keep reason=rewritten +[llm clean] completed=47200 submitted=47295 kept=2248458 rejected=79483 last=reject reason=The text is a fabricated, satirical, or fake news story presented as a factual report. It contains historically impossib +[llm clean] completed=47300 submitted=47395 kept=2248555 rejected=79486 last=keep reason=rewritten +[llm clean] completed=47400 submitted=47495 kept=2248646 rejected=79495 last=keep reason=rewritten +[llm clean] completed=47500 submitted=47595 kept=2248744 rejected=79497 last=keep reason=rewritten +[llm clean] completed=47600 submitted=47695 kept=2248840 rejected=79501 last=keep reason=rewritten +[llm clean] completed=47700 submitted=47795 kept=2248935 rejected=79506 last=keep reason=rewritten +[llm clean] completed=47800 submitted=47895 kept=2249034 rejected=79507 last=keep reason=rewritten +[llm clean] completed=47900 submitted=47995 kept=2249130 rejected=79511 last=keep reason=rewritten +[llm clean] completed=48000 submitted=48095 kept=2249225 rejected=79516 last=keep reason=rewritten +[llm clean] completed=48100 submitted=48195 kept=2249323 rejected=79518 last=keep reason=rewritten +[llm clean] completed=48200 submitted=48295 kept=2249411 rejected=79530 last=keep reason=rewritten +[llm clean] completed=48300 submitted=48395 kept=2249508 rejected=79533 last=keep reason=rewritten +[llm clean] completed=48400 submitted=48495 kept=2249602 rejected=79539 last=keep reason=rewritten +[llm clean] completed=48500 submitted=48595 kept=2249701 rejected=79540 last=keep reason=rewritten +[llm clean] completed=48600 submitted=48695 kept=2249796 rejected=79545 last=keep reason=rewritten +[llm clean] completed=48700 submitted=48795 kept=2249894 rejected=79547 last=keep reason=rewritten +[llm clean] completed=48800 submitted=48895 kept=2249993 rejected=79548 last=keep reason=rewritten +[llm clean] completed=48900 submitted=48995 kept=2250090 rejected=79551 last=keep reason=rewritten +[llm clean] completed=49000 submitted=49095 kept=2250187 rejected=79554 last=keep reason=rewritten +[llm clean] completed=49100 submitted=49195 kept=2250285 rejected=79556 last=keep reason=rewritten +[llm clean] completed=49200 submitted=49295 kept=2250382 rejected=79559 last=keep reason=rewritten +[llm clean] completed=49300 submitted=49395 kept=2250481 rejected=79560 last=keep reason=rewritten +[llm clean] completed=49400 submitted=49495 kept=2250580 rejected=79561 last=keep reason=rewritten +[llm clean] completed=49500 submitted=49595 kept=2250675 rejected=79566 last=keep reason=rewritten +[llm clean] completed=49600 submitted=49695 kept=2250774 rejected=79567 last=keep reason=rewritten +[llm clean] completed=49700 submitted=49795 kept=2250872 rejected=79569 last=keep reason=rewritten +[llm clean] completed=49800 submitted=49895 kept=2250967 rejected=79574 last=keep reason=rewritten +[llm clean] completed=49900 submitted=49995 kept=2251063 rejected=79578 last=keep reason=rewritten +[llm clean] completed=50000 submitted=50095 kept=2251140 rejected=79601 last=reject reason=The text is a garbled, incoherent mix of machine-translated or heavily corrupted patch notes and developer commentary. I +[llm clean] completed=50100 submitted=50195 kept=2251228 rejected=79613 last=keep reason=rewritten +[llm clean] completed=50200 submitted=50295 kept=2251320 rejected=79621 last=keep reason=rewritten +[llm clean] completed=50300 submitted=50395 kept=2251417 rejected=79624 last=keep reason=rewritten +[llm clean] completed=50400 submitted=50495 kept=2251513 rejected=79628 last=reject reason= +[llm clean] completed=50500 submitted=50595 kept=2251603 rejected=79638 last=keep reason=rewritten +[llm clean] completed=50600 submitted=50695 kept=2251699 rejected=79642 last=keep reason=rewritten +[llm clean] completed=50700 submitted=50795 kept=2251796 rejected=79645 last=keep reason=rewritten +[llm clean] completed=50800 submitted=50895 kept=2251891 rejected=79650 last=keep reason=rewritten +[llm clean] completed=50900 submitted=50995 kept=2251987 rejected=79654 last=reject reason= +[llm clean] completed=51000 submitted=51095 kept=2252082 rejected=79659 last=reject reason=The text is a collection of highly specific, unverified, and often defamatory claims, conspiracy theories, and political +[llm clean] completed=51100 submitted=51195 kept=2252178 rejected=79663 last=keep reason=rewritten +[llm clean] completed=51200 submitted=51295 kept=2252275 rejected=79666 last=keep reason=rewritten +[llm clean] completed=51300 submitted=51395 kept=2252374 rejected=79667 last=keep reason=rewritten +[llm clean] completed=51400 submitted=51495 kept=2252472 rejected=79669 last=keep reason=rewritten +[llm clean] completed=51500 submitted=51595 kept=2252549 rejected=79692 last=keep reason=rewritten +[llm clean] completed=51600 submitted=51695 kept=2252645 rejected=79696 last=keep reason=rewritten +[llm clean] completed=51700 submitted=51795 kept=2252744 rejected=79697 last=keep reason=rewritten +[llm clean] completed=51800 submitted=51895 kept=2252842 rejected=79699 last=keep reason=rewritten +[llm clean] completed=51900 submitted=51995 kept=2252942 rejected=79699 last=keep reason=rewritten +[llm clean] completed=52000 submitted=52095 kept=2253041 rejected=79700 last=keep reason=rewritten +[llm clean] completed=52100 submitted=52195 kept=2253138 rejected=79703 last=keep reason=rewritten +[llm clean] completed=52200 submitted=52295 kept=2253235 rejected=79706 last=keep reason=rewritten +[llm clean] completed=52300 submitted=52395 kept=2253330 rejected=79711 last=keep reason=rewritten +[llm clean] completed=52400 submitted=52495 kept=2253428 rejected=79713 last=keep reason=rewritten +[llm clean] completed=52500 submitted=52595 kept=2253525 rejected=79716 last=keep reason=rewritten +[llm clean] completed=52600 submitted=52695 kept=2253622 rejected=79719 last=keep reason=rewritten +[llm clean] completed=52700 submitted=52795 kept=2253721 rejected=79720 last=keep reason=rewritten +[llm clean] completed=52800 submitted=52895 kept=2253818 rejected=79723 last=keep reason=rewritten +[llm clean] completed=52900 submitted=52995 kept=2253918 rejected=79723 last=keep reason=rewritten +[llm clean] completed=53000 submitted=53095 kept=2254017 rejected=79724 last=keep reason=rewritten +[llm clean] completed=53100 submitted=53195 kept=2254114 rejected=79727 last=keep reason=rewritten +[llm clean] completed=53200 submitted=53295 kept=2254213 rejected=79728 last=keep reason=rewritten +[llm clean] completed=53300 submitted=53395 kept=2254307 rejected=79734 last=keep reason=rewritten +[llm clean] completed=53400 submitted=53495 kept=2254405 rejected=79736 last=keep reason=rewritten +[llm clean] completed=53500 submitted=53595 kept=2254504 rejected=79737 last=keep reason=rewritten +[llm clean] completed=53600 submitted=53695 kept=2254600 rejected=79741 last=keep reason=rewritten +[llm clean] completed=53700 submitted=53795 kept=2254696 rejected=79745 last=keep reason=rewritten +[llm clean] completed=53800 submitted=53895 kept=2254795 rejected=79746 last=keep reason=rewritten +[llm clean] completed=53900 submitted=53995 kept=2254895 rejected=79746 last=keep reason=rewritten +[llm clean] completed=54000 submitted=54095 kept=2254994 rejected=79747 last=keep reason=rewritten +[llm clean] completed=54100 submitted=54195 kept=2255089 rejected=79752 last=keep reason=rewritten +[llm clean] completed=54200 submitted=54295 kept=2255186 rejected=79755 last=keep reason=rewritten +[llm clean] completed=54300 submitted=54395 kept=2255284 rejected=79757 last=keep reason=rewritten +[llm clean] completed=54400 submitted=54495 kept=2255383 rejected=79758 last=keep reason=rewritten +[llm clean] completed=54500 submitted=54595 kept=2255467 rejected=79774 last=keep reason=rewritten +[llm clean] completed=54600 submitted=54695 kept=2255565 rejected=79776 last=keep reason=rewritten +[llm clean] completed=54700 submitted=54795 kept=2255663 rejected=79778 last=keep reason=rewritten +[llm clean] completed=54800 submitted=54895 kept=2255762 rejected=79779 last=keep reason=rewritten +[llm clean] completed=54900 submitted=54995 kept=2255862 rejected=79779 last=keep reason=rewritten +[llm clean] completed=55000 submitted=55095 kept=2255960 rejected=79781 last=keep reason=rewritten +[llm clean] completed=55100 submitted=55195 kept=2256060 rejected=79781 last=keep reason=rewritten +[llm clean] completed=55200 submitted=55295 kept=2256160 rejected=79781 last=keep reason=rewritten +[llm clean] completed=55300 submitted=55395 kept=2256253 rejected=79788 last=keep reason=rewritten +[llm clean] completed=55400 submitted=55495 kept=2256348 rejected=79793 last=keep reason=rewritten +[llm clean] completed=55500 submitted=55595 kept=2256446 rejected=79795 last=keep reason=rewritten +[llm clean] completed=55600 submitted=55695 kept=2256539 rejected=79802 last=keep reason=rewritten +[llm clean] completed=55700 submitted=55795 kept=2256638 rejected=79803 last=keep reason=rewritten +[llm clean] completed=55800 submitted=55895 kept=2256734 rejected=79807 last=keep reason=rewritten +[llm clean] completed=55900 submitted=55995 kept=2256834 rejected=79807 last=keep reason=rewritten +[llm clean] completed=56000 submitted=56095 kept=2256934 rejected=79807 last=keep reason=rewritten +[llm clean] completed=56100 submitted=56195 kept=2257025 rejected=79816 last=keep reason=rewritten +[llm clean] completed=56200 submitted=56295 kept=2257119 rejected=79822 last=keep reason=rewritten +[llm clean] completed=56300 submitted=56395 kept=2257217 rejected=79824 last=keep reason=rewritten +[llm clean] completed=56400 submitted=56495 kept=2257313 rejected=79828 last=keep reason=rewritten +[llm clean] completed=56500 submitted=56595 kept=2257402 rejected=79839 last=keep reason=rewritten +[llm clean] completed=56600 submitted=56695 kept=2257501 rejected=79840 last=keep reason=rewritten +[llm clean] completed=56700 submitted=56795 kept=2257591 rejected=79850 last=keep reason=rewritten +[llm clean] completed=56800 submitted=56895 kept=2257691 rejected=79850 last=keep reason=rewritten +[llm clean] completed=56900 submitted=56995 kept=2257787 rejected=79854 last=keep reason=rewritten +[llm clean] completed=57000 submitted=57095 kept=2257886 rejected=79855 last=keep reason=rewritten +[llm clean] completed=57100 submitted=57195 kept=2257983 rejected=79858 last=keep reason=rewritten +[llm clean] completed=57200 submitted=57295 kept=2258083 rejected=79858 last=keep reason=rewritten +[llm clean] completed=57300 submitted=57395 kept=2258170 rejected=79871 last=keep reason=rewritten +[llm clean] completed=57400 submitted=57495 kept=2258269 rejected=79872 last=keep reason=rewritten +[llm clean] completed=57500 submitted=57595 kept=2258363 rejected=79878 last=keep reason=rewritten +[llm clean] completed=57600 submitted=57695 kept=2258462 rejected=79879 last=keep reason=rewritten +[llm clean] completed=57700 submitted=57795 kept=2258558 rejected=79883 last=keep reason=rewritten +[llm clean] completed=57800 submitted=57895 kept=2258656 rejected=79885 last=keep reason=rewritten +[llm clean] completed=57900 submitted=57995 kept=2258756 rejected=79885 last=keep reason=rewritten +[llm clean] completed=58000 submitted=58095 kept=2258854 rejected=79887 last=keep reason=rewritten +[llm clean] completed=58100 submitted=58195 kept=2258952 rejected=79889 last=keep reason=rewritten +[llm clean] completed=58200 submitted=58295 kept=2259050 rejected=79891 last=keep reason=rewritten +[llm clean] completed=58300 submitted=58395 kept=2259147 rejected=79894 last=keep reason=rewritten +[llm clean] completed=58400 submitted=58495 kept=2259247 rejected=79894 last=keep reason=rewritten +[llm clean] completed=58500 submitted=58595 kept=2259346 rejected=79895 last=keep reason=rewritten +[llm clean] completed=58600 submitted=58695 kept=2259445 rejected=79896 last=keep reason=rewritten +[llm clean] completed=58700 submitted=58795 kept=2259540 rejected=79901 last=keep reason=rewritten +[llm clean] completed=58800 submitted=58895 kept=2259635 rejected=79906 last=reject reason=The input is a list of unrelated news headlines and brief summaries without coherent narrative structure, making it impo +[llm clean] completed=58900 submitted=58995 kept=2259733 rejected=79908 last=keep reason=rewritten +[llm clean] completed=59000 submitted=59095 kept=2259833 rejected=79908 last=keep reason=rewritten +[llm clean] completed=59100 submitted=59195 kept=2259908 rejected=79933 last=keep reason=rewritten +[llm clean] completed=59200 submitted=59295 kept=2260008 rejected=79933 last=keep reason=rewritten +[llm clean] completed=59300 submitted=59395 kept=2260105 rejected=79936 last=keep reason=rewritten +[llm clean] completed=59400 submitted=59495 kept=2260204 rejected=79937 last=keep reason=rewritten +[llm clean] completed=59500 submitted=59595 kept=2260299 rejected=79942 last=keep reason=rewritten +[llm clean] completed=59600 submitted=59695 kept=2260398 rejected=79943 last=keep reason=rewritten +[llm clean] completed=59700 submitted=59795 kept=2260496 rejected=79945 last=keep reason=rewritten +[llm clean] completed=59800 submitted=59895 kept=2260596 rejected=79945 last=keep reason=rewritten +[llm clean] completed=59900 submitted=59995 kept=2260694 rejected=79947 last=keep reason=rewritten +[llm clean] completed=60000 submitted=60095 kept=2260793 rejected=79948 last=keep reason=rewritten +[llm clean] completed=60100 submitted=60195 kept=2260892 rejected=79949 last=keep reason=rewritten +[llm clean] completed=60200 submitted=60295 kept=2260990 rejected=79951 last=keep reason=rewritten +[llm clean] completed=60300 submitted=60395 kept=2261089 rejected=79952 last=keep reason=rewritten +[llm clean] completed=60400 submitted=60495 kept=2261188 rejected=79953 last=keep reason=rewritten +[llm clean] completed=60500 submitted=60595 kept=2261284 rejected=79957 last=keep reason=rewritten +[llm clean] completed=60600 submitted=60695 kept=2261382 rejected=79959 last=keep reason=rewritten +[llm clean] completed=60700 submitted=60795 kept=2261475 rejected=79966 last=keep reason=rewritten +[llm clean] completed=60800 submitted=60895 kept=2261575 rejected=79966 last=keep reason=rewritten +[llm clean] completed=60900 submitted=60995 kept=2261669 rejected=79972 last=keep reason=rewritten +[llm clean] completed=61000 submitted=61095 kept=2261769 rejected=79972 last=keep reason=rewritten +[llm clean] completed=61100 submitted=61195 kept=2261857 rejected=79984 last=reject reason=The input is a list of article titles, authors, dates, and URLs from a newsletter or blog roll, not a coherent article-s +[llm clean] completed=61200 submitted=61295 kept=2261955 rejected=79986 last=keep reason=rewritten +[llm clean] completed=61300 submitted=61395 kept=2262055 rejected=79986 last=keep reason=rewritten +[llm clean] completed=61400 submitted=61495 kept=2262149 rejected=79992 last=reject reason=The text is primarily promotional boilerplate, link lists, and metadata for a YouTube video and network description, lac +[llm clean] completed=61500 submitted=61595 kept=2262247 rejected=79994 last=keep reason=rewritten +[llm clean] completed=61600 submitted=61695 kept=2262341 rejected=80000 last=keep reason=rewritten +[llm clean] completed=61700 submitted=61795 kept=2262438 rejected=80003 last=keep reason=rewritten +[llm clean] completed=61800 submitted=61895 kept=2262535 rejected=80006 last=keep reason=rewritten +[llm clean] completed=61900 submitted=61995 kept=2262633 rejected=80008 last=keep reason=rewritten +[llm clean] completed=62000 submitted=62095 kept=2262733 rejected=80008 last=keep reason=rewritten +[llm clean] completed=62100 submitted=62195 kept=2262832 rejected=80009 last=keep reason=rewritten +[llm clean] completed=62200 submitted=62295 kept=2262925 rejected=80016 last=keep reason=rewritten +[llm clean] completed=62300 submitted=62395 kept=2263019 rejected=80022 last=keep reason=rewritten +[llm clean] completed=62400 submitted=62495 kept=2263117 rejected=80024 last=keep reason=rewritten +[llm clean] completed=62500 submitted=62595 kept=2263217 rejected=80024 last=keep reason=rewritten +[llm clean] completed=62600 submitted=62695 kept=2263313 rejected=80028 last=keep reason=rewritten +[llm clean] completed=62700 submitted=62795 kept=2263412 rejected=80029 last=keep reason=rewritten +[llm clean] completed=62800 submitted=62895 kept=2263507 rejected=80034 last=keep reason=rewritten +[llm clean] completed=62900 submitted=62995 kept=2263601 rejected=80040 last=keep reason=rewritten +[llm clean] completed=63000 submitted=63095 kept=2263699 rejected=80042 last=keep reason=rewritten +[llm clean] completed=63100 submitted=63195 kept=2263792 rejected=80049 last=keep reason=rewritten +[llm clean] completed=63200 submitted=63295 kept=2263889 rejected=80052 last=keep reason=rewritten +[llm clean] completed=63300 submitted=63395 kept=2263986 rejected=80055 last=keep reason=rewritten +[llm clean] completed=63400 submitted=63495 kept=2264081 rejected=80060 last=keep reason=rewritten +[llm clean] completed=63500 submitted=63595 kept=2264174 rejected=80067 last=keep reason=rewritten +[llm clean] completed=63600 submitted=63695 kept=2264269 rejected=80072 last=keep reason=rewritten +[llm clean] completed=63700 submitted=63795 kept=2264367 rejected=80074 last=keep reason=rewritten +[llm clean] completed=63800 submitted=63895 kept=2264466 rejected=80075 last=keep reason=rewritten +[llm clean] completed=63900 submitted=63995 kept=2264565 rejected=80076 last=keep reason=rewritten +[llm clean] completed=64000 submitted=64095 kept=2264665 rejected=80076 last=keep reason=rewritten +[llm clean] completed=64100 submitted=64195 kept=2264763 rejected=80078 last=keep reason=rewritten +[llm clean] completed=64200 submitted=64295 kept=2264859 rejected=80082 last=keep reason=rewritten +[llm clean] completed=64300 submitted=64395 kept=2264956 rejected=80085 last=keep reason=rewritten +[llm clean] completed=64400 submitted=64495 kept=2265052 rejected=80089 last=keep reason=rewritten +[llm clean] completed=64500 submitted=64595 kept=2265143 rejected=80098 last=keep reason=rewritten +[llm clean] completed=64600 submitted=64695 kept=2265238 rejected=80103 last=keep reason=rewritten +[llm clean] completed=64700 submitted=64795 kept=2265333 rejected=80108 last=keep reason=rewritten +[llm clean] completed=64800 submitted=64895 kept=2265427 rejected=80114 last=keep reason=rewritten +[llm clean] completed=64900 submitted=64995 kept=2265526 rejected=80115 last=keep reason=rewritten +[llm clean] completed=65000 submitted=65095 kept=2265620 rejected=80121 last=keep reason=rewritten +[llm clean] completed=65100 submitted=65195 kept=2265719 rejected=80122 last=keep reason=rewritten +[llm clean] completed=65200 submitted=65295 kept=2265818 rejected=80123 last=keep reason=rewritten +[llm clean] completed=65300 submitted=65395 kept=2265914 rejected=80127 last=keep reason=rewritten +[llm clean] completed=65400 submitted=65495 kept=2266013 rejected=80128 last=keep reason=rewritten +[llm clean] completed=65500 submitted=65595 kept=2266110 rejected=80131 last=keep reason=rewritten +[llm clean] completed=65600 submitted=65695 kept=2266205 rejected=80136 last=keep reason=rewritten +[llm clean] completed=65700 submitted=65795 kept=2266304 rejected=80137 last=keep reason=rewritten +[llm clean] completed=65800 submitted=65895 kept=2266402 rejected=80139 last=keep reason=rewritten +[llm clean] completed=65900 submitted=65995 kept=2266494 rejected=80147 last=keep reason=rewritten +[llm clean] completed=66000 submitted=66095 kept=2266592 rejected=80149 last=keep reason=rewritten +[llm clean] completed=66100 submitted=66195 kept=2266674 rejected=80167 last=keep reason=rewritten +[llm clean] completed=66200 submitted=66295 kept=2266770 rejected=80171 last=keep reason=rewritten +[llm clean] completed=66300 submitted=66395 kept=2266865 rejected=80176 last=keep reason=rewritten +[llm clean] completed=66400 submitted=66495 kept=2266959 rejected=80182 last=keep reason=rewritten +[llm clean] completed=66500 submitted=66595 kept=2267051 rejected=80190 last=keep reason=rewritten +[llm clean] completed=66600 submitted=66695 kept=2267147 rejected=80194 last=keep reason=rewritten +[llm clean] completed=66700 submitted=66795 kept=2267243 rejected=80198 last=keep reason=rewritten +[llm clean] completed=66800 submitted=66895 kept=2267339 rejected=80202 last=keep reason=rewritten +[llm clean] completed=66900 submitted=66995 kept=2267437 rejected=80204 last=keep reason=rewritten +[llm clean] completed=67000 submitted=67095 kept=2267533 rejected=80208 last=keep reason=rewritten +[llm clean] completed=67100 submitted=67195 kept=2267629 rejected=80212 last=keep reason=rewritten +[llm clean] completed=67200 submitted=67295 kept=2267721 rejected=80220 last=keep reason=rewritten +[llm clean] completed=67300 submitted=67395 kept=2267820 rejected=80221 last=keep reason=rewritten +[llm clean] completed=67400 submitted=67495 kept=2267915 rejected=80226 last=keep reason=rewritten +[llm clean] completed=67500 submitted=67595 kept=2268014 rejected=80227 last=keep reason=rewritten +[llm clean] completed=67600 submitted=67695 kept=2268111 rejected=80230 last=keep reason=rewritten +[llm clean] completed=67700 submitted=67795 kept=2268209 rejected=80232 last=keep reason=rewritten +[llm clean] completed=67800 submitted=67895 kept=2268308 rejected=80233 last=keep reason=rewritten +[llm clean] completed=67900 submitted=67995 kept=2268406 rejected=80235 last=keep reason=rewritten +[llm clean] completed=68000 submitted=68095 kept=2268503 rejected=80238 last=keep reason=rewritten +[llm clean] completed=68100 submitted=68195 kept=2268598 rejected=80243 last=keep reason=rewritten +[llm clean] completed=68200 submitted=68295 kept=2268695 rejected=80246 last=keep reason=rewritten +[llm clean] completed=68300 submitted=68395 kept=2268794 rejected=80247 last=keep reason=rewritten +[llm clean] completed=68400 submitted=68495 kept=2268893 rejected=80248 last=keep reason=rewritten +[llm clean] completed=68500 submitted=68595 kept=2268989 rejected=80252 last=keep reason=rewritten +[llm clean] completed=68600 submitted=68695 kept=2269086 rejected=80255 last=keep reason=rewritten +[llm clean] completed=68700 submitted=68795 kept=2269181 rejected=80260 last=keep reason=rewritten +[llm clean] completed=68800 submitted=68895 kept=2269275 rejected=80266 last=keep reason=rewritten +[llm clean] completed=68900 submitted=68995 kept=2269373 rejected=80268 last=keep reason=rewritten +[llm clean] completed=69000 submitted=69095 kept=2269468 rejected=80273 last=keep reason=rewritten +[llm clean] completed=69100 submitted=69195 kept=2269566 rejected=80275 last=keep reason=rewritten +[llm clean] completed=69200 submitted=69295 kept=2269646 rejected=80295 last=keep reason=rewritten +[llm clean] completed=69300 submitted=69395 kept=2269745 rejected=80296 last=keep reason=rewritten +[llm clean] completed=69400 submitted=69495 kept=2269840 rejected=80301 last=keep reason=rewritten +[llm clean] completed=69500 submitted=69595 kept=2269937 rejected=80304 last=keep reason=rewritten +[llm clean] completed=69600 submitted=69695 kept=2270035 rejected=80306 last=keep reason=rewritten +[llm clean] completed=69700 submitted=69795 kept=2270126 rejected=80315 last=keep reason=rewritten +[llm clean] completed=69800 submitted=69895 kept=2270225 rejected=80316 last=keep reason=rewritten +[llm clean] completed=69900 submitted=69995 kept=2270325 rejected=80316 last=keep reason=rewritten +[llm clean] completed=70000 submitted=70095 kept=2270422 rejected=80319 last=keep reason=rewritten +[llm clean] completed=70100 submitted=70195 kept=2270520 rejected=80321 last=keep reason=rewritten +[llm clean] completed=70200 submitted=70295 kept=2270612 rejected=80329 last=keep reason=rewritten +[llm clean] completed=70300 submitted=70395 kept=2270711 rejected=80330 last=keep reason=rewritten +[llm clean] completed=70400 submitted=70495 kept=2270808 rejected=80333 last=keep reason=rewritten +[llm clean] completed=70500 submitted=70595 kept=2270907 rejected=80334 last=keep reason=rewritten +[llm clean] completed=70600 submitted=70695 kept=2271005 rejected=80336 last=keep reason=rewritten +[llm clean] completed=70700 submitted=70795 kept=2271104 rejected=80337 last=keep reason=rewritten +[llm clean] completed=70800 submitted=70895 kept=2271199 rejected=80342 last=keep reason=rewritten +[llm clean] completed=70900 submitted=70995 kept=2271295 rejected=80346 last=keep reason=rewritten +[llm clean] completed=71000 submitted=71095 kept=2271390 rejected=80351 last=keep reason=rewritten +[llm clean] completed=71100 submitted=71195 kept=2271487 rejected=80354 last=keep reason=rewritten +[llm clean] completed=71200 submitted=71295 kept=2271585 rejected=80356 last=keep reason=rewritten +[llm clean] completed=71300 submitted=71395 kept=2271676 rejected=80365 last=keep reason=rewritten +[llm clean] completed=71400 submitted=71495 kept=2271775 rejected=80366 last=keep reason=rewritten +[llm clean] completed=71500 submitted=71595 kept=2271868 rejected=80373 last=keep reason=rewritten +[llm clean] completed=71600 submitted=71695 kept=2271962 rejected=80379 last=keep reason=rewritten +[llm clean] completed=71700 submitted=71795 kept=2272060 rejected=80381 last=keep reason=rewritten +[llm clean] completed=71800 submitted=71895 kept=2272158 rejected=80383 last=keep reason=rewritten +[llm clean] completed=71900 submitted=71995 kept=2272255 rejected=80386 last=keep reason=rewritten +[llm clean] completed=72000 submitted=72095 kept=2272355 rejected=80386 last=keep reason=rewritten +[llm clean] completed=72100 submitted=72195 kept=2272454 rejected=80387 last=keep reason=rewritten +[llm clean] completed=72200 submitted=72295 kept=2272550 rejected=80391 last=keep reason=rewritten +[llm clean] completed=72300 submitted=72395 kept=2272645 rejected=80396 last=keep reason=rewritten +[llm clean] completed=72400 submitted=72495 kept=2272743 rejected=80398 last=keep reason=rewritten +[llm clean] completed=72500 submitted=72595 kept=2272842 rejected=80399 last=keep reason=rewritten +[llm clean] completed=72600 submitted=72695 kept=2272942 rejected=80399 last=keep reason=rewritten +[llm clean] completed=72700 submitted=72795 kept=2273035 rejected=80406 last=keep reason=rewritten +[llm clean] completed=72800 submitted=72895 kept=2273134 rejected=80407 last=keep reason=rewritten +[llm clean] completed=72900 submitted=72994 kept=2273219 rejected=80422 last=keep reason=rewritten +[llm clean] completed=73000 submitted=73095 kept=2273318 rejected=80423 last=keep reason=rewritten +[llm clean] completed=73100 submitted=73195 kept=2273413 rejected=80428 last=keep reason=rewritten +[llm clean] completed=73200 submitted=73295 kept=2273513 rejected=80428 last=keep reason=rewritten +[llm clean] completed=73300 submitted=73395 kept=2273606 rejected=80435 last=keep reason=rewritten +[llm clean] completed=73400 submitted=73495 kept=2273699 rejected=80442 last=keep reason=rewritten +[llm clean] completed=73500 submitted=73595 kept=2273798 rejected=80443 last=keep reason=rewritten +[llm clean] completed=73600 submitted=73695 kept=2273891 rejected=80450 last=keep reason=rewritten +[llm clean] completed=73700 submitted=73795 kept=2273989 rejected=80452 last=keep reason=rewritten +[llm clean] completed=73800 submitted=73895 kept=2274088 rejected=80453 last=reject reason=The text is a list of names, titles, and isolated quotes without coherent narrative structure or meaningful context to r +[llm clean] completed=73900 submitted=73995 kept=2274185 rejected=80456 last=keep reason=rewritten +[llm clean] completed=74000 submitted=74095 kept=2274280 rejected=80461 last=keep reason=rewritten +[llm clean] completed=74100 submitted=74195 kept=2274380 rejected=80461 last=keep reason=rewritten +[llm clean] completed=74200 submitted=74295 kept=2274477 rejected=80464 last=keep reason=rewritten +[llm clean] completed=74300 submitted=74395 kept=2274575 rejected=80466 last=keep reason=rewritten +[llm clean] completed=74400 submitted=74495 kept=2274673 rejected=80468 last=keep reason=rewritten +[llm clean] completed=74500 submitted=74595 kept=2274771 rejected=80470 last=keep reason=rewritten +[llm clean] completed=74600 submitted=74695 kept=2274869 rejected=80472 last=keep reason=rewritten +[llm clean] completed=74700 submitted=74795 kept=2274969 rejected=80472 last=keep reason=rewritten +[llm clean] completed=74800 submitted=74895 kept=2275068 rejected=80473 last=keep reason=rewritten +[llm clean] completed=74900 submitted=74995 kept=2275167 rejected=80474 last=keep reason=rewritten +[llm clean] completed=75000 submitted=75095 kept=2275263 rejected=80478 last=keep reason=rewritten +[llm clean] completed=75100 submitted=75195 kept=2275361 rejected=80480 last=keep reason=rewritten +[llm clean] completed=75200 submitted=75295 kept=2275458 rejected=80483 last=keep reason=rewritten +[llm clean] completed=75300 submitted=75395 kept=2275545 rejected=80496 last=keep reason=rewritten +[llm clean] completed=75400 submitted=75495 kept=2275644 rejected=80497 last=keep reason=rewritten +[llm clean] completed=75500 submitted=75595 kept=2275741 rejected=80500 last=keep reason=rewritten +[llm clean] completed=75600 submitted=75695 kept=2275841 rejected=80500 last=keep reason=rewritten +[llm clean] completed=75700 submitted=75795 kept=2275939 rejected=80502 last=keep reason=rewritten +[llm clean] completed=75800 submitted=75895 kept=2276039 rejected=80502 last=keep reason=rewritten +[llm clean] completed=75900 submitted=75995 kept=2276139 rejected=80502 last=keep reason=rewritten +[llm clean] completed=76000 submitted=76095 kept=2276239 rejected=80502 last=keep reason=rewritten +[llm clean] completed=76100 submitted=76195 kept=2276338 rejected=80503 last=keep reason=rewritten +[llm clean] completed=76200 submitted=76295 kept=2276435 rejected=80506 last=keep reason=rewritten +[llm clean] completed=76300 submitted=76395 kept=2276533 rejected=80508 last=keep reason=rewritten +[llm clean] completed=76400 submitted=76495 kept=2276629 rejected=80512 last=keep reason=rewritten +[llm clean] completed=76500 submitted=76595 kept=2276726 rejected=80515 last=keep reason=rewritten +[llm clean] completed=76600 submitted=76695 kept=2276823 rejected=80518 last=keep reason=rewritten +[llm clean] completed=76700 submitted=76795 kept=2276921 rejected=80520 last=keep reason=rewritten +[llm clean] completed=76800 submitted=76895 kept=2277021 rejected=80520 last=keep reason=rewritten +[llm clean] completed=76900 submitted=76995 kept=2277118 rejected=80523 last=keep reason=rewritten +[llm clean] completed=77000 submitted=77095 kept=2277215 rejected=80526 last=keep reason=rewritten +[llm clean] completed=77100 submitted=77195 kept=2277311 rejected=80530 last=keep reason=rewritten +[llm clean] completed=77200 submitted=77295 kept=2277410 rejected=80531 last=keep reason=rewritten +[llm clean] completed=77300 submitted=77395 kept=2277509 rejected=80532 last=keep reason=rewritten +[llm clean] completed=77400 submitted=77495 kept=2277609 rejected=80532 last=keep reason=rewritten +[llm clean] completed=77500 submitted=77595 kept=2277709 rejected=80532 last=keep reason=rewritten +[llm clean] completed=77600 submitted=77695 kept=2277804 rejected=80537 last=keep reason=rewritten +[llm clean] completed=77700 submitted=77795 kept=2277904 rejected=80537 last=keep reason=rewritten +[llm clean] completed=77800 submitted=77895 kept=2278003 rejected=80538 last=keep reason=rewritten +[llm clean] completed=77900 submitted=77995 kept=2278096 rejected=80545 last=keep reason=rewritten +[llm clean] completed=78000 submitted=78095 kept=2278179 rejected=80562 last=keep reason=rewritten +[llm clean] completed=78100 submitted=78195 kept=2278276 rejected=80565 last=keep reason=rewritten +[llm clean] completed=78200 submitted=78295 kept=2278374 rejected=80567 last=keep reason=rewritten +[llm clean] completed=78300 submitted=78395 kept=2278469 rejected=80572 last=keep reason=rewritten +[llm clean] completed=78400 submitted=78495 kept=2278565 rejected=80576 last=keep reason=rewritten +[llm clean] completed=78500 submitted=78595 kept=2278663 rejected=80578 last=keep reason=rewritten +[llm clean] completed=78600 submitted=78695 kept=2278762 rejected=80579 last=keep reason=rewritten +[llm clean] completed=78700 submitted=78795 kept=2278860 rejected=80581 last=keep reason=rewritten +[llm clean] completed=78800 submitted=78895 kept=2278959 rejected=80582 last=keep reason=rewritten +[llm clean] completed=78900 submitted=78995 kept=2279056 rejected=80585 last=keep reason=rewritten +[llm clean] completed=79000 submitted=79095 kept=2279156 rejected=80585 last=keep reason=rewritten +[llm clean] completed=79100 submitted=79195 kept=2279249 rejected=80592 last=keep reason=rewritten +[llm clean] completed=79200 submitted=79295 kept=2279345 rejected=80596 last=keep reason=rewritten +[llm clean] completed=79300 submitted=79395 kept=2279442 rejected=80599 last=keep reason=rewritten +[llm clean] completed=79400 submitted=79495 kept=2279540 rejected=80601 last=keep reason=rewritten +[llm clean] completed=79500 submitted=79595 kept=2279635 rejected=80606 last=keep reason=rewritten +[llm clean] completed=79600 submitted=79695 kept=2279730 rejected=80611 last=keep reason=rewritten +[llm clean] completed=79700 submitted=79795 kept=2279828 rejected=80613 last=keep reason=rewritten +[llm clean] completed=79800 submitted=79895 kept=2279926 rejected=80615 last=keep reason=rewritten +[llm clean] completed=79900 submitted=79995 kept=2280023 rejected=80618 last=keep reason=rewritten +[llm clean] completed=80000 submitted=80095 kept=2280121 rejected=80620 last=keep reason=rewritten +[llm clean] completed=80100 submitted=80195 kept=2280217 rejected=80624 last=keep reason=rewritten +[llm clean] completed=80200 submitted=80295 kept=2280311 rejected=80630 last=keep reason=rewritten +[llm clean] completed=80300 submitted=80395 kept=2280407 rejected=80634 last=keep reason=rewritten +[llm clean] completed=80400 submitted=80495 kept=2280507 rejected=80634 last=keep reason=rewritten +[llm clean] completed=80500 submitted=80595 kept=2280606 rejected=80635 last=keep reason=rewritten +[llm clean] completed=80600 submitted=80695 kept=2280704 rejected=80637 last=keep reason=rewritten +[llm clean] completed=80700 submitted=80795 kept=2280802 rejected=80639 last=keep reason=rewritten +[llm clean] completed=80800 submitted=80895 kept=2280901 rejected=80640 last=keep reason=rewritten +[llm clean] completed=80900 submitted=80995 kept=2280998 rejected=80643 last=keep reason=rewritten +[llm clean] completed=81000 submitted=81095 kept=2281095 rejected=80646 last=keep reason=rewritten +[llm clean] completed=81100 submitted=81195 kept=2281193 rejected=80648 last=keep reason=rewritten +[llm clean] completed=81200 submitted=81295 kept=2281286 rejected=80655 last=keep reason=rewritten +[llm clean] completed=81300 submitted=81395 kept=2281382 rejected=80659 last=keep reason=rewritten +[llm clean] completed=81400 submitted=81495 kept=2281481 rejected=80660 last=keep reason=rewritten +[llm clean] completed=81500 submitted=81595 kept=2281577 rejected=80664 last=keep reason=rewritten +[llm clean] completed=81600 submitted=81695 kept=2281675 rejected=80666 last=keep reason=rewritten +[llm clean] completed=81700 submitted=81795 kept=2281772 rejected=80669 last=keep reason=rewritten +[llm clean] completed=81800 submitted=81895 kept=2281870 rejected=80671 last=keep reason=rewritten +[llm clean] completed=81900 submitted=81995 kept=2281969 rejected=80672 last=keep reason=rewritten +[llm clean] completed=82000 submitted=82095 kept=2282067 rejected=80674 last=keep reason=rewritten +[llm clean] completed=82100 submitted=82195 kept=2282162 rejected=80679 last=keep reason=rewritten +[llm clean] completed=82200 submitted=82295 kept=2282259 rejected=80682 last=reject reason=The text is a meaningless, contextless list of anagrams and word permutations with no coherent prose structure or factua +[llm clean] completed=82300 submitted=82395 kept=2282327 rejected=80714 last=keep reason=rewritten +[llm clean] completed=82400 submitted=82495 kept=2282423 rejected=80718 last=keep reason=rewritten +[llm clean] completed=82500 submitted=82595 kept=2282523 rejected=80718 last=keep reason=rewritten +[llm clean] completed=82600 submitted=82695 kept=2282616 rejected=80725 last=keep reason=rewritten +[llm clean] completed=82700 submitted=82794 kept=2282707 rejected=80734 last=keep reason=rewritten +[llm clean] completed=82800 submitted=82895 kept=2282800 rejected=80741 last=keep reason=rewritten +[llm clean] completed=82900 submitted=82995 kept=2282900 rejected=80741 last=keep reason=rewritten +[llm clean] completed=83000 submitted=83095 kept=2282999 rejected=80742 last=keep reason=rewritten +[llm clean] completed=83100 submitted=83195 kept=2283096 rejected=80745 last=keep reason=rewritten +[llm clean] completed=83200 submitted=83295 kept=2283195 rejected=80746 last=keep reason=rewritten +[llm clean] completed=83300 submitted=83395 kept=2283289 rejected=80752 last=keep reason=rewritten +[llm clean] completed=83400 submitted=83495 kept=2283389 rejected=80752 last=keep reason=rewritten +[llm clean] completed=83500 submitted=83595 kept=2283489 rejected=80752 last=keep reason=rewritten +[llm clean] completed=83600 submitted=83695 kept=2283589 rejected=80752 last=keep reason=rewritten +[llm clean] completed=83700 submitted=83795 kept=2283687 rejected=80754 last=keep reason=rewritten +[llm clean] completed=83800 submitted=83895 kept=2283784 rejected=80757 last=keep reason=rewritten +[llm clean] completed=83900 submitted=83995 kept=2283877 rejected=80764 last=keep reason=rewritten +[llm clean] completed=84000 submitted=84095 kept=2283976 rejected=80765 last=keep reason=rewritten +[llm clean] completed=84100 submitted=84195 kept=2284076 rejected=80765 last=keep reason=rewritten +[llm clean] completed=84200 submitted=84295 kept=2284176 rejected=80765 last=keep reason=rewritten +[llm clean] completed=84300 submitted=84395 kept=2284275 rejected=80766 last=keep reason=rewritten +[llm clean] completed=84400 submitted=84495 kept=2284375 rejected=80766 last=keep reason=rewritten +[llm clean] completed=84500 submitted=84595 kept=2284472 rejected=80769 last=keep reason=rewritten +[llm clean] completed=84600 submitted=84695 kept=2284570 rejected=80771 last=keep reason=rewritten +[llm clean] completed=84700 submitted=84795 kept=2284662 rejected=80779 last=keep reason=rewritten +[llm clean] completed=84800 submitted=84895 kept=2284762 rejected=80779 last=keep reason=rewritten +[llm clean] completed=84900 submitted=84995 kept=2284861 rejected=80780 last=keep reason=rewritten +[llm clean] completed=85000 submitted=85095 kept=2284959 rejected=80782 last=keep reason=rewritten +[llm clean] completed=85100 submitted=85195 kept=2285058 rejected=80783 last=keep reason=rewritten +[llm clean] completed=85200 submitted=85295 kept=2285154 rejected=80787 last=keep reason=rewritten +[llm clean] completed=85300 submitted=85395 kept=2285248 rejected=80793 last=keep reason=rewritten +[llm clean] completed=85400 submitted=85495 kept=2285341 rejected=80800 last=keep reason=rewritten +[llm clean] completed=85500 submitted=85595 kept=2285439 rejected=80802 last=keep reason=rewritten +[llm clean] completed=85600 submitted=85695 kept=2285539 rejected=80802 last=keep reason=rewritten +[llm clean] completed=85700 submitted=85795 kept=2285639 rejected=80802 last=keep reason=rewritten +[llm clean] completed=85800 submitted=85895 kept=2285738 rejected=80803 last=keep reason=rewritten +[llm clean] completed=85900 submitted=85995 kept=2285830 rejected=80811 last=reject reason=The text is a disjointed collection of app store descriptions, metadata, and unrelated fragments (concert catalog, file +[llm clean] completed=86000 submitted=86095 kept=2285927 rejected=80814 last=keep reason=rewritten +[llm clean] completed=86100 submitted=86195 kept=2286027 rejected=80814 last=keep reason=rewritten +[llm clean] completed=86200 submitted=86295 kept=2286127 rejected=80814 last=keep reason=rewritten +[llm clean] completed=86300 submitted=86395 kept=2286225 rejected=80816 last=keep reason=rewritten +[llm clean] completed=86400 submitted=86495 kept=2286320 rejected=80821 last=keep reason=rewritten +[llm clean] completed=86500 submitted=86595 kept=2286418 rejected=80823 last=keep reason=rewritten +[llm clean] completed=86600 submitted=86695 kept=2286516 rejected=80825 last=keep reason=rewritten +[llm clean] completed=86700 submitted=86795 kept=2286606 rejected=80835 last=keep reason=rewritten +[llm clean] completed=86800 submitted=86895 kept=2286705 rejected=80836 last=keep reason=rewritten +[llm clean] completed=86900 submitted=86995 kept=2286802 rejected=80839 last=keep reason=rewritten +[llm clean] completed=87000 submitted=87095 kept=2286900 rejected=80841 last=keep reason=rewritten +[llm clean] completed=87100 submitted=87195 kept=2287000 rejected=80841 last=keep reason=rewritten +[llm clean] completed=87200 submitted=87295 kept=2287096 rejected=80845 last=keep reason=rewritten +[llm clean] completed=87300 submitted=87395 kept=2287190 rejected=80851 last=keep reason=rewritten +[llm clean] completed=87400 submitted=87495 kept=2287260 rejected=80881 last=keep reason=rewritten +[llm clean] completed=87500 submitted=87595 kept=2287359 rejected=80882 last=keep reason=rewritten +[llm clean] completed=87600 submitted=87695 kept=2287458 rejected=80883 last=keep reason=rewritten +[llm clean] completed=87700 submitted=87795 kept=2287553 rejected=80888 last=keep reason=rewritten +[llm clean] completed=87800 submitted=87895 kept=2287651 rejected=80890 last=keep reason=rewritten +[llm clean] completed=87900 submitted=87995 kept=2287751 rejected=80890 last=keep reason=rewritten +[llm clean] completed=88000 submitted=88095 kept=2287851 rejected=80890 last=keep reason=rewritten +[llm clean] completed=88100 submitted=88195 kept=2287951 rejected=80890 last=keep reason=rewritten +[llm clean] completed=88200 submitted=88295 kept=2288051 rejected=80890 last=keep reason=rewritten +[llm clean] completed=88300 submitted=88395 kept=2288149 rejected=80892 last=keep reason=rewritten +[llm clean] completed=88400 submitted=88495 kept=2288248 rejected=80893 last=keep reason=rewritten +[llm clean] completed=88500 submitted=88595 kept=2288346 rejected=80895 last=keep reason=rewritten +[llm clean] completed=88600 submitted=88695 kept=2288443 rejected=80898 last=keep reason=rewritten +[llm clean] completed=88700 submitted=88795 kept=2288542 rejected=80899 last=keep reason=rewritten +[llm clean] completed=88800 submitted=88895 kept=2288638 rejected=80903 last=keep reason=rewritten +[llm clean] completed=88900 submitted=88995 kept=2288730 rejected=80911 last=keep reason=rewritten +[llm clean] completed=89000 submitted=89095 kept=2288829 rejected=80912 last=keep reason=rewritten +[llm clean] completed=89100 submitted=89195 kept=2288926 rejected=80915 last=keep reason=rewritten +[llm clean] completed=89200 submitted=89295 kept=2289019 rejected=80922 last=keep reason=rewritten +[llm clean] completed=89300 submitted=89395 kept=2289113 rejected=80928 last=keep reason=rewritten +[llm clean] completed=89400 submitted=89495 kept=2289212 rejected=80929 last=keep reason=rewritten +[llm clean] completed=89500 submitted=89595 kept=2289307 rejected=80934 last=keep reason=rewritten +[llm clean] completed=89600 submitted=89695 kept=2289401 rejected=80940 last=keep reason=rewritten +[llm clean] completed=89700 submitted=89795 kept=2289497 rejected=80944 last=keep reason=rewritten +[llm clean] completed=89800 submitted=89895 kept=2289596 rejected=80945 last=keep reason=rewritten +[llm clean] completed=89900 submitted=89995 kept=2289692 rejected=80949 last=reject reason=The text is a bibliography or reference list, consisting of isolated citations, titles, and URLs without coherent prose +[llm clean] completed=90000 submitted=90095 kept=2289790 rejected=80951 last=keep reason=rewritten +[llm clean] completed=90100 submitted=90195 kept=2289885 rejected=80956 last=keep reason=rewritten +[llm clean] completed=90200 submitted=90295 kept=2289978 rejected=80963 last=keep reason=rewritten +[llm clean] completed=90300 submitted=90395 kept=2290076 rejected=80965 last=keep reason=rewritten +[llm clean] completed=90400 submitted=90495 kept=2290175 rejected=80966 last=keep reason=rewritten +[llm clean] completed=90500 submitted=90595 kept=2290275 rejected=80966 last=keep reason=rewritten +[llm clean] completed=90600 submitted=90695 kept=2290372 rejected=80969 last=keep reason=rewritten +[llm clean] completed=90700 submitted=90795 kept=2290469 rejected=80972 last=reject reason=The text is a raw alphabetical list of song titles, likely from a discography or tracklist, with minimal context. It doe +[llm clean] completed=90800 submitted=90895 kept=2290559 rejected=80982 last=keep reason=rewritten +[llm clean] completed=90900 submitted=90995 kept=2290659 rejected=80982 last=keep reason=rewritten +[llm clean] completed=91000 submitted=91095 kept=2290757 rejected=80984 last=keep reason=rewritten +[llm clean] completed=91100 submitted=91195 kept=2290857 rejected=80984 last=keep reason=rewritten +[llm clean] completed=91200 submitted=91295 kept=2290953 rejected=80988 last=keep reason=rewritten +[llm clean] completed=91300 submitted=91395 kept=2291051 rejected=80990 last=keep reason=rewritten +[llm clean] completed=91400 submitted=91495 kept=2291149 rejected=80992 last=reject reason=The text is primarily LaTeX source code and boilerplate instructions for a document editor (Overleaf), interspersed with +[llm clean] completed=91500 submitted=91595 kept=2291248 rejected=80993 last=keep reason=rewritten +[llm clean] completed=91600 submitted=91695 kept=2291345 rejected=80996 last=keep reason=rewritten +[llm clean] completed=91700 submitted=91795 kept=2291437 rejected=81004 last=keep reason=rewritten +[llm clean] completed=91800 submitted=91895 kept=2291533 rejected=81008 last=keep reason=rewritten +[llm clean] completed=91900 submitted=91995 kept=2291631 rejected=81010 last=keep reason=rewritten +[llm clean] completed=92000 submitted=92095 kept=2291731 rejected=81010 last=keep reason=rewritten +[llm clean] completed=92100 submitted=92195 kept=2291825 rejected=81016 last=keep reason=rewritten +[llm clean] completed=92200 submitted=92295 kept=2291923 rejected=81018 last=keep reason=rewritten +[llm clean] completed=92300 submitted=92395 kept=2292021 rejected=81020 last=keep reason=rewritten +[llm clean] completed=92400 submitted=92495 kept=2292119 rejected=81022 last=keep reason=rewritten +[llm clean] completed=92500 submitted=92595 kept=2292218 rejected=81023 last=keep reason=rewritten +[llm clean] completed=92600 submitted=92695 kept=2292315 rejected=81026 last=keep reason=rewritten +[llm clean] completed=92700 submitted=92795 kept=2292413 rejected=81028 last=keep reason=rewritten +[llm clean] completed=92800 submitted=92895 kept=2292513 rejected=81028 last=keep reason=rewritten +[llm clean] completed=92900 submitted=92995 kept=2292601 rejected=81040 last=keep reason=rewritten +[llm clean] completed=93000 submitted=93095 kept=2292697 rejected=81044 last=keep reason=rewritten +[llm clean] completed=93100 submitted=93195 kept=2292796 rejected=81045 last=keep reason=rewritten +[llm clean] completed=93200 submitted=93295 kept=2292894 rejected=81047 last=keep reason=rewritten +[llm clean] completed=93300 submitted=93395 kept=2292990 rejected=81051 last=keep reason=rewritten +[llm clean] completed=93400 submitted=93495 kept=2293070 rejected=81071 last=keep reason=rewritten +[llm clean] completed=93500 submitted=93595 kept=2293169 rejected=81072 last=keep reason=rewritten +[llm clean] completed=93600 submitted=93695 kept=2293262 rejected=81079 last=keep reason=rewritten +[llm clean] completed=93700 submitted=93795 kept=2293359 rejected=81082 last=keep reason=rewritten +[llm clean] completed=93800 submitted=93895 kept=2293456 rejected=81085 last=keep reason=rewritten +[llm clean] completed=93900 submitted=93995 kept=2293551 rejected=81090 last=keep reason=rewritten +[llm clean] completed=94000 submitted=94095 kept=2293646 rejected=81095 last=reject reason=The text is a collection of unrelated fragments, including spam disclaimers, promotional content for a bodybuilding site +[llm clean] completed=94100 submitted=94195 kept=2293742 rejected=81099 last=keep reason=rewritten +[llm clean] completed=94200 submitted=94295 kept=2293833 rejected=81108 last=keep reason=rewritten +[llm clean] completed=94300 submitted=94395 kept=2293929 rejected=81112 last=keep reason=rewritten +[llm clean] completed=94400 submitted=94495 kept=2294028 rejected=81113 last=keep reason=rewritten +[llm clean] completed=94500 submitted=94595 kept=2294123 rejected=81118 last=keep reason=rewritten +[llm clean] completed=94600 submitted=94695 kept=2294215 rejected=81126 last=keep reason=rewritten +[llm clean] completed=94700 submitted=94795 kept=2294314 rejected=81127 last=keep reason=rewritten +[llm clean] completed=94800 submitted=94895 kept=2294411 rejected=81130 last=keep reason=rewritten +[llm clean] completed=94900 submitted=94995 kept=2294508 rejected=81133 last=keep reason=rewritten +[llm clean] completed=95000 submitted=95095 kept=2294603 rejected=81138 last=keep reason=rewritten +[llm clean] completed=95100 submitted=95195 kept=2294702 rejected=81139 last=keep reason=rewritten +[llm clean] completed=95200 submitted=95295 kept=2294799 rejected=81142 last=keep reason=rewritten +[llm clean] completed=95300 submitted=95395 kept=2294888 rejected=81153 last=keep reason=rewritten +[llm clean] completed=95400 submitted=95495 kept=2294982 rejected=81159 last=reject reason=The text is a highly biased, politically charged opinion piece promoting a specific political party (Golden Dawn) and co +[llm clean] completed=95500 submitted=95595 kept=2295081 rejected=81160 last=keep reason=rewritten +[llm clean] completed=95600 submitted=95695 kept=2295181 rejected=81160 last=keep reason=rewritten +[llm clean] completed=95700 submitted=95795 kept=2295280 rejected=81161 last=keep reason=rewritten +[llm clean] completed=95800 submitted=95895 kept=2295364 rejected=81177 last=keep reason=rewritten +[llm clean] completed=95900 submitted=95995 kept=2295464 rejected=81177 last=keep reason=rewritten +[llm clean] completed=96000 submitted=96095 kept=2295563 rejected=81178 last=keep reason=rewritten +[llm clean] completed=96100 submitted=96195 kept=2295662 rejected=81179 last=keep reason=rewritten +[llm clean] completed=96200 submitted=96295 kept=2295760 rejected=81181 last=keep reason=rewritten +[llm clean] completed=96300 submitted=96395 kept=2295858 rejected=81183 last=keep reason=rewritten +[llm clean] completed=96400 submitted=96495 kept=2295957 rejected=81184 last=keep reason=rewritten +[llm clean] completed=96500 submitted=96595 kept=2296053 rejected=81188 last=keep reason=rewritten +[llm clean] completed=96600 submitted=96695 kept=2296151 rejected=81190 last=keep reason=rewritten +[llm clean] completed=96700 submitted=96795 kept=2296225 rejected=81216 last=reject reason= +[llm clean] completed=96800 submitted=96895 kept=2296322 rejected=81219 last=keep reason=rewritten +[llm clean] completed=96900 submitted=96995 kept=2296421 rejected=81220 last=keep reason=rewritten +[llm clean] completed=97000 submitted=97095 kept=2296521 rejected=81220 last=keep reason=rewritten +[llm clean] completed=97100 submitted=97195 kept=2296619 rejected=81222 last=keep reason=rewritten +[llm clean] completed=97200 submitted=97295 kept=2296717 rejected=81224 last=keep reason=rewritten +[llm clean] completed=97300 submitted=97395 kept=2296812 rejected=81229 last=keep reason=rewritten +[llm clean] completed=97400 submitted=97495 kept=2296911 rejected=81230 last=keep reason=rewritten +[llm clean] completed=97500 submitted=97595 kept=2297007 rejected=81234 last=keep reason=rewritten +[llm clean] completed=97600 submitted=97695 kept=2297101 rejected=81240 last=keep reason=rewritten +[llm clean] completed=97700 submitted=97795 kept=2297198 rejected=81243 last=keep reason=rewritten +[llm clean] completed=97800 submitted=97895 kept=2297292 rejected=81249 last=keep reason=rewritten +[llm clean] completed=97900 submitted=97995 kept=2297391 rejected=81250 last=keep reason=rewritten +[llm clean] completed=98000 submitted=98095 kept=2297489 rejected=81252 last=keep reason=rewritten +[llm clean] completed=98100 submitted=98195 kept=2297588 rejected=81253 last=keep reason=rewritten +[llm clean] completed=98200 submitted=98295 kept=2297685 rejected=81256 last=keep reason=rewritten +[llm clean] completed=98300 submitted=98395 kept=2297764 rejected=81277 last=keep reason=rewritten +[llm clean] completed=98400 submitted=98495 kept=2297861 rejected=81280 last=keep reason=rewritten +[llm clean] completed=98500 submitted=98595 kept=2297958 rejected=81283 last=keep reason=rewritten +[llm clean] completed=98600 submitted=98695 kept=2298051 rejected=81290 last=keep reason=rewritten +[llm clean] completed=98700 submitted=98795 kept=2298147 rejected=81294 last=keep reason=rewritten +[llm clean] completed=98800 submitted=98895 kept=2298245 rejected=81296 last=keep reason=rewritten +[llm clean] completed=98900 submitted=98995 kept=2298344 rejected=81297 last=keep reason=rewritten +[llm clean] completed=99000 submitted=99095 kept=2298443 rejected=81298 last=keep reason=rewritten +[llm clean] completed=99100 submitted=99195 kept=2298538 rejected=81303 last=keep reason=rewritten +[llm clean] completed=99200 submitted=99295 kept=2298635 rejected=81306 last=keep reason=rewritten +[llm clean] completed=99300 submitted=99395 kept=2298733 rejected=81308 last=keep reason=rewritten +[llm clean] completed=99400 submitted=99495 kept=2298832 rejected=81309 last=keep reason=rewritten +[llm clean] completed=99500 submitted=99595 kept=2298930 rejected=81311 last=keep reason=rewritten +[llm clean] completed=99600 submitted=99695 kept=2299028 rejected=81313 last=keep reason=rewritten +[llm clean] completed=99700 submitted=99795 kept=2299125 rejected=81316 last=keep reason=rewritten +[llm clean] completed=99800 submitted=99895 kept=2299223 rejected=81318 last=keep reason=rewritten +[llm clean] completed=99900 submitted=99995 kept=2299322 rejected=81319 last=keep reason=rewritten +[llm clean] completed=100000 submitted=100095 kept=2299421 rejected=81320 last=keep reason=rewritten +[llm clean] completed=100100 submitted=100195 kept=2299516 rejected=81325 last=keep reason=rewritten +[llm clean] completed=100200 submitted=100295 kept=2299616 rejected=81325 last=keep reason=rewritten +[llm clean] completed=100300 submitted=100395 kept=2299712 rejected=81329 last=keep reason=rewritten +[llm clean] completed=100400 submitted=100495 kept=2299808 rejected=81333 last=keep reason=rewritten +[llm clean] completed=100500 submitted=100595 kept=2299908 rejected=81333 last=keep reason=rewritten +[llm clean] completed=100600 submitted=100695 kept=2299995 rejected=81346 last=keep reason=rewritten +[llm clean] completed=100700 submitted=100795 kept=2300084 rejected=81357 last=reject reason= +[llm clean] completed=100800 submitted=100895 kept=2300163 rejected=81378 last=keep reason=rewritten +[llm clean] completed=100900 submitted=100995 kept=2300259 rejected=81382 last=keep reason=rewritten +[llm clean] completed=101000 submitted=101095 kept=2300359 rejected=81382 last=keep reason=rewritten +[llm clean] completed=101100 submitted=101195 kept=2300457 rejected=81384 last=keep reason=rewritten +[llm clean] completed=101200 submitted=101295 kept=2300556 rejected=81385 last=keep reason=rewritten +[llm clean] completed=101300 submitted=101395 kept=2300653 rejected=81388 last=keep reason=rewritten +[llm clean] completed=101400 submitted=101495 kept=2300752 rejected=81389 last=keep reason=rewritten +[llm clean] completed=101500 submitted=101595 kept=2300849 rejected=81392 last=keep reason=rewritten +[llm clean] completed=101600 submitted=101695 kept=2300945 rejected=81396 last=keep reason=rewritten +[llm clean] completed=101700 submitted=101795 kept=2301043 rejected=81398 last=keep reason=rewritten +[llm clean] completed=101800 submitted=101895 kept=2301143 rejected=81398 last=keep reason=rewritten +[llm clean] completed=101900 submitted=101995 kept=2301240 rejected=81401 last=keep reason=rewritten +[llm clean] completed=102000 submitted=102095 kept=2301339 rejected=81402 last=keep reason=rewritten +[llm clean] completed=102100 submitted=102195 kept=2301435 rejected=81406 last=keep reason=rewritten +[llm clean] completed=102200 submitted=102295 kept=2301534 rejected=81407 last=keep reason=rewritten +[llm clean] completed=102300 submitted=102395 kept=2301634 rejected=81407 last=keep reason=rewritten +[llm clean] completed=102400 submitted=102495 kept=2301733 rejected=81408 last=keep reason=rewritten +[llm clean] completed=102500 submitted=102595 kept=2301832 rejected=81409 last=keep reason=rewritten +[llm clean] completed=102600 submitted=102695 kept=2301925 rejected=81416 last=keep reason=rewritten +[llm clean] completed=102700 submitted=102795 kept=2302024 rejected=81417 last=keep reason=rewritten +[llm clean] completed=102800 submitted=102895 kept=2302123 rejected=81418 last=keep reason=rewritten +[llm clean] completed=102900 submitted=102995 kept=2302219 rejected=81422 last=keep reason=rewritten +[llm clean] completed=103000 submitted=103095 kept=2302316 rejected=81425 last=keep reason=rewritten +[llm clean] completed=103100 submitted=103195 kept=2302412 rejected=81429 last=keep reason=rewritten +[llm clean] completed=103200 submitted=103295 kept=2302505 rejected=81436 last=keep reason=rewritten +[llm clean] completed=103300 submitted=103395 kept=2302603 rejected=81438 last=keep reason=rewritten +[llm clean] completed=103400 submitted=103495 kept=2302698 rejected=81443 last=keep reason=rewritten +[llm clean] completed=103500 submitted=103595 kept=2302798 rejected=81443 last=keep reason=rewritten +[llm clean] completed=103600 submitted=103695 kept=2302894 rejected=81447 last=keep reason=rewritten +[llm clean] completed=103700 submitted=103795 kept=2302982 rejected=81459 last=reject reason=The input is a list of names, birth years, and professions (a roster of entries), not coherent article-style prose. +[llm clean] completed=103800 submitted=103895 kept=2303062 rejected=81479 last=keep reason=rewritten +[llm clean] completed=103900 submitted=103995 kept=2303156 rejected=81485 last=keep reason=rewritten +[llm clean] completed=104000 submitted=104095 kept=2303256 rejected=81485 last=keep reason=rewritten +[llm clean] completed=104100 submitted=104195 kept=2303352 rejected=81489 last=keep reason=rewritten +[llm clean] completed=104200 submitted=104295 kept=2303449 rejected=81492 last=keep reason=rewritten +[llm clean] completed=104300 submitted=104395 kept=2303547 rejected=81494 last=keep reason=rewritten +[llm clean] completed=104400 submitted=104495 kept=2303646 rejected=81495 last=keep reason=rewritten +[llm clean] completed=104500 submitted=104595 kept=2303740 rejected=81501 last=keep reason=rewritten +[llm clean] completed=104600 submitted=104695 kept=2303834 rejected=81507 last=keep reason=rewritten +[llm clean] completed=104700 submitted=104795 kept=2303933 rejected=81508 last=keep reason=rewritten +[llm clean] completed=104800 submitted=104895 kept=2304030 rejected=81511 last=keep reason=rewritten +[llm clean] completed=104900 submitted=104995 kept=2304130 rejected=81511 last=keep reason=rewritten +[llm clean] completed=105000 submitted=105095 kept=2304230 rejected=81511 last=keep reason=rewritten +[llm clean] completed=105100 submitted=105195 kept=2304322 rejected=81519 last=keep reason=rewritten +[llm clean] completed=105200 submitted=105295 kept=2304422 rejected=81519 last=keep reason=rewritten +[llm clean] completed=105300 submitted=105395 kept=2304517 rejected=81524 last=keep reason=rewritten +[llm clean] completed=105400 submitted=105495 kept=2304614 rejected=81527 last=keep reason=rewritten +[llm clean] completed=105500 submitted=105595 kept=2304708 rejected=81533 last=keep reason=rewritten +[llm clean] completed=105600 submitted=105695 kept=2304806 rejected=81535 last=keep reason=rewritten +[llm clean] completed=105700 submitted=105795 kept=2304905 rejected=81536 last=keep reason=rewritten +[llm clean] completed=105800 submitted=105895 kept=2305003 rejected=81538 last=keep reason=rewritten +[llm clean] completed=105900 submitted=105995 kept=2305101 rejected=81540 last=keep reason=rewritten +[llm clean] completed=106000 submitted=106095 kept=2305197 rejected=81544 last=keep reason=rewritten +[llm clean] completed=106100 submitted=106195 kept=2305291 rejected=81550 last=reject reason=The text is a collection of incoherent conspiracy theories, forum metadata, usernames, and low-context fragments that ca +[llm clean] completed=106200 submitted=106295 kept=2305385 rejected=81556 last=keep reason=rewritten +[llm clean] completed=106300 submitted=106395 kept=2305483 rejected=81558 last=keep reason=rewritten +[llm clean] completed=106400 submitted=106495 kept=2305581 rejected=81560 last=keep reason=rewritten +[llm clean] completed=106500 submitted=106595 kept=2305678 rejected=81563 last=keep reason=rewritten +[llm clean] completed=106600 submitted=106695 kept=2305777 rejected=81564 last=keep reason=rewritten +[llm clean] completed=106700 submitted=106795 kept=2305876 rejected=81565 last=keep reason=rewritten +[llm clean] completed=106800 submitted=106895 kept=2305976 rejected=81565 last=keep reason=rewritten +[llm clean] completed=106900 submitted=106995 kept=2306076 rejected=81565 last=keep reason=rewritten +[llm clean] completed=107000 submitted=107095 kept=2306174 rejected=81567 last=keep reason=rewritten +[llm clean] completed=107100 submitted=107195 kept=2306272 rejected=81569 last=keep reason=rewritten +[llm clean] completed=107200 submitted=107295 kept=2306371 rejected=81570 last=keep reason=rewritten +[llm clean] completed=107300 submitted=107395 kept=2306469 rejected=81572 last=keep reason=rewritten +[llm clean] completed=107400 submitted=107495 kept=2306569 rejected=81572 last=keep reason=rewritten +[llm clean] completed=107500 submitted=107595 kept=2306666 rejected=81575 last=keep reason=rewritten +[llm clean] completed=107600 submitted=107695 kept=2306764 rejected=81577 last=keep reason=rewritten +[llm clean] completed=107700 submitted=107795 kept=2306855 rejected=81586 last=keep reason=rewritten +[llm clean] completed=107800 submitted=107895 kept=2306951 rejected=81590 last=keep reason=rewritten +[llm clean] completed=107900 submitted=107995 kept=2307042 rejected=81599 last=keep reason=rewritten +[llm clean] completed=108000 submitted=108095 kept=2307141 rejected=81600 last=keep reason=rewritten +[llm clean] completed=108100 submitted=108195 kept=2307240 rejected=81601 last=reject reason=The text is a raw chat log consisting of short, fragmented messages, timestamps, and usernames. It lacks the coherent st +[llm clean] completed=108200 submitted=108295 kept=2307334 rejected=81607 last=keep reason=rewritten +[llm clean] completed=108300 submitted=108395 kept=2307423 rejected=81618 last=keep reason=rewritten +[llm clean] completed=108400 submitted=108495 kept=2307510 rejected=81631 last=keep reason=rewritten +[llm clean] completed=108500 submitted=108595 kept=2307609 rejected=81632 last=keep reason=rewritten +[llm clean] completed=108600 submitted=108695 kept=2307707 rejected=81634 last=keep reason=rewritten +[llm clean] completed=108700 submitted=108795 kept=2307805 rejected=81636 last=keep reason=rewritten +[llm clean] completed=108800 submitted=108895 kept=2307904 rejected=81637 last=keep reason=rewritten +[llm clean] completed=108900 submitted=108995 kept=2308004 rejected=81637 last=keep reason=rewritten +[llm clean] completed=109000 submitted=109095 kept=2308101 rejected=81640 last=keep reason=rewritten +[llm clean] completed=109100 submitted=109195 kept=2308193 rejected=81648 last=reject reason=The text consists primarily of raw Nmap scan output logs and a brief, contextless comment. It lacks coherent article-sty +[llm clean] completed=109200 submitted=109295 kept=2308284 rejected=81657 last=keep reason=rewritten +[llm clean] completed=109300 submitted=109395 kept=2308384 rejected=81657 last=keep reason=rewritten +[llm clean] completed=109400 submitted=109495 kept=2308480 rejected=81661 last=keep reason=rewritten +[llm clean] completed=109500 submitted=109595 kept=2308579 rejected=81662 last=keep reason=rewritten +[llm clean] completed=109600 submitted=109695 kept=2308674 rejected=81667 last=keep reason=rewritten +[llm clean] completed=109700 submitted=109795 kept=2308774 rejected=81667 last=keep reason=rewritten +[llm clean] completed=109800 submitted=109895 kept=2308867 rejected=81674 last=keep reason=rewritten +[llm clean] completed=109900 submitted=109995 kept=2308964 rejected=81677 last=keep reason=rewritten +[llm clean] completed=110000 submitted=110095 kept=2309064 rejected=81677 last=keep reason=rewritten +[llm clean] completed=110100 submitted=110195 kept=2309161 rejected=81680 last=keep reason=rewritten +[llm clean] completed=110200 submitted=110295 kept=2309256 rejected=81685 last=keep reason=rewritten +[llm clean] completed=110300 submitted=110395 kept=2309351 rejected=81690 last=keep reason=rewritten +[llm clean] completed=110400 submitted=110495 kept=2309421 rejected=81720 last=keep reason=rewritten +[llm clean] completed=110500 submitted=110595 kept=2309514 rejected=81727 last=keep reason=rewritten +[llm clean] completed=110600 submitted=110695 kept=2309606 rejected=81735 last=keep reason=rewritten +[llm clean] completed=110700 submitted=110795 kept=2309705 rejected=81736 last=keep reason=rewritten +[llm clean] completed=110800 submitted=110895 kept=2309805 rejected=81736 last=keep reason=rewritten +[llm clean] completed=110900 submitted=110995 kept=2309901 rejected=81740 last=keep reason=rewritten +[llm clean] completed=111000 submitted=111095 kept=2310001 rejected=81740 last=keep reason=rewritten +[llm clean] completed=111100 submitted=111195 kept=2310098 rejected=81743 last=keep reason=rewritten +[llm clean] completed=111200 submitted=111295 kept=2310194 rejected=81747 last=keep reason=rewritten +[llm clean] completed=111300 submitted=111395 kept=2310291 rejected=81750 last=keep reason=rewritten +[llm clean] completed=111400 submitted=111495 kept=2310385 rejected=81756 last=keep reason=rewritten +[llm clean] completed=111500 submitted=111595 kept=2310480 rejected=81761 last=keep reason=rewritten +[llm clean] completed=111600 submitted=111695 kept=2310578 rejected=81763 last=keep reason=rewritten +[llm clean] completed=111700 submitted=111795 kept=2310677 rejected=81764 last=keep reason=rewritten +[llm clean] completed=111800 submitted=111895 kept=2310775 rejected=81766 last=keep reason=rewritten +[llm clean] completed=111900 submitted=111995 kept=2310872 rejected=81769 last=keep reason=rewritten +[llm clean] completed=112000 submitted=112095 kept=2310971 rejected=81770 last=keep reason=rewritten +[llm clean] completed=112100 submitted=112195 kept=2311068 rejected=81773 last=keep reason=rewritten +[llm clean] completed=112200 submitted=112295 kept=2311165 rejected=81776 last=keep reason=rewritten +[llm clean] completed=112300 submitted=112395 kept=2311264 rejected=81777 last=keep reason=rewritten +[llm clean] completed=112400 submitted=112495 kept=2311364 rejected=81777 last=keep reason=rewritten +[llm clean] completed=112500 submitted=112595 kept=2311462 rejected=81779 last=reject reason=The text is a highly biased, pseudoscientific, and incoherent rant using dehumanizing language ('dudegoo', 'assassin-lev +[llm clean] completed=112600 submitted=112695 kept=2311558 rejected=81783 last=keep reason=rewritten +[llm clean] completed=112700 submitted=112795 kept=2311658 rejected=81783 last=keep reason=rewritten +[llm clean] completed=112800 submitted=112895 kept=2311755 rejected=81786 last=keep reason=rewritten +[llm clean] completed=112900 submitted=112995 kept=2311855 rejected=81786 last=keep reason=rewritten +[llm clean] completed=113000 submitted=113095 kept=2311954 rejected=81787 last=keep reason=rewritten +[llm clean] completed=113100 submitted=113195 kept=2312053 rejected=81788 last=keep reason=rewritten +[llm clean] completed=113200 submitted=113295 kept=2312150 rejected=81791 last=keep reason=rewritten +[llm clean] completed=113300 submitted=113395 kept=2312246 rejected=81795 last=reject reason=JSONDecodeError('Unterminated string starting at: line 4 column 5367 (char 5409)') +[llm clean] completed=113400 submitted=113495 kept=2312346 rejected=81795 last=keep reason=rewritten +[llm clean] completed=113500 submitted=113595 kept=2312444 rejected=81797 last=keep reason=rewritten +[llm clean] completed=113600 submitted=113695 kept=2312541 rejected=81800 last=keep reason=rewritten +[llm clean] completed=113700 submitted=113794 kept=2312635 rejected=81806 last=keep reason=rewritten +[llm clean] completed=113800 submitted=113895 kept=2312721 rejected=81820 last=keep reason=rewritten +[llm clean] completed=113900 submitted=113995 kept=2312816 rejected=81825 last=keep reason=rewritten +[llm clean] completed=114000 submitted=114095 kept=2312916 rejected=81825 last=keep reason=rewritten +[llm clean] completed=114100 submitted=114195 kept=2313016 rejected=81825 last=keep reason=rewritten +[llm clean] completed=114200 submitted=114295 kept=2313108 rejected=81833 last=keep reason=rewritten +[llm clean] completed=114300 submitted=114395 kept=2313207 rejected=81834 last=keep reason=rewritten +[llm clean] completed=114400 submitted=114495 kept=2313305 rejected=81836 last=keep reason=rewritten +[llm clean] completed=114500 submitted=114595 kept=2313402 rejected=81839 last=keep reason=rewritten +[llm clean] completed=114600 submitted=114695 kept=2313499 rejected=81842 last=keep reason=rewritten +[llm clean] completed=114700 submitted=114795 kept=2313595 rejected=81846 last=keep reason=rewritten +[llm clean] completed=114800 submitted=114895 kept=2313692 rejected=81849 last=keep reason=rewritten +[llm clean] completed=114900 submitted=114995 kept=2313791 rejected=81850 last=keep reason=rewritten +[llm clean] completed=115000 submitted=115095 kept=2313890 rejected=81851 last=keep reason=rewritten +[llm clean] completed=115100 submitted=115195 kept=2313986 rejected=81855 last=keep reason=rewritten +[llm clean] completed=115200 submitted=115295 kept=2314078 rejected=81863 last=keep reason=rewritten +[llm clean] completed=115300 submitted=115395 kept=2314173 rejected=81868 last=keep reason=rewritten +[llm clean] completed=115400 submitted=115495 kept=2314271 rejected=81870 last=keep reason=rewritten +[llm clean] completed=115500 submitted=115595 kept=2314371 rejected=81870 last=keep reason=rewritten +[llm clean] completed=115600 submitted=115695 kept=2314459 rejected=81882 last=keep reason=rewritten +[llm clean] completed=115700 submitted=115795 kept=2314554 rejected=81887 last=keep reason=rewritten +[llm clean] completed=115800 submitted=115895 kept=2314652 rejected=81889 last=keep reason=rewritten +[llm clean] completed=115900 submitted=115995 kept=2314750 rejected=81891 last=keep reason=rewritten +[llm clean] completed=116000 submitted=116095 kept=2314846 rejected=81895 last=keep reason=rewritten +[llm clean] completed=116100 submitted=116195 kept=2314942 rejected=81899 last=keep reason=rewritten +[llm clean] completed=116200 submitted=116295 kept=2315038 rejected=81903 last=keep reason=rewritten +[llm clean] completed=116300 submitted=116395 kept=2315138 rejected=81903 last=keep reason=rewritten +[llm clean] completed=116400 submitted=116495 kept=2315223 rejected=81918 last=keep reason=rewritten +[llm clean] completed=116500 submitted=116595 kept=2315323 rejected=81918 last=keep reason=rewritten +[llm clean] completed=116600 submitted=116695 kept=2315419 rejected=81922 last=keep reason=rewritten +[llm clean] completed=116700 submitted=116795 kept=2315517 rejected=81924 last=keep reason=rewritten +[llm clean] completed=116800 submitted=116895 kept=2315617 rejected=81924 last=keep reason=rewritten +[llm clean] completed=116900 submitted=116995 kept=2315712 rejected=81929 last=keep reason=rewritten +[llm clean] completed=117000 submitted=117095 kept=2315812 rejected=81929 last=keep reason=rewritten +[llm clean] completed=117100 submitted=117195 kept=2315912 rejected=81929 last=keep reason=rewritten +[llm clean] completed=117200 submitted=117295 kept=2316009 rejected=81932 last=keep reason=rewritten +[llm clean] completed=117300 submitted=117395 kept=2316108 rejected=81933 last=keep reason=rewritten +[llm clean] completed=117400 submitted=117495 kept=2316208 rejected=81933 last=keep reason=rewritten +[llm clean] completed=117500 submitted=117595 kept=2316307 rejected=81934 last=keep reason=rewritten +[llm clean] completed=117600 submitted=117695 kept=2316406 rejected=81935 last=keep reason=rewritten +[llm clean] completed=117700 submitted=117795 kept=2316477 rejected=81964 last=keep reason=rewritten +[llm clean] completed=117800 submitted=117895 kept=2316569 rejected=81972 last=keep reason=rewritten +[llm clean] completed=117900 submitted=117995 kept=2316656 rejected=81985 last=keep reason=rewritten +[llm clean] completed=118000 submitted=118095 kept=2316755 rejected=81986 last=keep reason=rewritten +[llm clean] completed=118100 submitted=118195 kept=2316852 rejected=81989 last=keep reason=rewritten +[llm clean] completed=118200 submitted=118295 kept=2316951 rejected=81990 last=keep reason=rewritten +[llm clean] completed=118300 submitted=118395 kept=2317047 rejected=81994 last=keep reason=rewritten +[llm clean] completed=118400 submitted=118495 kept=2317144 rejected=81997 last=keep reason=rewritten +[llm clean] completed=118500 submitted=118595 kept=2317233 rejected=82008 last=reject reason=The text is a fragmented collection of hockey statistics, player notes, and transaction logs (e.g., 'CHI (0-1-0):', 'Wai +[llm clean] completed=118600 submitted=118695 kept=2317328 rejected=82013 last=keep reason=rewritten +[llm clean] completed=118700 submitted=118795 kept=2317424 rejected=82017 last=keep reason=rewritten +[llm clean] completed=118800 submitted=118895 kept=2317523 rejected=82018 last=keep reason=rewritten +[llm clean] completed=118900 submitted=118995 kept=2317622 rejected=82019 last=keep reason=rewritten +[llm clean] completed=119000 submitted=119095 kept=2317719 rejected=82022 last=keep reason=rewritten +[llm clean] completed=119100 submitted=119195 kept=2317818 rejected=82023 last=keep reason=rewritten +[llm clean] completed=119200 submitted=119295 kept=2317916 rejected=82025 last=keep reason=rewritten +[llm clean] completed=119300 submitted=119395 kept=2318014 rejected=82027 last=keep reason=rewritten +[llm clean] completed=119400 submitted=119495 kept=2318114 rejected=82027 last=keep reason=rewritten +[llm clean] completed=119500 submitted=119595 kept=2318214 rejected=82027 last=keep reason=rewritten +[llm clean] completed=119600 submitted=119695 kept=2318314 rejected=82027 last=keep reason=rewritten +[llm clean] completed=119700 submitted=119795 kept=2318408 rejected=82033 last=keep reason=rewritten +[llm clean] completed=119800 submitted=119895 kept=2318502 rejected=82039 last=keep reason=rewritten +[llm clean] completed=119900 submitted=119995 kept=2318597 rejected=82044 last=keep reason=rewritten +[llm clean] completed=120000 submitted=120095 kept=2318692 rejected=82049 last=keep reason=rewritten +[llm clean] completed=120100 submitted=120195 kept=2318788 rejected=82053 last=keep reason=rewritten +[llm clean] completed=120200 submitted=120295 kept=2318885 rejected=82056 last=keep reason=rewritten +[llm clean] completed=120300 submitted=120395 kept=2318979 rejected=82062 last=reject reason=The text is a raw setlist and audio player metadata with no coherent prose, narrative, or meaningful context to rewrite +[llm clean] completed=120400 submitted=120495 kept=2319077 rejected=82064 last=keep reason=rewritten +[llm clean] completed=120500 submitted=120595 kept=2319171 rejected=82070 last=keep reason=rewritten +[llm clean] completed=120600 submitted=120695 kept=2319266 rejected=82075 last=keep reason=rewritten +[llm clean] completed=120700 submitted=120795 kept=2319362 rejected=82079 last=keep reason=rewritten +[llm clean] completed=120800 submitted=120895 kept=2319460 rejected=82081 last=keep reason=rewritten +[llm clean] completed=120900 submitted=120995 kept=2319557 rejected=82084 last=keep reason=rewritten +[llm clean] completed=121000 submitted=121095 kept=2319656 rejected=82085 last=keep reason=rewritten +[llm clean] completed=121100 submitted=121195 kept=2319755 rejected=82086 last=keep reason=rewritten +[llm clean] completed=121200 submitted=121295 kept=2319853 rejected=82088 last=keep reason=rewritten +[llm clean] completed=121300 submitted=121395 kept=2319951 rejected=82090 last=keep reason=rewritten +[llm clean] completed=121400 submitted=121495 kept=2320050 rejected=82091 last=keep reason=rewritten +[llm clean] completed=121500 submitted=121595 kept=2320145 rejected=82096 last=keep reason=rewritten +[llm clean] completed=121600 submitted=121695 kept=2320245 rejected=82096 last=keep reason=rewritten +[llm clean] completed=121700 submitted=121795 kept=2320334 rejected=82107 last=keep reason=rewritten +[llm clean] completed=121800 submitted=121895 kept=2320433 rejected=82108 last=keep reason=rewritten +[llm clean] completed=121900 submitted=121995 kept=2320531 rejected=82110 last=keep reason=rewritten +[llm clean] completed=122000 submitted=122095 kept=2320628 rejected=82113 last=keep reason=rewritten +[llm clean] completed=122100 submitted=122195 kept=2320720 rejected=82121 last=keep reason=rewritten +[llm clean] completed=122200 submitted=122295 kept=2320818 rejected=82123 last=keep reason=rewritten +[llm clean] completed=122300 submitted=122395 kept=2320913 rejected=82128 last=keep reason=rewritten +[llm clean] completed=122400 submitted=122495 kept=2321011 rejected=82130 last=keep reason=rewritten +[llm clean] completed=122500 submitted=122595 kept=2321107 rejected=82134 last=keep reason=rewritten +[llm clean] completed=122600 submitted=122695 kept=2321202 rejected=82139 last=keep reason=rewritten +[llm clean] completed=122700 submitted=122795 kept=2321297 rejected=82144 last=keep reason=rewritten +[llm clean] completed=122800 submitted=122895 kept=2321395 rejected=82146 last=keep reason=rewritten +[llm clean] completed=122900 submitted=122995 kept=2321494 rejected=82147 last=keep reason=rewritten +[llm clean] completed=123000 submitted=123095 kept=2321592 rejected=82149 last=keep reason=rewritten +[llm clean] completed=123100 submitted=123195 kept=2321691 rejected=82150 last=keep reason=rewritten +[llm clean] completed=123200 submitted=123295 kept=2321789 rejected=82152 last=keep reason=rewritten +[llm clean] completed=123300 submitted=123395 kept=2321886 rejected=82155 last=keep reason=rewritten +[llm clean] completed=123400 submitted=123495 kept=2321984 rejected=82157 last=keep reason=rewritten +[llm clean] completed=123500 submitted=123595 kept=2322080 rejected=82161 last=keep reason=rewritten +[llm clean] completed=123600 submitted=123695 kept=2322179 rejected=82162 last=keep reason=rewritten +[llm clean] completed=123700 submitted=123795 kept=2322276 rejected=82165 last=keep reason=rewritten +[llm clean] completed=123800 submitted=123895 kept=2322370 rejected=82171 last=keep reason=rewritten +[llm clean] completed=123900 submitted=123995 kept=2322468 rejected=82173 last=keep reason=rewritten +[llm clean] completed=124000 submitted=124095 kept=2322536 rejected=82205 last=keep reason=rewritten +[llm clean] completed=124100 submitted=124195 kept=2322630 rejected=82211 last=keep reason=rewritten +[llm clean] completed=124200 submitted=124295 kept=2322728 rejected=82213 last=keep reason=rewritten +[llm clean] completed=124300 submitted=124395 kept=2322826 rejected=82215 last=keep reason=rewritten +[llm clean] completed=124400 submitted=124495 kept=2322923 rejected=82218 last=keep reason=rewritten +[llm clean] completed=124500 submitted=124595 kept=2323021 rejected=82220 last=keep reason=rewritten +[llm clean] completed=124600 submitted=124695 kept=2323119 rejected=82222 last=keep reason=rewritten +[llm clean] completed=124700 submitted=124795 kept=2323212 rejected=82229 last=keep reason=rewritten +[llm clean] completed=124800 submitted=124895 kept=2323311 rejected=82230 last=keep reason=rewritten +[llm clean] completed=124900 submitted=124995 kept=2323409 rejected=82232 last=keep reason=rewritten +[llm clean] completed=125000 submitted=125095 kept=2323509 rejected=82232 last=keep reason=rewritten +[llm clean] completed=125100 submitted=125195 kept=2323608 rejected=82233 last=keep reason=rewritten +[llm clean] completed=125200 submitted=125295 kept=2323707 rejected=82234 last=keep reason=rewritten +[llm clean] completed=125300 submitted=125395 kept=2323804 rejected=82237 last=keep reason=rewritten +[llm clean] completed=125400 submitted=125495 kept=2323904 rejected=82237 last=keep reason=rewritten +[llm clean] completed=125500 submitted=125595 kept=2323997 rejected=82244 last=keep reason=rewritten +[llm clean] completed=125600 submitted=125695 kept=2324094 rejected=82247 last=keep reason=rewritten +[llm clean] completed=125700 submitted=125795 kept=2324190 rejected=82251 last=keep reason=rewritten +[llm clean] completed=125800 submitted=125895 kept=2324279 rejected=82262 last=keep reason=rewritten +[llm clean] completed=125900 submitted=125995 kept=2324378 rejected=82263 last=keep reason=rewritten +[llm clean] completed=126000 submitted=126095 kept=2324477 rejected=82264 last=keep reason=rewritten +[llm clean] completed=126100 submitted=126195 kept=2324575 rejected=82266 last=keep reason=rewritten +[llm clean] completed=126200 submitted=126295 kept=2324675 rejected=82266 last=keep reason=rewritten +[llm clean] completed=126300 submitted=126395 kept=2324770 rejected=82271 last=keep reason=rewritten +[llm clean] completed=126400 submitted=126495 kept=2324865 rejected=82276 last=keep reason=rewritten +[llm clean] completed=126500 submitted=126595 kept=2324964 rejected=82277 last=keep reason=rewritten +[llm clean] completed=126600 submitted=126695 kept=2325058 rejected=82283 last=keep reason=rewritten +[llm clean] completed=126700 submitted=126795 kept=2325152 rejected=82289 last=keep reason=rewritten +[llm clean] completed=126800 submitted=126895 kept=2325241 rejected=82300 last=keep reason=rewritten +[llm clean] completed=126900 submitted=126995 kept=2325339 rejected=82302 last=keep reason=rewritten +[llm clean] completed=127000 submitted=127095 kept=2325437 rejected=82304 last=keep reason=rewritten +[llm clean] completed=127100 submitted=127195 kept=2325536 rejected=82305 last=keep reason=rewritten +[llm clean] completed=127200 submitted=127295 kept=2325636 rejected=82305 last=keep reason=rewritten +[llm clean] completed=127300 submitted=127395 kept=2325725 rejected=82316 last=keep reason=rewritten +[llm clean] completed=127400 submitted=127495 kept=2325825 rejected=82316 last=keep reason=rewritten +[llm clean] completed=127500 submitted=127595 kept=2325913 rejected=82328 last=reject reason=The text is a roster/list of hockey players and card checklist items, lacking coherent article-style prose structure. +[llm clean] completed=127600 submitted=127695 kept=2325994 rejected=82347 last=keep reason=rewritten +[llm clean] completed=127700 submitted=127795 kept=2326092 rejected=82349 last=keep reason=rewritten +[llm clean] completed=127800 submitted=127895 kept=2326189 rejected=82352 last=keep reason=rewritten +[llm clean] completed=127900 submitted=127995 kept=2326287 rejected=82354 last=keep reason=rewritten +[llm clean] completed=128000 submitted=128095 kept=2326382 rejected=82359 last=keep reason=rewritten +[llm clean] completed=128100 submitted=128195 kept=2326481 rejected=82360 last=keep reason=rewritten +[llm clean] completed=128200 submitted=128295 kept=2326575 rejected=82366 last=keep reason=rewritten +[llm clean] completed=128300 submitted=128395 kept=2326674 rejected=82367 last=keep reason=rewritten +[llm clean] completed=128400 submitted=128495 kept=2326771 rejected=82370 last=keep reason=rewritten +[llm clean] completed=128500 submitted=128595 kept=2326868 rejected=82373 last=keep reason=rewritten +[llm clean] completed=128600 submitted=128695 kept=2326963 rejected=82378 last=reject reason= +[llm clean] completed=128700 submitted=128795 kept=2327062 rejected=82379 last=keep reason=rewritten +[llm clean] completed=128800 submitted=128895 kept=2327159 rejected=82382 last=keep reason=rewritten +[llm clean] completed=128900 submitted=128995 kept=2327257 rejected=82384 last=keep reason=rewritten +[llm clean] completed=129000 submitted=129095 kept=2327355 rejected=82386 last=keep reason=rewritten +[llm clean] completed=129100 submitted=129195 kept=2327453 rejected=82388 last=keep reason=rewritten +[llm clean] completed=129200 submitted=129295 kept=2327552 rejected=82389 last=keep reason=rewritten +[llm clean] completed=129300 submitted=129395 kept=2327642 rejected=82399 last=keep reason=rewritten +[llm clean] completed=129400 submitted=129495 kept=2327741 rejected=82400 last=keep reason=rewritten +[llm clean] completed=129500 submitted=129595 kept=2327837 rejected=82404 last=keep reason=rewritten +[llm clean] completed=129600 submitted=129695 kept=2327925 rejected=82416 last=keep reason=rewritten +[llm clean] completed=129700 submitted=129795 kept=2328024 rejected=82417 last=keep reason=rewritten +[llm clean] completed=129800 submitted=129895 kept=2328124 rejected=82417 last=keep reason=rewritten +[llm clean] completed=129900 submitted=129995 kept=2328220 rejected=82421 last=keep reason=rewritten +[llm clean] completed=130000 submitted=130095 kept=2328316 rejected=82425 last=keep reason=rewritten +[llm clean] completed=130100 submitted=130195 kept=2328415 rejected=82426 last=keep reason=rewritten +[llm clean] completed=130200 submitted=130295 kept=2328511 rejected=82430 last=keep reason=rewritten +[llm clean] completed=130300 submitted=130395 kept=2328608 rejected=82433 last=keep reason=rewritten +[llm clean] completed=130400 submitted=130495 kept=2328703 rejected=82438 last=keep reason=rewritten +[llm clean] completed=130500 submitted=130595 kept=2328798 rejected=82443 last=keep reason=rewritten +[llm clean] completed=130600 submitted=130695 kept=2328892 rejected=82449 last=keep reason=rewritten +[llm clean] completed=130700 submitted=130795 kept=2328989 rejected=82452 last=keep reason=rewritten +[llm clean] completed=130800 submitted=130895 kept=2329087 rejected=82454 last=keep reason=rewritten +[llm clean] completed=130900 submitted=130995 kept=2329186 rejected=82455 last=keep reason=rewritten +[llm clean] completed=131000 submitted=131095 kept=2329286 rejected=82455 last=keep reason=rewritten +[llm clean] completed=131100 submitted=131195 kept=2329384 rejected=82457 last=keep reason=rewritten +[llm clean] completed=131200 submitted=131295 kept=2329482 rejected=82459 last=keep reason=rewritten +[llm clean] completed=131300 submitted=131395 kept=2329582 rejected=82459 last=keep reason=rewritten +[llm clean] completed=131400 submitted=131495 kept=2329678 rejected=82463 last=keep reason=rewritten +[llm clean] completed=131500 submitted=131595 kept=2329777 rejected=82464 last=keep reason=rewritten +[llm clean] completed=131600 submitted=131695 kept=2329876 rejected=82465 last=keep reason=rewritten +[llm clean] completed=131700 submitted=131795 kept=2329971 rejected=82470 last=keep reason=rewritten +[llm clean] completed=131800 submitted=131895 kept=2330070 rejected=82471 last=keep reason=rewritten +[llm clean] completed=131900 submitted=131995 kept=2330168 rejected=82473 last=keep reason=rewritten +[llm clean] completed=132000 submitted=132095 kept=2330267 rejected=82474 last=keep reason=rewritten +[llm clean] completed=132100 submitted=132195 kept=2330361 rejected=82480 last=keep reason=rewritten +[llm clean] completed=132200 submitted=132295 kept=2330459 rejected=82482 last=keep reason=rewritten +[llm clean] completed=132300 submitted=132395 kept=2330556 rejected=82485 last=keep reason=rewritten +[llm clean] completed=132400 submitted=132495 kept=2330654 rejected=82487 last=keep reason=rewritten +[llm clean] completed=132500 submitted=132595 kept=2330746 rejected=82495 last=keep reason=rewritten +[llm clean] completed=132600 submitted=132695 kept=2330846 rejected=82495 last=keep reason=rewritten +[llm clean] completed=132700 submitted=132795 kept=2330937 rejected=82504 last=keep reason=rewritten +[llm clean] completed=132800 submitted=132895 kept=2331033 rejected=82508 last=keep reason=rewritten +[llm clean] completed=132900 submitted=132995 kept=2331130 rejected=82511 last=reject reason= +[llm clean] completed=133000 submitted=133095 kept=2331229 rejected=82512 last=keep reason=rewritten +[llm clean] completed=133100 submitted=133195 kept=2331328 rejected=82513 last=keep reason=rewritten +[llm clean] completed=133200 submitted=133295 kept=2331424 rejected=82517 last=keep reason=rewritten +[llm clean] completed=133300 submitted=133395 kept=2331522 rejected=82519 last=keep reason=rewritten +[llm clean] completed=133400 submitted=133495 kept=2331622 rejected=82519 last=keep reason=rewritten +[llm clean] completed=133500 submitted=133595 kept=2331720 rejected=82521 last=keep reason=rewritten +[llm clean] completed=133600 submitted=133695 kept=2331817 rejected=82524 last=keep reason=rewritten +[llm clean] completed=133700 submitted=133795 kept=2331917 rejected=82524 last=keep reason=rewritten +[llm clean] completed=133800 submitted=133895 kept=2332016 rejected=82525 last=keep reason=rewritten +[llm clean] completed=133900 submitted=133995 kept=2332113 rejected=82528 last=keep reason=rewritten +[llm clean] completed=134000 submitted=134095 kept=2332207 rejected=82534 last=keep reason=rewritten +[llm clean] completed=134100 submitted=134195 kept=2332304 rejected=82537 last=keep reason=rewritten +[llm clean] completed=134200 submitted=134295 kept=2332403 rejected=82538 last=keep reason=rewritten +[llm clean] completed=134300 submitted=134395 kept=2332502 rejected=82539 last=keep reason=rewritten +[llm clean] completed=134400 submitted=134495 kept=2332601 rejected=82540 last=keep reason=rewritten +[llm clean] completed=134500 submitted=134595 kept=2332700 rejected=82541 last=keep reason=rewritten +[llm clean] completed=134600 submitted=134695 kept=2332798 rejected=82543 last=keep reason=rewritten +[llm clean] completed=134700 submitted=134795 kept=2332898 rejected=82543 last=keep reason=rewritten +[llm clean] completed=134800 submitted=134895 kept=2332993 rejected=82548 last=keep reason=rewritten +[llm clean] completed=134900 submitted=134995 kept=2333089 rejected=82552 last=keep reason=rewritten +[llm clean] completed=135000 submitted=135095 kept=2333187 rejected=82554 last=keep reason=rewritten +[llm clean] completed=135100 submitted=135195 kept=2333285 rejected=82556 last=keep reason=rewritten +[llm clean] completed=135200 submitted=135295 kept=2333381 rejected=82560 last=keep reason=rewritten +[llm clean] completed=135300 submitted=135395 kept=2333480 rejected=82561 last=keep reason=rewritten +[llm clean] completed=135400 submitted=135495 kept=2333577 rejected=82564 last=keep reason=rewritten +[llm clean] completed=135500 submitted=135595 kept=2333677 rejected=82564 last=keep reason=rewritten +[llm clean] completed=135600 submitted=135695 kept=2333777 rejected=82564 last=keep reason=rewritten +[llm clean] completed=135700 submitted=135795 kept=2333872 rejected=82569 last=keep reason=rewritten +[llm clean] completed=135800 submitted=135895 kept=2333971 rejected=82570 last=keep reason=rewritten +[llm clean] completed=135900 submitted=135995 kept=2334069 rejected=82572 last=keep reason=rewritten +[llm clean] completed=136000 submitted=136095 kept=2334159 rejected=82582 last=keep reason=rewritten +[llm clean] completed=136100 submitted=136195 kept=2334259 rejected=82582 last=keep reason=rewritten +[llm clean] completed=136200 submitted=136295 kept=2334358 rejected=82583 last=keep reason=rewritten +[llm clean] completed=136300 submitted=136395 kept=2334458 rejected=82583 last=keep reason=rewritten +[llm clean] completed=136400 submitted=136495 kept=2334558 rejected=82583 last=keep reason=rewritten +[llm clean] completed=136500 submitted=136595 kept=2334652 rejected=82589 last=keep reason=rewritten +[llm clean] completed=136600 submitted=136695 kept=2334744 rejected=82597 last=keep reason=rewritten +[llm clean] completed=136700 submitted=136795 kept=2334841 rejected=82600 last=keep reason=rewritten +[llm clean] completed=136800 submitted=136895 kept=2334940 rejected=82601 last=keep reason=rewritten +[llm clean] completed=136900 submitted=136995 kept=2335039 rejected=82602 last=keep reason=rewritten +[llm clean] completed=137000 submitted=137095 kept=2335138 rejected=82603 last=keep reason=rewritten +[llm clean] completed=137100 submitted=137195 kept=2335234 rejected=82607 last=keep reason=rewritten +[llm clean] completed=137200 submitted=137295 kept=2335329 rejected=82612 last=keep reason=rewritten +[llm clean] completed=137300 submitted=137395 kept=2335426 rejected=82615 last=keep reason=rewritten +[llm clean] completed=137400 submitted=137495 kept=2335522 rejected=82619 last=keep reason=rewritten +[llm clean] completed=137500 submitted=137595 kept=2335622 rejected=82619 last=keep reason=rewritten +[llm clean] completed=137600 submitted=137695 kept=2335719 rejected=82622 last=keep reason=rewritten +[llm clean] completed=137700 submitted=137795 kept=2335815 rejected=82626 last=keep reason=rewritten +[llm clean] completed=137800 submitted=137895 kept=2335913 rejected=82628 last=keep reason=rewritten +[llm clean] completed=137900 submitted=137995 kept=2336009 rejected=82632 last=keep reason=rewritten +[llm clean] completed=138000 submitted=138095 kept=2336108 rejected=82633 last=keep reason=rewritten +[llm clean] completed=138100 submitted=138195 kept=2336208 rejected=82633 last=keep reason=rewritten +[llm clean] completed=138200 submitted=138295 kept=2336308 rejected=82633 last=keep reason=rewritten +[llm clean] completed=138300 submitted=138395 kept=2336405 rejected=82636 last=keep reason=rewritten +[llm clean] completed=138400 submitted=138495 kept=2336502 rejected=82639 last=keep reason=rewritten +[llm clean] completed=138500 submitted=138595 kept=2336600 rejected=82641 last=keep reason=rewritten +[llm clean] completed=138600 submitted=138695 kept=2336698 rejected=82643 last=keep reason=rewritten +[llm clean] completed=138700 submitted=138795 kept=2336798 rejected=82643 last=keep reason=rewritten +[llm clean] completed=138800 submitted=138895 kept=2336897 rejected=82644 last=keep reason=rewritten +[llm clean] completed=138900 submitted=138995 kept=2336993 rejected=82648 last=keep reason=rewritten +[llm clean] completed=139000 submitted=139095 kept=2337092 rejected=82649 last=keep reason=rewritten +[llm clean] completed=139100 submitted=139195 kept=2337191 rejected=82650 last=keep reason=rewritten +[llm clean] completed=139200 submitted=139295 kept=2337290 rejected=82651 last=keep reason=rewritten +[llm clean] completed=139300 submitted=139395 kept=2337382 rejected=82659 last=keep reason=rewritten +[llm clean] completed=139400 submitted=139495 kept=2337480 rejected=82661 last=keep reason=rewritten +[llm clean] completed=139500 submitted=139595 kept=2337572 rejected=82669 last=keep reason=rewritten +[llm clean] completed=139600 submitted=139695 kept=2337665 rejected=82676 last=keep reason=rewritten +[llm clean] completed=139700 submitted=139795 kept=2337764 rejected=82677 last=keep reason=rewritten +[llm clean] completed=139800 submitted=139895 kept=2337862 rejected=82679 last=keep reason=rewritten +[llm clean] completed=139900 submitted=139995 kept=2337954 rejected=82687 last=keep reason=rewritten +[llm clean] completed=140000 submitted=140095 kept=2338054 rejected=82687 last=keep reason=rewritten +[llm clean] completed=140100 submitted=140195 kept=2338152 rejected=82689 last=keep reason=rewritten +[llm clean] completed=140200 submitted=140295 kept=2338251 rejected=82690 last=keep reason=rewritten +[llm clean] completed=140300 submitted=140395 kept=2338350 rejected=82691 last=keep reason=rewritten +[llm clean] completed=140400 submitted=140495 kept=2338449 rejected=82692 last=keep reason=rewritten +[llm clean] completed=140500 submitted=140595 kept=2338541 rejected=82700 last=keep reason=rewritten +[llm clean] completed=140600 submitted=140695 kept=2338641 rejected=82700 last=keep reason=rewritten +[llm clean] completed=140700 submitted=140795 kept=2338739 rejected=82702 last=keep reason=rewritten +[llm clean] completed=140800 submitted=140895 kept=2338838 rejected=82703 last=keep reason=rewritten +[llm clean] completed=140900 submitted=140995 kept=2338935 rejected=82706 last=keep reason=rewritten +[llm clean] completed=141000 submitted=141095 kept=2339029 rejected=82712 last=keep reason=rewritten +[llm clean] completed=141100 submitted=141195 kept=2339129 rejected=82712 last=keep reason=rewritten +[llm clean] completed=141200 submitted=141295 kept=2339227 rejected=82714 last=keep reason=rewritten +[llm clean] completed=141300 submitted=141395 kept=2339322 rejected=82719 last=keep reason=rewritten +[llm clean] completed=141400 submitted=141495 kept=2339422 rejected=82719 last=keep reason=rewritten +[llm clean] completed=141500 submitted=141595 kept=2339522 rejected=82719 last=keep reason=rewritten +[llm clean] completed=141600 submitted=141695 kept=2339615 rejected=82726 last=keep reason=rewritten +[llm clean] completed=141700 submitted=141795 kept=2339703 rejected=82738 last=reject reason=The text consists of fragmented bibliographic entries, ISBNs, OCLC numbers, and reference metadata without coherent pros +[llm clean] completed=141800 submitted=141895 kept=2339801 rejected=82740 last=keep reason=rewritten +[llm clean] completed=141900 submitted=141995 kept=2339900 rejected=82741 last=keep reason=rewritten +[llm clean] completed=142000 submitted=142095 kept=2339999 rejected=82742 last=keep reason=rewritten +[llm clean] completed=142100 submitted=142195 kept=2340095 rejected=82746 last=reject reason= +[llm clean] completed=142200 submitted=142295 kept=2340192 rejected=82749 last=keep reason=rewritten +[llm clean] completed=142300 submitted=142395 kept=2340287 rejected=82754 last=keep reason=rewritten +[llm clean] completed=142400 submitted=142495 kept=2340383 rejected=82758 last=keep reason=rewritten +[llm clean] completed=142500 submitted=142595 kept=2340482 rejected=82759 last=reject reason=The text is a list of patch notes or bug fixes, mostly consisting of fragmented sentences and isolated statements withou +[llm clean] completed=142600 submitted=142695 kept=2340580 rejected=82761 last=keep reason=rewritten +[llm clean] completed=142700 submitted=142795 kept=2340676 rejected=82765 last=keep reason=rewritten +[llm clean] completed=142800 submitted=142895 kept=2340775 rejected=82766 last=keep reason=rewritten +[llm clean] completed=142900 submitted=142995 kept=2340874 rejected=82767 last=keep reason=rewritten +[llm clean] completed=143000 submitted=143095 kept=2340973 rejected=82768 last=keep reason=rewritten +[llm clean] completed=143100 submitted=143195 kept=2341072 rejected=82769 last=keep reason=rewritten +[llm clean] completed=143200 submitted=143295 kept=2341170 rejected=82771 last=keep reason=rewritten +[llm clean] completed=143300 submitted=143395 kept=2341270 rejected=82771 last=keep reason=rewritten +[llm clean] completed=143400 submitted=143495 kept=2341366 rejected=82775 last=keep reason=rewritten +[llm clean] completed=143500 submitted=143595 kept=2341461 rejected=82780 last=keep reason=rewritten +[llm clean] completed=143600 submitted=143695 kept=2341554 rejected=82787 last=keep reason=rewritten +[llm clean] completed=143700 submitted=143795 kept=2341653 rejected=82788 last=keep reason=rewritten +[llm clean] completed=143800 submitted=143895 kept=2341753 rejected=82788 last=keep reason=rewritten +[llm clean] completed=143900 submitted=143995 kept=2341846 rejected=82795 last=keep reason=rewritten +[llm clean] completed=144000 submitted=144095 kept=2341941 rejected=82800 last=keep reason=rewritten +[llm clean] completed=144100 submitted=144195 kept=2342040 rejected=82801 last=keep reason=rewritten +[llm clean] completed=144200 submitted=144295 kept=2342137 rejected=82804 last=keep reason=rewritten +[llm clean] completed=144300 submitted=144395 kept=2342228 rejected=82813 last=keep reason=rewritten +[llm clean] completed=144400 submitted=144495 kept=2342298 rejected=82843 last=keep reason=rewritten +[llm clean] completed=144500 submitted=144595 kept=2342398 rejected=82843 last=keep reason=rewritten +[llm clean] completed=144600 submitted=144695 kept=2342497 rejected=82844 last=keep reason=rewritten +[llm clean] completed=144700 submitted=144795 kept=2342597 rejected=82844 last=keep reason=rewritten +[llm clean] completed=144800 submitted=144895 kept=2342693 rejected=82848 last=keep reason=rewritten +[llm clean] completed=144900 submitted=144995 kept=2342793 rejected=82848 last=keep reason=rewritten +[llm clean] completed=145000 submitted=145095 kept=2342870 rejected=82871 last=reject reason=The text is a roster/list of names, titles, and low-context fragments (StarCraft 2 player profiles) without meaningful p +[llm clean] completed=145100 submitted=145195 kept=2342956 rejected=82885 last=keep reason=rewritten +[llm clean] completed=145200 submitted=145295 kept=2343055 rejected=82886 last=keep reason=rewritten +[llm clean] completed=145300 submitted=145395 kept=2343152 rejected=82889 last=keep reason=rewritten +[llm clean] completed=145400 submitted=145495 kept=2343251 rejected=82890 last=keep reason=rewritten +[llm clean] completed=145500 submitted=145595 kept=2343347 rejected=82894 last=keep reason=rewritten +[llm clean] completed=145600 submitted=145695 kept=2343443 rejected=82898 last=keep reason=rewritten +[llm clean] completed=145700 submitted=145795 kept=2343542 rejected=82899 last=keep reason=rewritten +[llm clean] completed=145800 submitted=145895 kept=2343638 rejected=82903 last=keep reason=rewritten +[llm clean] completed=145900 submitted=145995 kept=2343719 rejected=82922 last=reject reason=The input is a list of bibliographic references and citations, not a coherent article or prose passage. +[llm clean] completed=146000 submitted=146095 kept=2343811 rejected=82930 last=keep reason=rewritten +[llm clean] completed=146100 submitted=146195 kept=2343897 rejected=82944 last=reject reason=The text is a fragmented list of rhetorical questions and comments, likely from a forum or comment section, discussing s +[llm clean] completed=146200 submitted=146295 kept=2343993 rejected=82948 last=keep reason=rewritten +[llm clean] completed=146300 submitted=146395 kept=2344089 rejected=82952 last=keep reason=rewritten +[llm clean] completed=146400 submitted=146495 kept=2344188 rejected=82953 last=keep reason=rewritten +[llm clean] completed=146500 submitted=146595 kept=2344287 rejected=82954 last=keep reason=rewritten +[llm clean] completed=146600 submitted=146695 kept=2344380 rejected=82961 last=keep reason=rewritten +[llm clean] completed=146700 submitted=146795 kept=2344475 rejected=82966 last=keep reason=rewritten +[llm clean] completed=146800 submitted=146895 kept=2344575 rejected=82966 last=keep reason=rewritten +[llm clean] completed=146900 submitted=146995 kept=2344666 rejected=82975 last=keep reason=rewritten +[llm clean] completed=147000 submitted=147095 kept=2344762 rejected=82979 last=keep reason=rewritten +[llm clean] completed=147100 submitted=147195 kept=2344862 rejected=82979 last=keep reason=rewritten +[llm clean] completed=147200 submitted=147295 kept=2344961 rejected=82980 last=keep reason=rewritten +[llm clean] completed=147300 submitted=147395 kept=2345056 rejected=82985 last=keep reason=rewritten +[llm clean] completed=147400 submitted=147495 kept=2345155 rejected=82986 last=keep reason=rewritten +[llm clean] completed=147500 submitted=147595 kept=2345250 rejected=82991 last=keep reason=rewritten +[llm clean] completed=147600 submitted=147695 kept=2345347 rejected=82994 last=keep reason=rewritten +[llm clean] completed=147700 submitted=147795 kept=2345443 rejected=82998 last=keep reason=rewritten +[llm clean] completed=147800 submitted=147895 kept=2345533 rejected=83008 last=keep reason=rewritten +[llm clean] completed=147900 submitted=147995 kept=2345629 rejected=83012 last=keep reason=rewritten +[llm clean] completed=148000 submitted=148095 kept=2345726 rejected=83015 last=keep reason=rewritten +[llm clean] completed=148100 submitted=148195 kept=2345822 rejected=83019 last=keep reason=rewritten +[llm clean] completed=148200 submitted=148295 kept=2345919 rejected=83022 last=keep reason=rewritten +[llm clean] completed=148300 submitted=148395 kept=2346011 rejected=83030 last=keep reason=rewritten +[llm clean] completed=148400 submitted=148495 kept=2346106 rejected=83035 last=keep reason=rewritten +[llm clean] completed=148500 submitted=148595 kept=2346205 rejected=83036 last=keep reason=rewritten +[llm clean] completed=148600 submitted=148695 kept=2346299 rejected=83042 last=reject reason=The text is heavily biased, uses derogatory and non-standard terminology ('moonbats', 'climate commies', 'climate clown' +[llm clean] completed=148700 submitted=148795 kept=2346392 rejected=83049 last=keep reason=rewritten +[llm clean] completed=148800 submitted=148895 kept=2346490 rejected=83051 last=keep reason=rewritten +[llm clean] completed=148900 submitted=148995 kept=2346585 rejected=83056 last=keep reason=rewritten +[llm clean] completed=149000 submitted=149095 kept=2346682 rejected=83059 last=keep reason=rewritten +[llm clean] completed=149100 submitted=149195 kept=2346779 rejected=83062 last=keep reason=rewritten +[llm clean] completed=149200 submitted=149295 kept=2346872 rejected=83069 last=keep reason=rewritten +[llm clean] completed=149300 submitted=149395 kept=2346971 rejected=83070 last=keep reason=rewritten +[llm clean] completed=149400 submitted=149495 kept=2347070 rejected=83071 last=keep reason=rewritten +[llm clean] completed=149500 submitted=149595 kept=2347165 rejected=83076 last=reject reason= +[llm clean] completed=149600 submitted=149695 kept=2347262 rejected=83079 last=keep reason=rewritten +[llm clean] completed=149700 submitted=149795 kept=2347340 rejected=83101 last=reject reason= +[llm clean] completed=149800 submitted=149895 kept=2347432 rejected=83109 last=keep reason=rewritten +[llm clean] completed=149900 submitted=149995 kept=2347530 rejected=83111 last=keep reason=rewritten +[llm clean] completed=150000 submitted=150095 kept=2347628 rejected=83113 last=keep reason=rewritten +[llm clean] completed=150100 submitted=150195 kept=2347727 rejected=83114 last=keep reason=rewritten +[llm clean] completed=150200 submitted=150295 kept=2347823 rejected=83118 last=keep reason=rewritten +[llm clean] completed=150300 submitted=150395 kept=2347923 rejected=83118 last=keep reason=rewritten +[llm clean] completed=150400 submitted=150495 kept=2348022 rejected=83119 last=keep reason=rewritten +[llm clean] completed=150500 submitted=150595 kept=2348117 rejected=83124 last=keep reason=rewritten +[llm clean] completed=150600 submitted=150695 kept=2348202 rejected=83139 last=keep reason=rewritten +[llm clean] completed=150700 submitted=150795 kept=2348300 rejected=83141 last=keep reason=rewritten +[llm clean] completed=150800 submitted=150895 kept=2348378 rejected=83163 last=keep reason=rewritten +[llm clean] completed=150900 submitted=150995 kept=2348477 rejected=83164 last=keep reason=rewritten +[llm clean] completed=151000 submitted=151095 kept=2348573 rejected=83168 last=keep reason=rewritten +[llm clean] completed=151100 submitted=151195 kept=2348673 rejected=83168 last=keep reason=rewritten +[llm clean] completed=151200 submitted=151295 kept=2348772 rejected=83169 last=keep reason=rewritten +[llm clean] completed=151300 submitted=151395 kept=2348871 rejected=83170 last=keep reason=rewritten +[llm clean] completed=151400 submitted=151495 kept=2348969 rejected=83172 last=keep reason=rewritten +[llm clean] completed=151500 submitted=151595 kept=2349069 rejected=83172 last=keep reason=rewritten +[llm clean] completed=151600 submitted=151695 kept=2349168 rejected=83173 last=keep reason=rewritten +[llm clean] completed=151700 submitted=151795 kept=2349265 rejected=83176 last=keep reason=rewritten +[llm clean] completed=151800 submitted=151895 kept=2349360 rejected=83181 last=keep reason=rewritten +[llm clean] completed=151900 submitted=151995 kept=2349448 rejected=83193 last=keep reason=rewritten +[llm clean] completed=152000 submitted=152095 kept=2349538 rejected=83203 last=keep reason=rewritten +[llm clean] completed=152100 submitted=152195 kept=2349636 rejected=83205 last=keep reason=rewritten +[llm clean] completed=152200 submitted=152295 kept=2349733 rejected=83208 last=keep reason=rewritten +[llm clean] completed=152300 submitted=152395 kept=2349827 rejected=83214 last=keep reason=rewritten +[llm clean] completed=152400 submitted=152495 kept=2349923 rejected=83218 last=keep reason=rewritten +[llm clean] completed=152500 submitted=152595 kept=2350022 rejected=83219 last=keep reason=rewritten +[llm clean] completed=152600 submitted=152695 kept=2350122 rejected=83219 last=keep reason=rewritten +[llm clean] completed=152700 submitted=152795 kept=2350217 rejected=83224 last=keep reason=rewritten +[llm clean] completed=152800 submitted=152895 kept=2350306 rejected=83235 last=reject reason=The text is a collection of quotes and claims heavily focused on ethnic and religious stereotypes, promoting conspiracy +[llm clean] completed=152900 submitted=152995 kept=2350402 rejected=83239 last=keep reason=rewritten +[llm clean] completed=153000 submitted=153095 kept=2350500 rejected=83241 last=keep reason=rewritten +[llm clean] completed=153100 submitted=153195 kept=2350596 rejected=83245 last=keep reason=rewritten +[llm clean] completed=153200 submitted=153295 kept=2350695 rejected=83246 last=keep reason=rewritten +[llm clean] completed=153300 submitted=153395 kept=2350787 rejected=83254 last=keep reason=rewritten +[llm clean] completed=153400 submitted=153495 kept=2350885 rejected=83256 last=keep reason=rewritten +[llm clean] completed=153500 submitted=153595 kept=2350985 rejected=83256 last=keep reason=rewritten +[llm clean] completed=153600 submitted=153695 kept=2351082 rejected=83259 last=keep reason=rewritten +[llm clean] completed=153700 submitted=153795 kept=2351179 rejected=83262 last=keep reason=rewritten +[llm clean] completed=153800 submitted=153895 kept=2351279 rejected=83262 last=keep reason=rewritten +[llm clean] completed=153900 submitted=153995 kept=2351377 rejected=83264 last=keep reason=rewritten +[llm clean] completed=154000 submitted=154095 kept=2351476 rejected=83265 last=keep reason=rewritten +[llm clean] completed=154100 submitted=154195 kept=2351570 rejected=83271 last=keep reason=rewritten +[llm clean] completed=154200 submitted=154295 kept=2351664 rejected=83277 last=keep reason=rewritten +[llm clean] completed=154300 submitted=154395 kept=2351763 rejected=83278 last=keep reason=rewritten +[llm clean] completed=154400 submitted=154495 kept=2351863 rejected=83278 last=keep reason=rewritten +[llm clean] completed=154500 submitted=154595 kept=2351962 rejected=83279 last=keep reason=rewritten +[llm clean] completed=154600 submitted=154695 kept=2352059 rejected=83282 last=keep reason=rewritten +[llm clean] completed=154700 submitted=154795 kept=2352145 rejected=83296 last=keep reason=rewritten +[llm clean] completed=154800 submitted=154895 kept=2352241 rejected=83300 last=keep reason=rewritten +[llm clean] completed=154900 submitted=154995 kept=2352341 rejected=83300 last=keep reason=rewritten +[llm clean] completed=155000 submitted=155095 kept=2352435 rejected=83306 last=keep reason=rewritten +[llm clean] completed=155100 submitted=155195 kept=2352533 rejected=83308 last=keep reason=rewritten +[llm clean] completed=155200 submitted=155295 kept=2352627 rejected=83314 last=keep reason=rewritten +[llm clean] completed=155300 submitted=155395 kept=2352723 rejected=83318 last=keep reason=rewritten +[llm clean] completed=155400 submitted=155495 kept=2352822 rejected=83319 last=keep reason=rewritten +[llm clean] completed=155500 submitted=155595 kept=2352919 rejected=83322 last=keep reason=rewritten +[llm clean] completed=155600 submitted=155695 kept=2353015 rejected=83326 last=keep reason=rewritten +[llm clean] completed=155700 submitted=155795 kept=2353114 rejected=83327 last=keep reason=rewritten +[llm clean] completed=155800 submitted=155895 kept=2353188 rejected=83353 last=keep reason=rewritten +[llm clean] completed=155900 submitted=155995 kept=2353274 rejected=83367 last=keep reason=rewritten +[llm clean] completed=156000 submitted=156095 kept=2353373 rejected=83368 last=keep reason=rewritten +[llm clean] completed=156100 submitted=156195 kept=2353471 rejected=83370 last=keep reason=rewritten +[llm clean] completed=156200 submitted=156295 kept=2353571 rejected=83370 last=keep reason=rewritten +[llm clean] completed=156300 submitted=156395 kept=2353670 rejected=83371 last=keep reason=rewritten +[llm clean] completed=156400 submitted=156495 kept=2353768 rejected=83373 last=keep reason=rewritten +[llm clean] completed=156500 submitted=156595 kept=2353866 rejected=83375 last=keep reason=rewritten +[llm clean] completed=156600 submitted=156695 kept=2353962 rejected=83379 last=keep reason=rewritten +[llm clean] completed=156700 submitted=156795 kept=2354057 rejected=83384 last=keep reason=rewritten +[llm clean] completed=156800 submitted=156895 kept=2354151 rejected=83390 last=keep reason=rewritten +[llm clean] completed=156900 submitted=156995 kept=2354248 rejected=83393 last=keep reason=rewritten +[llm clean] completed=157000 submitted=157095 kept=2354345 rejected=83396 last=keep reason=rewritten +[llm clean] completed=157100 submitted=157195 kept=2354440 rejected=83401 last=keep reason=rewritten +[llm clean] completed=157200 submitted=157295 kept=2354536 rejected=83405 last=keep reason=rewritten +[llm clean] completed=157300 submitted=157395 kept=2354635 rejected=83406 last=keep reason=rewritten +[llm clean] completed=157400 submitted=157495 kept=2354733 rejected=83408 last=keep reason=rewritten +[llm clean] completed=157500 submitted=157595 kept=2354829 rejected=83412 last=keep reason=rewritten +[llm clean] completed=157600 submitted=157695 kept=2354927 rejected=83414 last=keep reason=rewritten +[llm clean] completed=157700 submitted=157795 kept=2355026 rejected=83415 last=keep reason=rewritten +[llm clean] completed=157800 submitted=157895 kept=2355122 rejected=83419 last=keep reason=rewritten +[llm clean] completed=157900 submitted=157995 kept=2355217 rejected=83424 last=keep reason=rewritten +[llm clean] completed=158000 submitted=158095 kept=2355302 rejected=83439 last=keep reason=rewritten +[llm clean] completed=158100 submitted=158195 kept=2355401 rejected=83440 last=keep reason=rewritten +[llm clean] completed=158200 submitted=158295 kept=2355498 rejected=83443 last=keep reason=rewritten +[llm clean] completed=158300 submitted=158395 kept=2355595 rejected=83446 last=keep reason=rewritten +[llm clean] completed=158400 submitted=158495 kept=2355693 rejected=83448 last=keep reason=rewritten +[llm clean] completed=158500 submitted=158595 kept=2355792 rejected=83449 last=keep reason=rewritten +[llm clean] completed=158600 submitted=158695 kept=2355890 rejected=83451 last=keep reason=rewritten +[llm clean] completed=158700 submitted=158795 kept=2355988 rejected=83453 last=keep reason=rewritten +[llm clean] completed=158800 submitted=158895 kept=2356087 rejected=83454 last=keep reason=rewritten +[llm clean] completed=158900 submitted=158995 kept=2356184 rejected=83457 last=keep reason=rewritten +[llm clean] completed=159000 submitted=159095 kept=2356276 rejected=83465 last=keep reason=rewritten +[llm clean] completed=159100 submitted=159195 kept=2356375 rejected=83466 last=keep reason=rewritten +[llm clean] completed=159200 submitted=159295 kept=2356470 rejected=83471 last=keep reason=rewritten +[llm clean] completed=159300 submitted=159395 kept=2356569 rejected=83472 last=keep reason=rewritten +[llm clean] completed=159400 submitted=159495 kept=2356666 rejected=83475 last=keep reason=rewritten +[llm clean] completed=159500 submitted=159595 kept=2356760 rejected=83481 last=keep reason=rewritten +[llm clean] completed=159600 submitted=159695 kept=2356858 rejected=83483 last=keep reason=rewritten +[llm clean] completed=159700 submitted=159795 kept=2356958 rejected=83483 last=keep reason=rewritten +[llm clean] completed=159800 submitted=159895 kept=2357051 rejected=83490 last=keep reason=rewritten +[llm clean] completed=159900 submitted=159995 kept=2357141 rejected=83500 last=keep reason=rewritten +[llm clean] completed=160000 submitted=160095 kept=2357239 rejected=83502 last=keep reason=rewritten +[llm clean] completed=160100 submitted=160195 kept=2357336 rejected=83505 last=keep reason=rewritten +[llm clean] completed=160200 submitted=160295 kept=2357433 rejected=83508 last=keep reason=rewritten +[llm clean] completed=160300 submitted=160395 kept=2357530 rejected=83511 last=keep reason=rewritten +[llm clean] completed=160400 submitted=160495 kept=2357625 rejected=83516 last=keep reason=rewritten +[llm clean] completed=160500 submitted=160595 kept=2357725 rejected=83516 last=keep reason=rewritten +[llm clean] completed=160600 submitted=160695 kept=2357823 rejected=83518 last=keep reason=rewritten +[llm clean] completed=160700 submitted=160795 kept=2357922 rejected=83519 last=keep reason=rewritten +[llm clean] completed=160800 submitted=160895 kept=2358020 rejected=83521 last=keep reason=rewritten +[llm clean] completed=160900 submitted=160995 kept=2358119 rejected=83522 last=keep reason=rewritten +[llm clean] completed=161000 submitted=161095 kept=2358217 rejected=83524 last=keep reason=rewritten +[llm clean] completed=161100 submitted=161195 kept=2358317 rejected=83524 last=keep reason=rewritten +[llm clean] completed=161200 submitted=161295 kept=2358416 rejected=83525 last=keep reason=rewritten +[llm clean] completed=161300 submitted=161395 kept=2358513 rejected=83528 last=keep reason=rewritten +[llm clean] completed=161400 submitted=161495 kept=2358613 rejected=83528 last=keep reason=rewritten +[llm clean] completed=161500 submitted=161595 kept=2358712 rejected=83529 last=keep reason=rewritten +[llm clean] completed=161600 submitted=161695 kept=2358812 rejected=83529 last=keep reason=rewritten +[llm clean] completed=161700 submitted=161795 kept=2358912 rejected=83529 last=keep reason=rewritten +[llm clean] completed=161800 submitted=161895 kept=2359008 rejected=83533 last=reject reason= +[llm clean] completed=161900 submitted=161995 kept=2359105 rejected=83536 last=keep reason=rewritten +[llm clean] completed=162000 submitted=162095 kept=2359205 rejected=83536 last=keep reason=rewritten +[llm clean] completed=162100 submitted=162195 kept=2359304 rejected=83537 last=keep reason=rewritten +[llm clean] completed=162200 submitted=162295 kept=2359398 rejected=83543 last=keep reason=rewritten +[llm clean] completed=162300 submitted=162395 kept=2359498 rejected=83543 last=keep reason=rewritten +[llm clean] completed=162400 submitted=162495 kept=2359597 rejected=83544 last=keep reason=rewritten +[llm clean] completed=162500 submitted=162595 kept=2359693 rejected=83548 last=keep reason=rewritten +[llm clean] completed=162600 submitted=162695 kept=2359792 rejected=83549 last=keep reason=rewritten +[llm clean] completed=162700 submitted=162795 kept=2359886 rejected=83555 last=keep reason=rewritten +[llm clean] completed=162800 submitted=162895 kept=2359976 rejected=83565 last=reject reason=The input is a raw, unstructured list of football league standings or statistics (team names, codes, numbers, percentage +[llm clean] completed=162900 submitted=162995 kept=2360069 rejected=83572 last=keep reason=rewritten +[llm clean] completed=163000 submitted=163095 kept=2360163 rejected=83578 last=keep reason=rewritten +[llm clean] completed=163100 submitted=163195 kept=2360261 rejected=83580 last=keep reason=rewritten +[llm clean] completed=163200 submitted=163295 kept=2360360 rejected=83581 last=keep reason=rewritten +[llm clean] completed=163300 submitted=163395 kept=2360460 rejected=83581 last=keep reason=rewritten +[llm clean] completed=163400 submitted=163494 kept=2360559 rejected=83582 last=keep reason=rewritten +[llm clean] completed=163500 submitted=163595 kept=2360656 rejected=83585 last=keep reason=rewritten +[llm clean] completed=163600 submitted=163695 kept=2360755 rejected=83586 last=keep reason=rewritten +[llm clean] completed=163700 submitted=163795 kept=2360853 rejected=83588 last=keep reason=rewritten +[llm clean] completed=163800 submitted=163895 kept=2360949 rejected=83592 last=keep reason=rewritten +[llm clean] completed=163900 submitted=163995 kept=2361044 rejected=83597 last=keep reason=rewritten +[llm clean] completed=164000 submitted=164095 kept=2361139 rejected=83602 last=keep reason=rewritten +[llm clean] completed=164100 submitted=164195 kept=2361237 rejected=83604 last=keep reason=rewritten +[llm clean] completed=164200 submitted=164295 kept=2361335 rejected=83606 last=keep reason=rewritten +[llm clean] completed=164300 submitted=164395 kept=2361431 rejected=83610 last=keep reason=rewritten +[llm clean] completed=164400 submitted=164495 kept=2361530 rejected=83611 last=keep reason=rewritten +[llm clean] completed=164500 submitted=164595 kept=2361627 rejected=83614 last=keep reason=rewritten +[llm clean] completed=164600 submitted=164695 kept=2361723 rejected=83618 last=keep reason=rewritten +[llm clean] completed=164700 submitted=164795 kept=2361822 rejected=83619 last=keep reason=rewritten +[llm clean] completed=164800 submitted=164895 kept=2361917 rejected=83624 last=keep reason=rewritten +[llm clean] completed=164900 submitted=164995 kept=2362017 rejected=83624 last=keep reason=rewritten +[llm clean] completed=165000 submitted=165095 kept=2362114 rejected=83627 last=keep reason=rewritten +[llm clean] completed=165100 submitted=165195 kept=2362214 rejected=83627 last=keep reason=rewritten +[llm clean] completed=165200 submitted=165295 kept=2362311 rejected=83630 last=keep reason=rewritten +[llm clean] completed=165300 submitted=165395 kept=2362410 rejected=83631 last=keep reason=rewritten +[llm clean] completed=165400 submitted=165495 kept=2362496 rejected=83645 last=keep reason=rewritten +[llm clean] completed=165500 submitted=165595 kept=2362595 rejected=83646 last=keep reason=rewritten +[llm clean] completed=165600 submitted=165695 kept=2362692 rejected=83649 last=keep reason=rewritten +[llm clean] completed=165700 submitted=165795 kept=2362788 rejected=83653 last=keep reason=rewritten +[llm clean] completed=165800 submitted=165895 kept=2362883 rejected=83658 last=keep reason=rewritten +[llm clean] completed=165900 submitted=165995 kept=2362976 rejected=83665 last=keep reason=rewritten +[llm clean] completed=166000 submitted=166095 kept=2363075 rejected=83666 last=keep reason=rewritten +[llm clean] completed=166100 submitted=166195 kept=2363174 rejected=83667 last=keep reason=rewritten +[llm clean] completed=166200 submitted=166295 kept=2363271 rejected=83670 last=keep reason=rewritten +[llm clean] completed=166300 submitted=166395 kept=2363370 rejected=83671 last=keep reason=rewritten +[llm clean] completed=166400 submitted=166495 kept=2363467 rejected=83674 last=keep reason=rewritten +[llm clean] completed=166500 submitted=166595 kept=2363566 rejected=83675 last=reject reason=The text contains highly offensive racial slurs, promotes harmful stereotypes, and expresses racist ideology. It violate +[llm clean] completed=166600 submitted=166695 kept=2363661 rejected=83680 last=keep reason=rewritten +[llm clean] completed=166700 submitted=166795 kept=2363755 rejected=83686 last=keep reason=rewritten +[llm clean] completed=166800 submitted=166895 kept=2363853 rejected=83688 last=keep reason=rewritten +[llm clean] completed=166900 submitted=166995 kept=2363926 rejected=83715 last=keep reason=rewritten +[llm clean] completed=167000 submitted=167095 kept=2364025 rejected=83716 last=keep reason=rewritten +[llm clean] completed=167100 submitted=167195 kept=2364120 rejected=83721 last=keep reason=rewritten +[llm clean] completed=167200 submitted=167295 kept=2364218 rejected=83723 last=keep reason=rewritten +[llm clean] completed=167300 submitted=167395 kept=2364316 rejected=83725 last=keep reason=rewritten +[llm clean] completed=167400 submitted=167495 kept=2364415 rejected=83726 last=reject reason=The text is a raw script transcript with stage directions, character names, and formatting artifacts, lacking coherent a +[llm clean] completed=167500 submitted=167595 kept=2364513 rejected=83728 last=keep reason=rewritten +[llm clean] completed=167600 submitted=167695 kept=2364610 rejected=83731 last=keep reason=rewritten +[llm clean] completed=167700 submitted=167795 kept=2364709 rejected=83732 last=keep reason=rewritten +[llm clean] completed=167800 submitted=167895 kept=2364805 rejected=83736 last=keep reason=rewritten +[llm clean] completed=167900 submitted=167995 kept=2364898 rejected=83743 last=keep reason=rewritten +[llm clean] completed=168000 submitted=168095 kept=2364998 rejected=83743 last=keep reason=rewritten +[llm clean] completed=168100 submitted=168195 kept=2365097 rejected=83744 last=keep reason=rewritten +[llm clean] completed=168200 submitted=168295 kept=2365196 rejected=83745 last=keep reason=rewritten +[llm clean] completed=168300 submitted=168395 kept=2365289 rejected=83752 last=keep reason=rewritten +[llm clean] completed=168400 submitted=168495 kept=2365387 rejected=83754 last=keep reason=rewritten +[llm clean] completed=168500 submitted=168595 kept=2365487 rejected=83754 last=keep reason=rewritten +[llm clean] completed=168600 submitted=168695 kept=2365585 rejected=83756 last=keep reason=rewritten +[llm clean] completed=168700 submitted=168795 kept=2365683 rejected=83758 last=keep reason=rewritten +[llm clean] completed=168800 submitted=168895 kept=2365776 rejected=83765 last=reject reason=The text is a disjointed list of blog post summaries, commentary snippets, and isolated quotes without a coherent narrat +[llm clean] completed=168900 submitted=168995 kept=2365867 rejected=83774 last=keep reason=rewritten +[llm clean] completed=169000 submitted=169095 kept=2365967 rejected=83774 last=keep reason=rewritten +[llm clean] completed=169100 submitted=169195 kept=2366067 rejected=83774 last=keep reason=rewritten +[llm clean] completed=169200 submitted=169295 kept=2366165 rejected=83776 last=keep reason=rewritten +[llm clean] completed=169300 submitted=169395 kept=2366265 rejected=83776 last=keep reason=rewritten +[llm clean] completed=169400 submitted=169495 kept=2366362 rejected=83779 last=keep reason=rewritten +[llm clean] completed=169500 submitted=169595 kept=2366460 rejected=83781 last=keep reason=rewritten +[llm clean] completed=169600 submitted=169695 kept=2366556 rejected=83785 last=keep reason=rewritten +[llm clean] completed=169700 submitted=169795 kept=2366653 rejected=83788 last=keep reason=rewritten +[llm clean] completed=169800 submitted=169895 kept=2366746 rejected=83795 last=keep reason=rewritten +[llm clean] completed=169900 submitted=169995 kept=2366844 rejected=83797 last=keep reason=rewritten +[llm clean] completed=170000 submitted=170095 kept=2366937 rejected=83804 last=keep reason=rewritten +[llm clean] completed=170100 submitted=170195 kept=2367035 rejected=83806 last=keep reason=rewritten +[llm clean] completed=170200 submitted=170295 kept=2367134 rejected=83807 last=keep reason=rewritten +[llm clean] completed=170300 submitted=170395 kept=2367234 rejected=83807 last=keep reason=rewritten +[llm clean] completed=170400 submitted=170495 kept=2367329 rejected=83812 last=keep reason=rewritten +[llm clean] completed=170500 submitted=170595 kept=2367426 rejected=83815 last=keep reason=rewritten +[llm clean] completed=170600 submitted=170695 kept=2367526 rejected=83815 last=keep reason=rewritten +[llm clean] completed=170700 submitted=170795 kept=2367624 rejected=83817 last=keep reason=rewritten +[llm clean] completed=170800 submitted=170895 kept=2367720 rejected=83821 last=keep reason=rewritten +[llm clean] completed=170900 submitted=170995 kept=2367810 rejected=83831 last=keep reason=rewritten +[llm clean] completed=171000 submitted=171095 kept=2367907 rejected=83834 last=keep reason=rewritten +[llm clean] completed=171100 submitted=171195 kept=2368003 rejected=83838 last=keep reason=rewritten +[llm clean] completed=171200 submitted=171295 kept=2368103 rejected=83838 last=keep reason=rewritten +[llm clean] completed=171300 submitted=171395 kept=2368198 rejected=83843 last=keep reason=rewritten +[llm clean] completed=171400 submitted=171495 kept=2368298 rejected=83843 last=keep reason=rewritten +[llm clean] completed=171500 submitted=171595 kept=2368391 rejected=83850 last=keep reason=rewritten +[llm clean] completed=171600 submitted=171695 kept=2368489 rejected=83852 last=keep reason=rewritten +[llm clean] completed=171700 submitted=171795 kept=2368589 rejected=83852 last=keep reason=rewritten +[llm clean] completed=171800 submitted=171895 kept=2368688 rejected=83853 last=keep reason=rewritten +[llm clean] completed=171900 submitted=171995 kept=2368786 rejected=83855 last=keep reason=rewritten +[llm clean] completed=172000 submitted=172095 kept=2368885 rejected=83856 last=keep reason=rewritten +[llm clean] completed=172100 submitted=172195 kept=2368984 rejected=83857 last=keep reason=rewritten +[llm clean] completed=172200 submitted=172295 kept=2369081 rejected=83860 last=keep reason=rewritten +[llm clean] completed=172300 submitted=172395 kept=2369171 rejected=83870 last=keep reason=rewritten +[llm clean] completed=172400 submitted=172495 kept=2369267 rejected=83874 last=keep reason=rewritten +[llm clean] completed=172500 submitted=172595 kept=2369365 rejected=83876 last=keep reason=rewritten +[llm clean] completed=172600 submitted=172695 kept=2369461 rejected=83880 last=keep reason=rewritten +[llm clean] completed=172700 submitted=172795 kept=2369556 rejected=83885 last=keep reason=rewritten +[llm clean] completed=172800 submitted=172895 kept=2369649 rejected=83892 last=keep reason=rewritten +[llm clean] completed=172900 submitted=172995 kept=2369747 rejected=83894 last=keep reason=rewritten +[llm clean] completed=173000 submitted=173095 kept=2369847 rejected=83894 last=keep reason=rewritten +[llm clean] completed=173100 submitted=173195 kept=2369942 rejected=83899 last=keep reason=rewritten +[llm clean] completed=173200 submitted=173295 kept=2370040 rejected=83901 last=keep reason=rewritten +[llm clean] completed=173300 submitted=173395 kept=2370139 rejected=83902 last=keep reason=rewritten +[llm clean] completed=173400 submitted=173495 kept=2370233 rejected=83908 last=keep reason=rewritten +[llm clean] completed=173500 submitted=173595 kept=2370333 rejected=83908 last=keep reason=rewritten +[llm clean] completed=173600 submitted=173695 kept=2370432 rejected=83909 last=keep reason=rewritten +[llm clean] completed=173700 submitted=173795 kept=2370531 rejected=83910 last=keep reason=rewritten +[llm clean] completed=173800 submitted=173895 kept=2370630 rejected=83911 last=keep reason=rewritten +[llm clean] completed=173900 submitted=173995 kept=2370730 rejected=83911 last=keep reason=rewritten +[llm clean] completed=174000 submitted=174095 kept=2370828 rejected=83913 last=keep reason=rewritten +[llm clean] completed=174100 submitted=174195 kept=2370922 rejected=83919 last=keep reason=rewritten +[llm clean] completed=174200 submitted=174295 kept=2371019 rejected=83922 last=keep reason=rewritten +[llm clean] completed=174300 submitted=174395 kept=2371118 rejected=83923 last=keep reason=rewritten +[llm clean] completed=174400 submitted=174495 kept=2371217 rejected=83924 last=keep reason=rewritten +[llm clean] completed=174500 submitted=174595 kept=2371316 rejected=83925 last=keep reason=rewritten +[llm clean] completed=174600 submitted=174695 kept=2371410 rejected=83931 last=keep reason=rewritten +[llm clean] completed=174700 submitted=174795 kept=2371506 rejected=83935 last=keep reason=rewritten +[llm clean] completed=174800 submitted=174895 kept=2371603 rejected=83938 last=keep reason=rewritten +[llm clean] completed=174900 submitted=174995 kept=2371701 rejected=83940 last=keep reason=rewritten +[llm clean] completed=175000 submitted=175095 kept=2371799 rejected=83942 last=keep reason=rewritten +[llm clean] completed=175100 submitted=175195 kept=2371898 rejected=83943 last=keep reason=rewritten +[llm clean] completed=175200 submitted=175295 kept=2371997 rejected=83944 last=keep reason=rewritten +[llm clean] completed=175300 submitted=175395 kept=2372096 rejected=83945 last=keep reason=rewritten +[llm clean] completed=175400 submitted=175495 kept=2372190 rejected=83951 last=keep reason=rewritten +[llm clean] completed=175500 submitted=175595 kept=2372288 rejected=83953 last=keep reason=rewritten +[llm clean] completed=175600 submitted=175695 kept=2372377 rejected=83964 last=keep reason=rewritten +[llm clean] completed=175700 submitted=175795 kept=2372475 rejected=83966 last=keep reason=rewritten +[llm clean] completed=175800 submitted=175895 kept=2372571 rejected=83970 last=keep reason=rewritten +[llm clean] completed=175900 submitted=175995 kept=2372668 rejected=83973 last=keep reason=rewritten +[llm clean] completed=176000 submitted=176095 kept=2372767 rejected=83974 last=keep reason=rewritten +[llm clean] completed=176100 submitted=176195 kept=2372864 rejected=83977 last=keep reason=rewritten +[llm clean] completed=176200 submitted=176295 kept=2372961 rejected=83980 last=keep reason=rewritten +[llm clean] completed=176300 submitted=176395 kept=2373060 rejected=83981 last=keep reason=rewritten +[llm clean] completed=176400 submitted=176495 kept=2373155 rejected=83986 last=keep reason=rewritten +[llm clean] completed=176500 submitted=176595 kept=2373252 rejected=83989 last=keep reason=rewritten +[llm clean] completed=176600 submitted=176695 kept=2373351 rejected=83990 last=keep reason=rewritten +[llm clean] completed=176700 submitted=176794 kept=2373450 rejected=83991 last=keep reason=rewritten +[llm clean] completed=176800 submitted=176895 kept=2373548 rejected=83993 last=keep reason=rewritten +[llm clean] completed=176900 submitted=176995 kept=2373648 rejected=83993 last=keep reason=rewritten +[llm clean] completed=177000 submitted=177095 kept=2373746 rejected=83995 last=keep reason=rewritten +[llm clean] completed=177100 submitted=177195 kept=2373845 rejected=83996 last=keep reason=rewritten +[llm clean] completed=177200 submitted=177295 kept=2373944 rejected=83997 last=keep reason=rewritten +[llm clean] completed=177300 submitted=177395 kept=2374044 rejected=83997 last=keep reason=rewritten +[llm clean] completed=177400 submitted=177495 kept=2374137 rejected=84004 last=keep reason=rewritten +[llm clean] completed=177500 submitted=177595 kept=2374237 rejected=84004 last=keep reason=rewritten +[llm clean] completed=177600 submitted=177695 kept=2374332 rejected=84009 last=keep reason=rewritten +[llm clean] completed=177700 submitted=177795 kept=2374427 rejected=84014 last=keep reason=rewritten +[llm clean] completed=177800 submitted=177895 kept=2374523 rejected=84018 last=keep reason=rewritten +[llm clean] completed=177900 submitted=177995 kept=2374623 rejected=84018 last=keep reason=rewritten +[llm clean] completed=178000 submitted=178095 kept=2374720 rejected=84021 last=keep reason=rewritten +[llm clean] completed=178100 submitted=178195 kept=2374817 rejected=84024 last=keep reason=rewritten +[llm clean] completed=178200 submitted=178295 kept=2374917 rejected=84024 last=keep reason=rewritten +[llm clean] completed=178300 submitted=178395 kept=2375013 rejected=84028 last=reject reason=The text is primarily boilerplate, navigation links, contact information, and metadata without substantive article-style +[llm clean] completed=178400 submitted=178495 kept=2375110 rejected=84031 last=keep reason=rewritten +[llm clean] completed=178500 submitted=178595 kept=2375207 rejected=84034 last=keep reason=rewritten +[llm clean] completed=178600 submitted=178695 kept=2375306 rejected=84035 last=keep reason=rewritten +[llm clean] completed=178700 submitted=178795 kept=2375400 rejected=84041 last=keep reason=rewritten +[llm clean] completed=178800 submitted=178895 kept=2375499 rejected=84042 last=keep reason=rewritten +[llm clean] completed=178900 submitted=178995 kept=2375597 rejected=84044 last=keep reason=rewritten +[llm clean] completed=179000 submitted=179095 kept=2375695 rejected=84046 last=keep reason=rewritten +[llm clean] completed=179100 submitted=179195 kept=2375794 rejected=84047 last=keep reason=rewritten +[llm clean] completed=179200 submitted=179295 kept=2375892 rejected=84049 last=keep reason=rewritten +[llm clean] completed=179300 submitted=179395 kept=2375988 rejected=84053 last=keep reason=rewritten +[llm clean] completed=179400 submitted=179495 kept=2376085 rejected=84056 last=keep reason=rewritten +[llm clean] completed=179500 submitted=179595 kept=2376184 rejected=84057 last=keep reason=rewritten +[llm clean] completed=179600 submitted=179695 kept=2376284 rejected=84057 last=keep reason=rewritten +[llm clean] completed=179700 submitted=179795 kept=2376383 rejected=84058 last=keep reason=rewritten +[llm clean] completed=179800 submitted=179895 kept=2376481 rejected=84060 last=keep reason=rewritten +[llm clean] completed=179900 submitted=179995 kept=2376572 rejected=84069 last=reject reason=The text is primarily a list of references, citations, and links with minimal surrounding prose, making it unsuitable fo +[llm clean] completed=180000 submitted=180095 kept=2376668 rejected=84073 last=keep reason=rewritten +[llm clean] completed=180100 submitted=180195 kept=2376764 rejected=84077 last=keep reason=rewritten +[llm clean] completed=180200 submitted=180295 kept=2376860 rejected=84081 last=keep reason=rewritten +[llm clean] completed=180300 submitted=180395 kept=2376957 rejected=84084 last=keep reason=rewritten +[llm clean] completed=180400 submitted=180495 kept=2377050 rejected=84091 last=keep reason=rewritten +[llm clean] completed=180500 submitted=180595 kept=2377147 rejected=84094 last=keep reason=rewritten +[llm clean] completed=180600 submitted=180695 kept=2377241 rejected=84100 last=keep reason=rewritten +[llm clean] completed=180700 submitted=180795 kept=2377336 rejected=84105 last=reject reason=The text consists of a list of organization names, URLs, contact information, and a series of news article links without +[llm clean] completed=180800 submitted=180895 kept=2377428 rejected=84113 last=keep reason=rewritten +[llm clean] completed=180900 submitted=180995 kept=2377524 rejected=84117 last=keep reason=rewritten +[llm clean] completed=181000 submitted=181095 kept=2377622 rejected=84119 last=keep reason=rewritten +[llm clean] completed=181100 submitted=181195 kept=2377721 rejected=84120 last=keep reason=rewritten +[llm clean] completed=181200 submitted=181295 kept=2377819 rejected=84122 last=keep reason=rewritten +[llm clean] completed=181300 submitted=181395 kept=2377919 rejected=84122 last=keep reason=rewritten +[llm clean] completed=181400 submitted=181495 kept=2378019 rejected=84122 last=keep reason=rewritten +[llm clean] completed=181500 submitted=181595 kept=2378117 rejected=84124 last=keep reason=rewritten +[llm clean] completed=181600 submitted=181695 kept=2378193 rejected=84148 last=keep reason=rewritten +[llm clean] completed=181700 submitted=181795 kept=2378293 rejected=84148 last=keep reason=rewritten +[llm clean] completed=181800 submitted=181895 kept=2378392 rejected=84149 last=keep reason=rewritten +[llm clean] completed=181900 submitted=181995 kept=2378491 rejected=84150 last=keep reason=rewritten +[llm clean] completed=182000 submitted=182095 kept=2378591 rejected=84150 last=keep reason=rewritten +[llm clean] completed=182100 submitted=182195 kept=2378689 rejected=84152 last=keep reason=rewritten +[llm clean] completed=182200 submitted=182295 kept=2378784 rejected=84157 last=reject reason=The text consists of repetitive menu item descriptions, allergen warnings, and boilerplate nutritional disclaimers. It l +[llm clean] completed=182300 submitted=182394 kept=2378881 rejected=84160 last=keep reason=rewritten +[llm clean] completed=182400 submitted=182495 kept=2378969 rejected=84172 last=keep reason=rewritten +[llm clean] completed=182500 submitted=182595 kept=2379061 rejected=84180 last=keep reason=rewritten +[llm clean] completed=182600 submitted=182695 kept=2379158 rejected=84183 last=keep reason=rewritten +[llm clean] completed=182700 submitted=182795 kept=2379256 rejected=84185 last=keep reason=rewritten +[llm clean] completed=182800 submitted=182895 kept=2379356 rejected=84185 last=keep reason=rewritten +[llm clean] completed=182900 submitted=182995 kept=2379454 rejected=84187 last=reject reason=The text is primarily code snippets (JavaScript/jQuery) with line numbers and formatting artifacts, followed by a brief +[llm clean] completed=183000 submitted=183095 kept=2379550 rejected=84191 last=keep reason=rewritten +[llm clean] completed=183100 submitted=183195 kept=2379649 rejected=84192 last=keep reason=rewritten +[llm clean] completed=183200 submitted=183295 kept=2379745 rejected=84196 last=keep reason=rewritten +[llm clean] completed=183300 submitted=183395 kept=2379842 rejected=84199 last=keep reason=rewritten +[llm clean] completed=183400 submitted=183495 kept=2379941 rejected=84200 last=keep reason=rewritten +[llm clean] completed=183500 submitted=183595 kept=2380039 rejected=84202 last=keep reason=rewritten +[llm clean] completed=183600 submitted=183695 kept=2380139 rejected=84202 last=keep reason=rewritten +[llm clean] completed=183700 submitted=183795 kept=2380239 rejected=84202 last=keep reason=rewritten +[llm clean] completed=183800 submitted=183895 kept=2380334 rejected=84207 last=keep reason=rewritten +[llm clean] completed=183900 submitted=183995 kept=2380432 rejected=84209 last=keep reason=rewritten +[llm clean] completed=184000 submitted=184094 kept=2380531 rejected=84210 last=keep reason=rewritten +[llm clean] completed=184100 submitted=184195 kept=2380630 rejected=84211 last=keep reason=rewritten +[llm clean] completed=184200 submitted=184295 kept=2380728 rejected=84213 last=keep reason=rewritten +[llm clean] completed=184300 submitted=184395 kept=2380826 rejected=84215 last=keep reason=rewritten +[llm clean] completed=184400 submitted=184495 kept=2380924 rejected=84217 last=keep reason=rewritten +[llm clean] completed=184500 submitted=184595 kept=2381020 rejected=84221 last=keep reason=rewritten +[llm clean] completed=184600 submitted=184695 kept=2381113 rejected=84228 last=keep reason=rewritten +[llm clean] completed=184700 submitted=184795 kept=2381211 rejected=84230 last=keep reason=rewritten +[llm clean] completed=184800 submitted=184895 kept=2381303 rejected=84238 last=keep reason=rewritten +[llm clean] completed=184900 submitted=184995 kept=2381400 rejected=84241 last=keep reason=rewritten +[llm clean] completed=185000 submitted=185095 kept=2381499 rejected=84242 last=keep reason=rewritten diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/vllm_qwen36_35b_a3b_gpu6_port8014.log b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/vllm_qwen36_35b_a3b_gpu6_port8014.log new file mode 100644 index 0000000000000000000000000000000000000000..5653b32b9ea43ced82f13c9da3ec5fdc996130cc --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/vllm_qwen36_35b_a3b_gpu6_port8014.log @@ -0,0 +1,19672 @@ +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] █ █ █▄ ▄█ +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.17.1 +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] █▄█▀ █ █ █ █ model /e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀ +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:302] +(APIServer pid=10265) INFO 05-31 21:05:04 [utils.py:238] non-default args: {'model_tag': '/e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B', 'host': '127.0.0.1', 'port': 8014, 'model': '/e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B', 'trust_remote_code': True, 'dtype': 'bfloat16', 'max_model_len': 4096, 'served_model_name': ['qwen36-35b-a3b']} +(APIServer pid=10265) WARNING 05-31 21:05:04 [envs.py:1710] Unknown vLLM environment variable detected: VLLM_BIN +(APIServer pid=10265) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=10265) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=10265) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=10265) Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} +(APIServer pid=10265) Unrecognized keys in `rope_parameters` for 'rope_type'='default': {'mrope_interleaved', 'mrope_section'} +(APIServer pid=10265) INFO 05-31 21:05:15 [model.py:531] Resolved architecture: Qwen3_5MoeForConditionalGeneration +(APIServer pid=10265) INFO 05-31 21:05:15 [model.py:1554] Using max model len 4096 +(APIServer pid=10265) INFO 05-31 21:05:15 [scheduler.py:231] Chunked prefill is enabled with max_num_batched_tokens=8192. +(APIServer pid=10265) INFO 05-31 21:05:16 [config.py:544] Setting attention block size to 1056 tokens to ensure that attention page size is >= mamba page size. +(APIServer pid=10265) INFO 05-31 21:05:16 [config.py:575] Padding mamba page size by 0.76% to ensure that mamba page size and attention page size are exactly equal. +(APIServer pid=10265) INFO 05-31 21:05:16 [vllm.py:747] Asynchronous scheduling is enabled. +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:36 [core.py:101] Initializing a V1 LLM engine (v0.17.1) with config: model='/e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=qwen36-35b-a3b, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []} +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:38 [parallel_state.py:1393] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.82.64.37:45477 backend=nccl +[W531 21:05:38.641596860 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3 +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:38 [parallel_state.py:1715] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:46 [base.py:106] Offloader set to NoopOffloader +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:46 [gpu_model_runner.py:4281] Starting to load model /e2e-data/evad-tech-vla/wanghan58/models/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B... +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:47 [cuda.py:453] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:47 [mm_encoder_attention.py:215] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:47 [qwen3_next.py:157] Using FlashInfer GDN prefill kernel on CUDA compute capability 90 +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:48 [unquantized.py:186] Using TRITON backend for Unquantized MoE +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:48 [cuda.py:405] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(EngineCore_DP0 pid=14857) INFO 05-31 21:05:48 [flash_attn.py:587] Using FlashAttention version 3 +(EngineCore_DP0 pid=14857) :1301: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead. +(EngineCore_DP0 pid=14857) :1301: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead. +(EngineCore_DP0 pid=14857) Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00